Revision 58911
Added by Dimitris Pierrakos almost 4 years ago
PiwikStatsDB.java | ||
---|---|---|
429 | 429 |
sql = "DROP VIEW IF EXISTS result_views_monthly_tmp"; |
430 | 430 |
stmt.executeUpdate(sql); |
431 | 431 |
|
432 |
sql = "DROP TABLE IF EXISTS views_stats_tmp;"; |
|
433 |
stmt.executeUpdate(sql); |
|
434 |
|
|
435 |
sql = "DROP TABLE IF EXISTS downloads_stats_tmp;"; |
|
436 |
stmt.executeUpdate(sql); |
|
437 |
|
|
438 |
sql = "DROP TABLE IF EXISTS process_portal_log_tmp;"; |
|
439 |
stmt.executeUpdate(sql); |
|
440 |
|
|
432 | 441 |
sql = "DROP TABLE IF EXISTS piwiklogtmp;"; |
433 | 442 |
stmt.executeUpdate(sql); |
434 | 443 |
|
... | ... | |
565 | 574 |
int batch_size = 0; |
566 | 575 |
JSONParser parser = new JSONParser(); |
567 | 576 |
for (String jsonFile : jsonFiles) { |
577 |
System.out.println(jsonFile); |
|
568 | 578 |
JSONArray jsonArray = (JSONArray) parser.parse(readHDFSFile(jsonFile)); |
569 | 579 |
|
570 | 580 |
for (Object aJsonArray : jsonArray) { |
... | ... | |
642 | 652 |
|
643 | 653 |
String sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'oaItem\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.result_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;"; |
644 | 654 |
stmt.executeUpdate(sql); |
645 |
stmt.close(); |
|
655 |
// stmt.close();
|
|
646 | 656 |
// con.commit(); |
647 | 657 |
|
648 | 658 |
stmt = con.createStatement(); |
649 | 659 |
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'datasource\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.datasource_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;"; |
650 | 660 |
stmt.executeUpdate(sql); |
651 |
stmt.close(); |
|
661 |
// stmt.close();
|
|
652 | 662 |
// con.commit(); |
653 | 663 |
|
654 | 664 |
stmt = con.createStatement(); |
655 | 665 |
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'organization\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.organization_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;"; |
656 | 666 |
stmt.executeUpdate(sql); |
657 |
stmt.close(); |
|
667 |
// stmt.close();
|
|
658 | 668 |
// con.commit(); |
659 | 669 |
|
660 | 670 |
stmt = con.createStatement(); |
661 | 671 |
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'project\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.project_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;"; |
662 | 672 |
stmt.executeUpdate(sql); |
673 |
// stmt.close(); |
|
674 |
// con.commit(); |
|
663 | 675 |
stmt.close(); |
664 |
// con.commit(); |
|
665 |
|
|
666 |
con.close(); |
|
676 |
ConnectDB.getConnection().commit(); |
|
677 |
ConnectDB.getConnection().close(); |
|
667 | 678 |
} |
668 | 679 |
|
669 | 680 |
private void cleanOAI() throws Exception { |
... | ... | |
894 | 905 |
sql = "insert into pageviews_stats select * from pageviews_stats_tmp;"; |
895 | 906 |
stmt.executeUpdate(sql); |
896 | 907 |
|
897 |
sql = "DROP TABLE IF EXISTS views_stats_tmp;"; |
|
898 |
stmt.executeUpdate(sql); |
|
899 |
|
|
900 |
sql = "DROP TABLE IF EXISTS downloads_stats_tmp;"; |
|
901 |
stmt.executeUpdate(sql); |
|
902 |
|
|
903 |
sql = "DROP TABLE IF EXISTS pageviews_stats_tmp;"; |
|
904 |
stmt.executeUpdate(sql); |
|
905 |
|
|
906 |
sql = "DROP TABLE IF EXISTS process_portal_log_tmp;"; |
|
907 |
stmt.executeUpdate(sql); |
|
908 |
|
|
909 |
|
|
910 | 908 |
stmt.close(); |
911 | 909 |
ConnectDB.getConnection().commit(); |
912 | 910 |
ConnectDB.getConnection().close(); |
Also available in: Unified diff
Split json files