429 |
429 |
sql = "DROP VIEW IF EXISTS result_views_monthly_tmp";
|
430 |
430 |
stmt.executeUpdate(sql);
|
431 |
431 |
|
|
432 |
sql = "DROP TABLE IF EXISTS views_stats_tmp;";
|
|
433 |
stmt.executeUpdate(sql);
|
|
434 |
|
|
435 |
sql = "DROP TABLE IF EXISTS downloads_stats_tmp;";
|
|
436 |
stmt.executeUpdate(sql);
|
|
437 |
|
|
438 |
sql = "DROP TABLE IF EXISTS process_portal_log_tmp;";
|
|
439 |
stmt.executeUpdate(sql);
|
|
440 |
|
432 |
441 |
sql = "DROP TABLE IF EXISTS piwiklogtmp;";
|
433 |
442 |
stmt.executeUpdate(sql);
|
434 |
443 |
|
... | ... | |
565 |
574 |
int batch_size = 0;
|
566 |
575 |
JSONParser parser = new JSONParser();
|
567 |
576 |
for (String jsonFile : jsonFiles) {
|
|
577 |
System.out.println(jsonFile);
|
568 |
578 |
JSONArray jsonArray = (JSONArray) parser.parse(readHDFSFile(jsonFile));
|
569 |
579 |
|
570 |
580 |
for (Object aJsonArray : jsonArray) {
|
... | ... | |
642 |
652 |
|
643 |
653 |
String sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'oaItem\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.result_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
|
644 |
654 |
stmt.executeUpdate(sql);
|
645 |
|
stmt.close();
|
|
655 |
// stmt.close();
|
646 |
656 |
// con.commit();
|
647 |
657 |
|
648 |
658 |
stmt = con.createStatement();
|
649 |
659 |
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'datasource\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.datasource_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
|
650 |
660 |
stmt.executeUpdate(sql);
|
651 |
|
stmt.close();
|
|
661 |
// stmt.close();
|
652 |
662 |
// con.commit();
|
653 |
663 |
|
654 |
664 |
stmt = con.createStatement();
|
655 |
665 |
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'organization\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.organization_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
|
656 |
666 |
stmt.executeUpdate(sql);
|
657 |
|
stmt.close();
|
|
667 |
// stmt.close();
|
658 |
668 |
// con.commit();
|
659 |
669 |
|
660 |
670 |
stmt = con.createStatement();
|
661 |
671 |
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'project\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.project_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
|
662 |
672 |
stmt.executeUpdate(sql);
|
|
673 |
// stmt.close();
|
|
674 |
// con.commit();
|
663 |
675 |
stmt.close();
|
664 |
|
// con.commit();
|
665 |
|
|
666 |
|
con.close();
|
|
676 |
ConnectDB.getConnection().commit();
|
|
677 |
ConnectDB.getConnection().close();
|
667 |
678 |
}
|
668 |
679 |
|
669 |
680 |
private void cleanOAI() throws Exception {
|
... | ... | |
894 |
905 |
sql = "insert into pageviews_stats select * from pageviews_stats_tmp;";
|
895 |
906 |
stmt.executeUpdate(sql);
|
896 |
907 |
|
897 |
|
sql = "DROP TABLE IF EXISTS views_stats_tmp;";
|
898 |
|
stmt.executeUpdate(sql);
|
899 |
|
|
900 |
|
sql = "DROP TABLE IF EXISTS downloads_stats_tmp;";
|
901 |
|
stmt.executeUpdate(sql);
|
902 |
|
|
903 |
|
sql = "DROP TABLE IF EXISTS pageviews_stats_tmp;";
|
904 |
|
stmt.executeUpdate(sql);
|
905 |
|
|
906 |
|
sql = "DROP TABLE IF EXISTS process_portal_log_tmp;";
|
907 |
|
stmt.executeUpdate(sql);
|
908 |
|
|
909 |
|
|
910 |
908 |
stmt.close();
|
911 |
909 |
ConnectDB.getConnection().commit();
|
912 |
910 |
ConnectDB.getConnection().close();
|
Split json files