Project

General

Profile

« Previous | Next » 

Revision 47256

Added by Tsampikos Livisianos almost 7 years ago

pre-calculate datasource stats

View differences:

modules/dnet-openaire-stats/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/daos/StatsDAO.java
160 160
    }
161 161

  
162 162
    /**
163
     * Create charts.
164
     *
165
     * @throws Exception
166
     */
167
    public void createCharts() throws Exception {
168
        log.info(" Creating Chart Tables...");
169
        Connection con = statsDatasource.getConnection();
170

  
171
        startTime = System.currentTimeMillis();
172
        String q = "{call create_charts()}";
173

  
174
        CallableStatement st = con.prepareCall(q);
175
        st.execute();
176

  
177
        st.close();
178
        con.close();
179

  
180
        endtime = System.currentTimeMillis();
181
        log.info("Time to create chart tables: " + ((endtime - startTime) / 60000) + " minutes ");
182
    }
183

  
184
    /**
185
     * Create chart indexes.
186
     *
187
     * @throws Exception
188
     */
189
    public void createChartIndexes() throws Exception {
190
        log.info(" Create Chart Indexes...");
191
        Connection con = statsDatasource.getConnection();
192

  
193
        startTime = System.currentTimeMillis();
194
        String q = "{call create_chart_indexes()}";
195

  
196
        CallableStatement st = con.prepareCall(q);
197
        st.execute();
198

  
199
        st.close();
200
        con.close();
201

  
202
        endtime = System.currentTimeMillis();
203
        log.info("Time to create chart indexes : " + ((endtime - startTime) / 60000) + " minutes ");
204
    }
205

  
206
    /**
163 207
     * Builds indexes.
164 208
     *
165 209
     * @throws Exception
modules/dnet-openaire-stats/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/drivers/DBDriver.java
43 43
            statsDao.buildIndexes();
44 44
            statsDao.buildViews();
45 45
            statsDao.executeExtraInserts();
46
            statsDao.createCharts();
47
            statsDao.createChartIndexes();
46 48

  
47 49
        } catch (Exception e) {
48 50
            log.error("Error while finalizing SQL DB", e);
modules/dnet-openaire-stats/trunk/src/main/resources/eu/dnetlib/data/mapreduce/hbase/statsExport/stats_db_schema.sql
896 896

  
897 897

  
898 898
  END;$$;
899
--
900
-- Name: create_charts(); Type: FUNCTION; Schema: shadow; Owner: sqoop
901
--
899 902

  
903
CREATE or replace FUNCTION "shadow".create_charts() RETURNS void
904
    LANGUAGE plpgsql
905
    AS $$
906

  
907
BEGIN
908

  
909
SELECT rd.datasource, rc.type, count(distinct rd.id) INTO "shadow".chart_datasource_pubs FROM "shadow".result_datasources rd, "shadow".result_classifications rc WHERE rd.id=rc.id GROUP BY rd.datasource, rc.type;
910
SELECT rd.datasource, r.year, count(distinct rd.id) INTO "shadow".chart_datasource_year FROM "shadow".result r, "shadow".result_datasources rd WHERE rd.id=r.id GROUP By rd.datasource, r.year;
911
SELECT rd.datasource, p.funder, count(distinct rd.id) INTO "shadow".chart_datasource_funders FROM "shadow".result_datasources rd, "shadow".project p, "shadow".project_results pr WHERE p.id=pr.id AND pr.result=rd.id GROUP BY rd.datasource, p.funder;
912
SELECT rd.datasource, p.title, count(distinct rd.id) INTO "shadow".chart_datasource_projects FROM "shadow".result_datasources rd, "shadow".project p, "shadow".project_results pr WHERE p.id=pr.id AND pr.result=rd.id GROUP BY rd.datasource, p.title;
913

  
914
  END;$$;
915

  
900 916
--
917
-- Name: create_charts(); Type: FUNCTION; Schema: shadow; Owner: sqoop
918
--
919

  
920
CREATE or replace FUNCTION "shadow".create_chart_indexes() RETURNS void
921
    LANGUAGE plpgsql
922
    AS $$
923

  
924
BEGIN
925

  
926
CREATE INDEX chart_datasource_pubs_datasource ON "shadow".chart_datasource_pubs USING btree(datasource);
927
CREATE INDEX chart_datasource_year_datasource ON "shadow".chart_datasource_year USING btree(datasource);
928
CREATE INDEX chart_datasource_funders_datasource ON "shadow".chart_datasource_funders USING btree(datasource);
929
CREATE INDEX chart_datasource_projects_datasource ON "shadow".chart_datasource_projects USING btree(datasource);
930

  
931

  
932
  END;$$;
933

  
934
--
901 935
-- Name: clean_tables(); Type: FUNCTION; Schema: shadow; Owner: sqoop
902 936
--
903 937

  

Also available in: Unified diff