Revision 47256
Added by Tsampikos Livisianos almost 7 years ago
modules/dnet-openaire-stats/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/daos/StatsDAO.java | ||
---|---|---|
160 | 160 |
} |
161 | 161 |
|
162 | 162 |
/** |
163 |
* Create charts. |
|
164 |
* |
|
165 |
* @throws Exception |
|
166 |
*/ |
|
167 |
public void createCharts() throws Exception { |
|
168 |
log.info(" Creating Chart Tables..."); |
|
169 |
Connection con = statsDatasource.getConnection(); |
|
170 |
|
|
171 |
startTime = System.currentTimeMillis(); |
|
172 |
String q = "{call create_charts()}"; |
|
173 |
|
|
174 |
CallableStatement st = con.prepareCall(q); |
|
175 |
st.execute(); |
|
176 |
|
|
177 |
st.close(); |
|
178 |
con.close(); |
|
179 |
|
|
180 |
endtime = System.currentTimeMillis(); |
|
181 |
log.info("Time to create chart tables: " + ((endtime - startTime) / 60000) + " minutes "); |
|
182 |
} |
|
183 |
|
|
184 |
/** |
|
185 |
* Create chart indexes. |
|
186 |
* |
|
187 |
* @throws Exception |
|
188 |
*/ |
|
189 |
public void createChartIndexes() throws Exception { |
|
190 |
log.info(" Create Chart Indexes..."); |
|
191 |
Connection con = statsDatasource.getConnection(); |
|
192 |
|
|
193 |
startTime = System.currentTimeMillis(); |
|
194 |
String q = "{call create_chart_indexes()}"; |
|
195 |
|
|
196 |
CallableStatement st = con.prepareCall(q); |
|
197 |
st.execute(); |
|
198 |
|
|
199 |
st.close(); |
|
200 |
con.close(); |
|
201 |
|
|
202 |
endtime = System.currentTimeMillis(); |
|
203 |
log.info("Time to create chart indexes : " + ((endtime - startTime) / 60000) + " minutes "); |
|
204 |
} |
|
205 |
|
|
206 |
/** |
|
163 | 207 |
* Builds indexes. |
164 | 208 |
* |
165 | 209 |
* @throws Exception |
modules/dnet-openaire-stats/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/statsExport/drivers/DBDriver.java | ||
---|---|---|
43 | 43 |
statsDao.buildIndexes(); |
44 | 44 |
statsDao.buildViews(); |
45 | 45 |
statsDao.executeExtraInserts(); |
46 |
statsDao.createCharts(); |
|
47 |
statsDao.createChartIndexes(); |
|
46 | 48 |
|
47 | 49 |
} catch (Exception e) { |
48 | 50 |
log.error("Error while finalizing SQL DB", e); |
modules/dnet-openaire-stats/trunk/src/main/resources/eu/dnetlib/data/mapreduce/hbase/statsExport/stats_db_schema.sql | ||
---|---|---|
896 | 896 |
|
897 | 897 |
|
898 | 898 |
END;$$; |
899 |
-- |
|
900 |
-- Name: create_charts(); Type: FUNCTION; Schema: shadow; Owner: sqoop |
|
901 |
-- |
|
899 | 902 |
|
903 |
CREATE or replace FUNCTION "shadow".create_charts() RETURNS void |
|
904 |
LANGUAGE plpgsql |
|
905 |
AS $$ |
|
906 |
|
|
907 |
BEGIN |
|
908 |
|
|
909 |
SELECT rd.datasource, rc.type, count(distinct rd.id) INTO "shadow".chart_datasource_pubs FROM "shadow".result_datasources rd, "shadow".result_classifications rc WHERE rd.id=rc.id GROUP BY rd.datasource, rc.type; |
|
910 |
SELECT rd.datasource, r.year, count(distinct rd.id) INTO "shadow".chart_datasource_year FROM "shadow".result r, "shadow".result_datasources rd WHERE rd.id=r.id GROUP By rd.datasource, r.year; |
|
911 |
SELECT rd.datasource, p.funder, count(distinct rd.id) INTO "shadow".chart_datasource_funders FROM "shadow".result_datasources rd, "shadow".project p, "shadow".project_results pr WHERE p.id=pr.id AND pr.result=rd.id GROUP BY rd.datasource, p.funder; |
|
912 |
SELECT rd.datasource, p.title, count(distinct rd.id) INTO "shadow".chart_datasource_projects FROM "shadow".result_datasources rd, "shadow".project p, "shadow".project_results pr WHERE p.id=pr.id AND pr.result=rd.id GROUP BY rd.datasource, p.title; |
|
913 |
|
|
914 |
END;$$; |
|
915 |
|
|
900 | 916 |
-- |
917 |
-- Name: create_charts(); Type: FUNCTION; Schema: shadow; Owner: sqoop |
|
918 |
-- |
|
919 |
|
|
920 |
CREATE or replace FUNCTION "shadow".create_chart_indexes() RETURNS void |
|
921 |
LANGUAGE plpgsql |
|
922 |
AS $$ |
|
923 |
|
|
924 |
BEGIN |
|
925 |
|
|
926 |
CREATE INDEX chart_datasource_pubs_datasource ON "shadow".chart_datasource_pubs USING btree(datasource); |
|
927 |
CREATE INDEX chart_datasource_year_datasource ON "shadow".chart_datasource_year USING btree(datasource); |
|
928 |
CREATE INDEX chart_datasource_funders_datasource ON "shadow".chart_datasource_funders USING btree(datasource); |
|
929 |
CREATE INDEX chart_datasource_projects_datasource ON "shadow".chart_datasource_projects USING btree(datasource); |
|
930 |
|
|
931 |
|
|
932 |
END;$$; |
|
933 |
|
|
934 |
-- |
|
901 | 935 |
-- Name: clean_tables(); Type: FUNCTION; Schema: shadow; Owner: sqoop |
902 | 936 |
-- |
903 | 937 |
|
Also available in: Unified diff
pre-calculate datasource stats