1
|
package eu.dnetlib.usagestats.export;
|
2
|
/**
|
3
|
*
|
4
|
* @author dpie
|
5
|
*/
|
6
|
import java.io.*;
|
7
|
//import java.io.BufferedReader;
|
8
|
//import java.io.InputStreamReader;
|
9
|
import java.net.URL;
|
10
|
import java.net.URLConnection;
|
11
|
import java.sql.ResultSet;
|
12
|
import java.text.SimpleDateFormat;
|
13
|
import java.util.Date;
|
14
|
import java.util.Calendar;
|
15
|
|
16
|
import java.sql.Connection;
|
17
|
import java.sql.DriverManager;
|
18
|
import java.sql.PreparedStatement;
|
19
|
import java.sql.Statement;
|
20
|
|
21
|
import org.json.simple.JSONArray;
|
22
|
import org.json.simple.JSONObject;
|
23
|
import org.json.simple.parser.JSONParser;
|
24
|
|
25
|
import org.apache.log4j.Logger;
|
26
|
|
27
|
/**
|
28
|
* Created by dpie on 20/01/2020.
|
29
|
*/
|
30
|
public class IrusStats {
|
31
|
|
32
|
private String dbUrl;
|
33
|
private String dbSchema;
|
34
|
private String dbUserName;
|
35
|
private String dbPassword;
|
36
|
private String irusUKURL;
|
37
|
|
38
|
private Connection conn = null;
|
39
|
private Statement stmt = null;
|
40
|
|
41
|
private final Logger log = Logger.getLogger(this.getClass());
|
42
|
|
43
|
public IrusStats(String dbUrl, String dbUsername, String dbPassword) throws Exception {
|
44
|
this.dbUrl = dbUrl;
|
45
|
this.dbSchema = "shadow";
|
46
|
this.dbUserName = dbUsername;
|
47
|
this.dbPassword = dbPassword;
|
48
|
|
49
|
connectDB();
|
50
|
createTables();
|
51
|
}
|
52
|
|
53
|
public IrusStats(String irusUKURL) throws Exception {
|
54
|
this.irusUKURL = irusUKURL;
|
55
|
if (ConnectDB.DB_CONNECTION.isClosed()) {
|
56
|
connectDB();
|
57
|
}
|
58
|
createTables();
|
59
|
createTmpTables();
|
60
|
}
|
61
|
|
62
|
private void connectDB() throws Exception {
|
63
|
try {
|
64
|
ConnectDB connectDB = new ConnectDB();
|
65
|
} catch (Exception e) {
|
66
|
log.error("Connect to db failed: " + e);
|
67
|
throw new Exception("Failed to connect to db: " + e.toString(), e);
|
68
|
}
|
69
|
}
|
70
|
|
71
|
private void connectDBOld() throws Exception {
|
72
|
try {
|
73
|
Class.forName("org.postgresql.Driver");
|
74
|
conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword);
|
75
|
|
76
|
stmt = conn.createStatement();
|
77
|
String sqlSetSearchPath = "SET search_path TO " + dbSchema + ";";
|
78
|
stmt.executeUpdate(sqlSetSearchPath);
|
79
|
|
80
|
log.info("Opened database successfully");
|
81
|
|
82
|
} catch (Exception e) {
|
83
|
log.error("Connect to db failed: " + e);
|
84
|
throw new Exception("Failed to connect to db: " + e.toString(), e);
|
85
|
}
|
86
|
}
|
87
|
|
88
|
private void createTables() throws Exception {
|
89
|
if (ConnectDB.DB_CONNECTION.isClosed()) {
|
90
|
connectDB();
|
91
|
}
|
92
|
|
93
|
try {
|
94
|
|
95
|
stmt = ConnectDB.DB_CONNECTION.createStatement();
|
96
|
String sqlCreateTableSushiLog = "CREATE TABLE IF NOT EXISTS sushilog(source TEXT, repository TEXT, rid TEXT, date TEXT, metric_type TEXT, count INT, PRIMARY KEY(source, repository, rid, date, metric_type));";
|
97
|
stmt.executeUpdate(sqlCreateTableSushiLog);
|
98
|
|
99
|
stmt.executeUpdate("CREATE TABLE IF NOT EXISTS public.sushilog AS TABLE sushilog;");
|
100
|
|
101
|
String sqlCopyPublicSushiLog = "INSERT INTO sushilog SELECT * FROM public.sushilog;";
|
102
|
stmt.executeUpdate(sqlCopyPublicSushiLog);
|
103
|
|
104
|
String sqlcreateRuleSushiLog = "CREATE OR REPLACE RULE ignore_duplicate_inserts AS "
|
105
|
+ " ON INSERT TO sushilog "
|
106
|
+ " WHERE (EXISTS ( SELECT sushilog.source, sushilog.repository,"
|
107
|
+ "sushilog.rid, sushilog.date "
|
108
|
+ "FROM sushilog "
|
109
|
+ "WHERE sushilog.source = new.source AND sushilog.repository = new.repository AND sushilog.rid = new.rid AND sushilog.date = new.date AND sushilog.metric_type = new.metric_type)) DO INSTEAD NOTHING;";
|
110
|
stmt.executeUpdate(sqlcreateRuleSushiLog);
|
111
|
|
112
|
stmt.close();
|
113
|
ConnectDB.DB_CONNECTION.close();
|
114
|
log.info("Sushi Tables Created");
|
115
|
} catch (Exception e) {
|
116
|
log.error("Failed to create tables: " + e);
|
117
|
throw new Exception("Failed to create tables: " + e.toString(), e);
|
118
|
}
|
119
|
}
|
120
|
|
121
|
private void createTmpTables() throws Exception {
|
122
|
if (ConnectDB.DB_CONNECTION.isClosed()) {
|
123
|
connectDB();
|
124
|
}
|
125
|
|
126
|
try {
|
127
|
|
128
|
stmt = ConnectDB.DB_CONNECTION.createStatement();
|
129
|
String sqlCreateTableSushiLog = "CREATE TABLE IF NOT EXISTS sushilogtmp(source TEXT, repository TEXT, rid TEXT, date TEXT, metric_type TEXT, count INT, PRIMARY KEY(source, repository, rid, date, metric_type));";
|
130
|
stmt.executeUpdate(sqlCreateTableSushiLog);
|
131
|
|
132
|
//stmt.executeUpdate("CREATE TABLE IF NOT EXISTS public.sushilog AS TABLE sushilog;");
|
133
|
//String sqlCopyPublicSushiLog = "INSERT INTO sushilog SELECT * FROM public.sushilog;";
|
134
|
//stmt.executeUpdate(sqlCopyPublicSushiLog);
|
135
|
String sqlcreateRuleSushiLog = "CREATE OR REPLACE RULE ignore_duplicate_inserts AS "
|
136
|
+ " ON INSERT TO sushilogtmp "
|
137
|
+ " WHERE (EXISTS ( SELECT sushilog.source, sushilog.repository,"
|
138
|
+ "sushilog.rid, sushilog.date "
|
139
|
+ "FROM sushilog "
|
140
|
+ "WHERE sushilog.source = new.source AND sushilog.repository = new.repository AND sushilog.rid = new.rid AND sushilog.date = new.date AND sushilog.metric_type = new.metric_type)) DO INSTEAD NOTHING;";
|
141
|
stmt.executeUpdate(sqlcreateRuleSushiLog);
|
142
|
|
143
|
stmt.close();
|
144
|
ConnectDB.DB_CONNECTION.close();
|
145
|
log.info("Sushi Tmp Tables Created");
|
146
|
} catch (Exception e) {
|
147
|
log.error("Failed to create tables: " + e);
|
148
|
throw new Exception("Failed to create tables: " + e.toString(), e);
|
149
|
}
|
150
|
}
|
151
|
|
152
|
public void irusStats() throws Exception {
|
153
|
if (ConnectDB.DB_CONNECTION.isClosed()) {
|
154
|
connectDB();
|
155
|
}
|
156
|
|
157
|
stmt = conn.createStatement();
|
158
|
conn.setAutoCommit(false);
|
159
|
|
160
|
//String sql = "INSERT INTO sushi_result_downloads SELECT s.source, d.id AS repository, ro.id, s.date, s.count FROM sushilog s, datasource_oids d, result_oids ro WHERE s.repository=d.orid AND s.oai=ro.orid AND metric_type='ft_total'";
|
161
|
//String sql = "SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count INTO downloads_stats FROM sushilog s, datasource_oids d, result_oids ro WHERE s.repository=d.orid AND s.oai=ro.orid AND metric_type='ft_total'";
|
162
|
//String sql = "INSERT INTO downloads_stats SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count FROM sushilog s, datasource_oids d, result_oids ro WHERE s.repository=d.orid AND s.oai=ro.orid AND metric_type='ft_total';";
|
163
|
String sql = "INSERT INTO downloads_stats SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count, '0' FROM sushilogtmp s, datasource_oids d, result_oids ro WHERE s.repository=d.orid AND s.rid=ro.orid AND metric_type='ft_total' AND s.source='IRUS-UK';";
|
164
|
stmt.executeUpdate(sql);
|
165
|
|
166
|
stmt.close();
|
167
|
|
168
|
sql = "Insert into sushilog select * from sushilogtmp;";
|
169
|
stmt.executeUpdate(sql);
|
170
|
|
171
|
sql = "drop table sushilogtmp;";
|
172
|
stmt.executeUpdate(sql);
|
173
|
|
174
|
ConnectDB.DB_CONNECTION.commit();
|
175
|
ConnectDB.DB_CONNECTION.close();
|
176
|
}
|
177
|
|
178
|
public void processIrusRRReport() throws Exception {
|
179
|
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM");
|
180
|
//String reportUrl = "https://irus.jisc.ac.uk" + "/api/sushilite/v1_7/GetReport/?Report=RR1&Release=4&RequestorID=OpenAIRE&BeginDate=2016-01&EndDate=" + simpleDateFormat.format(new Date()) + "&RepositoryIdentifier=&ItemDataType=&NewJiscBand=&Granularity=Monthly&Callback=";
|
181
|
String reportUrl = irusUKURL+"GetReport/?Report=RR1&Release=4&RequestorID=OpenAIRE&BeginDate=2016-01&EndDate=" + simpleDateFormat.format(new Date()) + "&RepositoryIdentifier=&ItemDataType=&NewJiscBand=&Granularity=Monthly&Callback=";
|
182
|
|
183
|
log.info("Getting Irus report: " + reportUrl);
|
184
|
System.out.println("Getting Irus report: " + reportUrl);
|
185
|
|
186
|
String text = getJson(reportUrl, "", "");
|
187
|
|
188
|
log.info("Report: " + text);
|
189
|
|
190
|
JSONParser parser = new JSONParser();
|
191
|
JSONObject jsonObject = (JSONObject) parser.parse(text);
|
192
|
jsonObject = (JSONObject) jsonObject.get("ReportResponse");
|
193
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
194
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
195
|
jsonObject = (JSONObject) jsonObject.get("Customer");
|
196
|
JSONArray jsonArray = (JSONArray) jsonObject.get("ReportItems");
|
197
|
int i = 0;
|
198
|
for (Object aJsonArray : jsonArray) {
|
199
|
JSONObject jsonObjectRow = (JSONObject) aJsonArray;
|
200
|
JSONArray itemIdentifier = (JSONArray) jsonObjectRow.get("ItemIdentifier");
|
201
|
for (Object identifier : itemIdentifier) {
|
202
|
JSONObject opendoar = (JSONObject) identifier;
|
203
|
if (opendoar.get("Type").toString().equals("OpenDOAR")) {
|
204
|
//System.out.println(i + ": " + opendoar.get("Value").toString());
|
205
|
log.info(i + ": " + opendoar.get("Value").toString());
|
206
|
i++;
|
207
|
processIrusIRReport(opendoar.get("Value").toString());
|
208
|
break;
|
209
|
}
|
210
|
}
|
211
|
//break;
|
212
|
}
|
213
|
}
|
214
|
|
215
|
private void processIrusIRReport(String opendoar) throws Exception {
|
216
|
System.out.println(opendoar);
|
217
|
if (ConnectDB.DB_CONNECTION.isClosed()) {
|
218
|
connectDB();
|
219
|
}
|
220
|
|
221
|
ConnectDB.DB_CONNECTION.setAutoCommit(false);
|
222
|
|
223
|
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM");
|
224
|
|
225
|
Calendar start = Calendar.getInstance();
|
226
|
start.set(Calendar.YEAR, 2016);
|
227
|
start.set(Calendar.MONTH, Calendar.JANUARY);
|
228
|
//start.setTime(simpleDateFormat.parse("2016-01"));
|
229
|
|
230
|
Calendar end = Calendar.getInstance();
|
231
|
end.add(Calendar.DAY_OF_MONTH, -1);
|
232
|
|
233
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
|
234
|
PreparedStatement st = conn.prepareStatement("SELECT max(date) FROM sushilog WHERE repository=?;");
|
235
|
st.setString(1, "opendoar____::" + opendoar);
|
236
|
ResultSet rs_date = st.executeQuery();
|
237
|
while (rs_date.next()) {
|
238
|
if (rs_date.getString(1) != null && !rs_date.getString(1).equals("null") && !rs_date.getString(1).equals("")) {
|
239
|
start.setTime(sdf.parse(rs_date.getString(1)));
|
240
|
}
|
241
|
}
|
242
|
rs_date.close();
|
243
|
PreparedStatement preparedStatement = ConnectDB.DB_CONNECTION.prepareStatement("INSERT INTO sushilogtmp (source, repository, rid, date, metric_type, count) VALUES (?,?,?,?,?,?)");
|
244
|
int batch_size = 0;
|
245
|
|
246
|
while (start.before(end)) {
|
247
|
//log.info("date: " + simpleDateFormat.format(start.getTime()));
|
248
|
String reportUrl = this.irusUKURL+"GetReport/?Report=IR1&Release=4&RequestorID=OpenAIRE&BeginDate=" + simpleDateFormat.format(start.getTime()) + "&EndDate=" + simpleDateFormat.format(start.getTime()) + "&RepositoryIdentifier=opendoar%3A" + opendoar + "&ItemIdentifier=&ItemDataType=&hasDOI=&Granularity=Monthly&Callback=";
|
249
|
start.add(Calendar.MONTH, 1);
|
250
|
|
251
|
String text = getJson(reportUrl, "", "");
|
252
|
if (text == null) {
|
253
|
continue;
|
254
|
}
|
255
|
|
256
|
JSONParser parser = new JSONParser();
|
257
|
JSONObject jsonObject = (JSONObject) parser.parse(text);
|
258
|
jsonObject = (JSONObject) jsonObject.get("ReportResponse");
|
259
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
260
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
261
|
jsonObject = (JSONObject) jsonObject.get("Customer");
|
262
|
JSONArray jsonArray = (JSONArray) jsonObject.get("ReportItems");
|
263
|
if (jsonArray == null) {
|
264
|
continue;
|
265
|
}
|
266
|
String oai = "";
|
267
|
for (Object aJsonArray : jsonArray) {
|
268
|
JSONObject jsonObjectRow = (JSONObject) aJsonArray;
|
269
|
JSONArray itemIdentifier = (JSONArray) jsonObjectRow.get("ItemIdentifier");
|
270
|
for (Object identifier : itemIdentifier) {
|
271
|
JSONObject oaiPmh = (JSONObject) identifier;
|
272
|
if (oaiPmh.get("Type").toString().equals("OAI")) {
|
273
|
oai = oaiPmh.get("Value").toString();
|
274
|
//System.out.println("OAI: " + oai);
|
275
|
break;
|
276
|
}
|
277
|
}
|
278
|
|
279
|
JSONArray itemPerformance = (JSONArray) jsonObjectRow.get("ItemPerformance");
|
280
|
String period;
|
281
|
String type;
|
282
|
String count;
|
283
|
for (Object perf : itemPerformance) {
|
284
|
JSONObject performance = (JSONObject) perf;
|
285
|
JSONObject periodObj = (JSONObject) performance.get("Period");
|
286
|
period = periodObj.get("Begin").toString();
|
287
|
JSONObject instanceObj = (JSONObject) performance.get("Instance");
|
288
|
type = instanceObj.get("MetricType").toString();
|
289
|
count = instanceObj.get("Count").toString();
|
290
|
//System.out.println(oai + " : " + period + " : " + count);
|
291
|
|
292
|
preparedStatement.setString(1, "IRUS-UK");
|
293
|
preparedStatement.setString(2, "opendoar____::" + opendoar);
|
294
|
preparedStatement.setString(3, oai);
|
295
|
preparedStatement.setString(4, period);
|
296
|
preparedStatement.setString(5, type);
|
297
|
preparedStatement.setInt(6, Integer.parseInt(count));
|
298
|
preparedStatement.addBatch();
|
299
|
batch_size++;
|
300
|
if (batch_size == 10000) {
|
301
|
preparedStatement.executeBatch();
|
302
|
ConnectDB.DB_CONNECTION.commit();
|
303
|
batch_size = 0;
|
304
|
}
|
305
|
}
|
306
|
//break;
|
307
|
}
|
308
|
//break;
|
309
|
}
|
310
|
|
311
|
preparedStatement.executeBatch();
|
312
|
ConnectDB.DB_CONNECTION.commit();
|
313
|
ConnectDB.DB_CONNECTION.close();
|
314
|
}
|
315
|
|
316
|
public void processIrusIRReport(String opendoar, String startDate) throws Exception {
|
317
|
if (conn.isClosed()) {
|
318
|
connectDB();
|
319
|
}
|
320
|
|
321
|
conn.setAutoCommit(false);
|
322
|
|
323
|
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM");
|
324
|
|
325
|
Calendar start = Calendar.getInstance();
|
326
|
start.set(Calendar.YEAR, 2016);
|
327
|
start.set(Calendar.MONTH, Calendar.JANUARY);
|
328
|
//start.setTime(simpleDateFormat.parse("2016-01"));
|
329
|
|
330
|
Calendar end = Calendar.getInstance();
|
331
|
end.add(Calendar.DAY_OF_MONTH, -1);
|
332
|
|
333
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
|
334
|
start.setTime(sdf.parse(startDate));
|
335
|
|
336
|
String createTablesQuery = "-- Table: shadow.sushilog" + opendoar + "\n"
|
337
|
+ "\n"
|
338
|
+ "-- DROP TABLE shadow.sushilog" + opendoar + ";\n"
|
339
|
+ "\n"
|
340
|
+ "CREATE TABLE shadow.sushilog" + opendoar + "\n"
|
341
|
+ "(\n"
|
342
|
+ " source text COLLATE pg_catalog.\"default\" NOT NULL,\n"
|
343
|
+ " repository text COLLATE pg_catalog.\"default\" NOT NULL,\n"
|
344
|
+ " rid text COLLATE pg_catalog.\"default\" NOT NULL,\n"
|
345
|
+ " date text COLLATE pg_catalog.\"default\" NOT NULL,\n"
|
346
|
+ " metric_type text COLLATE pg_catalog.\"default\" NOT NULL,\n"
|
347
|
+ " count integer,\n"
|
348
|
+ " CONSTRAINT sushilog" + opendoar + "_pkey PRIMARY KEY (source, repository, rid, date, metric_type)\n"
|
349
|
+ " USING INDEX TABLESPACE index_storage\n"
|
350
|
+ ")\n"
|
351
|
+ "\n"
|
352
|
+ "TABLESPACE pg_default;\n"
|
353
|
+ "\n"
|
354
|
+ "ALTER TABLE shadow.sushilog" + opendoar + "\n"
|
355
|
+ " OWNER to sqoop;\n"
|
356
|
+ "\n"
|
357
|
+ "-- Rule: ignore_duplicate_inserts ON shadow.sushilog" + opendoar + "\n"
|
358
|
+ "\n"
|
359
|
+ "-- DROP Rule ignore_duplicate_inserts ON shadow.sushilog" + opendoar + ";\n"
|
360
|
+ "\n"
|
361
|
+ "CREATE OR REPLACE RULE ignore_duplicate_inserts AS\n"
|
362
|
+ " ON INSERT TO shadow.sushilog" + opendoar + "\n"
|
363
|
+ " WHERE (EXISTS ( SELECT sushilog" + opendoar + ".source,\n"
|
364
|
+ " sushilog" + opendoar + ".repository,\n"
|
365
|
+ " sushilog" + opendoar + ".rid,\n"
|
366
|
+ " sushilog" + opendoar + ".date\n"
|
367
|
+ " FROM sushilog" + opendoar + "\n"
|
368
|
+ " WHERE sushilog" + opendoar + ".source = new.source AND sushilog" + opendoar + ".repository = new.repository AND sushilog" + opendoar + ".rid = new.rid AND sushilog" + opendoar + ".date = new.date AND sushilog" + opendoar + ".metric_type = new.metric_type))\n"
|
369
|
+ " DO INSTEAD\n"
|
370
|
+ "NOTHING;";
|
371
|
|
372
|
Statement stCreateTables = conn.createStatement();
|
373
|
stCreateTables.execute(createTablesQuery);
|
374
|
conn.commit();
|
375
|
|
376
|
PreparedStatement preparedStatement = conn.prepareStatement("INSERT INTO sushilog" + opendoar + " (source, repository, rid, date, metric_type, count) VALUES (?,?,?,?,?,?)");
|
377
|
int batch_size = 0;
|
378
|
|
379
|
while (start.before(end)) {
|
380
|
//log.info("date: " + simpleDateFormat.format(start.getTime()));
|
381
|
String reportUrl = "https://irus.jisc.ac.uk/api/sushilite/v1_7/GetReport/?Report=IR1&Release=4&RequestorID=OpenAIRE&BeginDate=" + simpleDateFormat.format(start.getTime()) + "&EndDate=2019-10-31&RepositoryIdentifier=opendoar%3A" + opendoar + "&ItemIdentifier=&ItemDataType=&hasDOI=&Granularity=Monthly&Callback=";
|
382
|
start.add(Calendar.MONTH, 1);
|
383
|
|
384
|
String text = getJson(reportUrl, "", "");
|
385
|
if (text == null) {
|
386
|
continue;
|
387
|
}
|
388
|
|
389
|
JSONParser parser = new JSONParser();
|
390
|
JSONObject jsonObject = (JSONObject) parser.parse(text);
|
391
|
jsonObject = (JSONObject) jsonObject.get("ReportResponse");
|
392
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
393
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
394
|
jsonObject = (JSONObject) jsonObject.get("Customer");
|
395
|
JSONArray jsonArray = (JSONArray) jsonObject.get("ReportItems");
|
396
|
if (jsonArray == null) {
|
397
|
continue;
|
398
|
}
|
399
|
String oai = "";
|
400
|
for (Object aJsonArray : jsonArray) {
|
401
|
JSONObject jsonObjectRow = (JSONObject) aJsonArray;
|
402
|
JSONArray itemIdentifier = (JSONArray) jsonObjectRow.get("ItemIdentifier");
|
403
|
for (Object identifier : itemIdentifier) {
|
404
|
JSONObject oaiPmh = (JSONObject) identifier;
|
405
|
if (oaiPmh.get("Type").toString().equals("OAI")) {
|
406
|
oai = oaiPmh.get("Value").toString();
|
407
|
//System.out.println("OAI: " + oai);
|
408
|
break;
|
409
|
}
|
410
|
}
|
411
|
|
412
|
JSONArray itemPerformance = (JSONArray) jsonObjectRow.get("ItemPerformance");
|
413
|
String period;
|
414
|
String type;
|
415
|
String count;
|
416
|
for (Object perf : itemPerformance) {
|
417
|
JSONObject performance = (JSONObject) perf;
|
418
|
JSONObject periodObj = (JSONObject) performance.get("Period");
|
419
|
period = periodObj.get("Begin").toString();
|
420
|
JSONObject instanceObj = (JSONObject) performance.get("Instance");
|
421
|
type = instanceObj.get("MetricType").toString();
|
422
|
count = instanceObj.get("Count").toString();
|
423
|
//System.out.println(oai + " : " + period + " : " + count);
|
424
|
|
425
|
preparedStatement.setString(1, "IRUS-UK");
|
426
|
preparedStatement.setString(2, "opendoar____::" + opendoar);
|
427
|
preparedStatement.setString(3, oai);
|
428
|
preparedStatement.setString(4, period);
|
429
|
preparedStatement.setString(5, type);
|
430
|
preparedStatement.setInt(6, Integer.parseInt(count));
|
431
|
preparedStatement.addBatch();
|
432
|
batch_size++;
|
433
|
if (batch_size == 10000) {
|
434
|
preparedStatement.executeBatch();
|
435
|
conn.commit();
|
436
|
batch_size = 0;
|
437
|
}
|
438
|
}
|
439
|
//break;
|
440
|
}
|
441
|
//break;
|
442
|
}
|
443
|
|
444
|
preparedStatement.executeBatch();
|
445
|
conn.commit();
|
446
|
conn.close();
|
447
|
}
|
448
|
|
449
|
private String getJson(String url, String username, String password) throws Exception {
|
450
|
//String cred=username+":"+password;
|
451
|
//String encoded = new sun.misc.BASE64Encoder().encode (cred.getBytes());
|
452
|
try {
|
453
|
URL website = new URL(url);
|
454
|
URLConnection connection = website.openConnection();
|
455
|
//connection.setRequestProperty ("Authorization", "Basic "+encoded);
|
456
|
StringBuilder response;
|
457
|
try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
|
458
|
response = new StringBuilder();
|
459
|
String inputLine;
|
460
|
while ((inputLine = in.readLine()) != null) {
|
461
|
response.append(inputLine);
|
462
|
response.append("\n");
|
463
|
}
|
464
|
}
|
465
|
return response.toString();
|
466
|
} catch (Exception e) {
|
467
|
log.error("Failed to get URL", e);
|
468
|
return null;
|
469
|
}
|
470
|
}
|
471
|
|
472
|
/*
|
473
|
private void flushString(String data, String destination) throws Exception {
|
474
|
FSDataOutputStream fin;
|
475
|
try {
|
476
|
FileSystem fs = FileSystem.get(new Configuration());
|
477
|
fin = fs.create(new Path(destination), true);
|
478
|
fin.write(data.getBytes());
|
479
|
fin.close();
|
480
|
} catch (Exception e) {
|
481
|
log.error("Failed to write exported data to a file : ", e);
|
482
|
throw new Exception("Failed to write exported data to a file : " + e.toString(), e);
|
483
|
}
|
484
|
}
|
485
|
*/
|
486
|
}
|