|
1
|
package eu.dnetlib.usagestats.export;
|
|
2
|
/**
|
|
3
|
*
|
|
4
|
* @author dpie
|
|
5
|
*/
|
|
6
|
import java.io.*;
|
|
7
|
//import java.io.BufferedReader;
|
|
8
|
//import java.io.InputStreamReader;
|
|
9
|
import java.net.URL;
|
|
10
|
import java.net.URLConnection;
|
|
11
|
import java.sql.ResultSet;
|
|
12
|
import java.text.SimpleDateFormat;
|
|
13
|
import java.util.Date;
|
|
14
|
import java.util.Calendar;
|
|
15
|
|
|
16
|
import java.sql.Connection;
|
|
17
|
import java.sql.PreparedStatement;
|
|
18
|
import java.sql.Statement;
|
|
19
|
|
|
20
|
import org.json.simple.JSONArray;
|
|
21
|
import org.json.simple.JSONObject;
|
|
22
|
import org.json.simple.parser.JSONParser;
|
|
23
|
|
|
24
|
import org.apache.log4j.Logger;
|
|
25
|
|
|
26
|
/**
|
|
27
|
* Created by dpie on 20/01/2020.
|
|
28
|
*/
|
|
29
|
public class IrusStats {
|
|
30
|
|
|
31
|
private String irusUKURL;
|
|
32
|
|
|
33
|
// private Connection conn = null;
|
|
34
|
// private Statement stmt = null;
|
|
35
|
|
|
36
|
private final Logger log = Logger.getLogger(this.getClass());
|
|
37
|
|
|
38
|
public IrusStats(String irusUKURL) throws Exception {
|
|
39
|
this.irusUKURL = irusUKURL;
|
|
40
|
createTables();
|
|
41
|
createTmpTables();
|
|
42
|
}
|
|
43
|
|
|
44
|
private void createTables() throws Exception {
|
|
45
|
try {
|
|
46
|
|
|
47
|
Statement stmt = ConnectDB.getConnection().createStatement();
|
|
48
|
String sqlCreateTableSushiLog = "CREATE TABLE IF NOT EXISTS sushilog(source TEXT, repository TEXT, rid TEXT, date TEXT, metric_type TEXT, count INT, PRIMARY KEY(source, repository, rid, date, metric_type));";
|
|
49
|
stmt.executeUpdate(sqlCreateTableSushiLog);
|
|
50
|
String sqlcreateRuleSushiLog = "CREATE OR REPLACE RULE ignore_duplicate_inserts AS "
|
|
51
|
+ " ON INSERT TO sushilog "
|
|
52
|
+ " WHERE (EXISTS ( SELECT sushilog.source, sushilog.repository,"
|
|
53
|
+ "sushilog.rid, sushilog.date "
|
|
54
|
+ "FROM sushilog "
|
|
55
|
+ "WHERE sushilog.source = new.source AND sushilog.repository = new.repository AND sushilog.rid = new.rid AND sushilog.date = new.date AND sushilog.metric_type = new.metric_type)) DO INSTEAD NOTHING;";
|
|
56
|
stmt.executeUpdate(sqlcreateRuleSushiLog);
|
|
57
|
String createSushiIndex = "create index if not exists sushilog_duplicates on sushilog(source, repository, rid, date, metric_type);";
|
|
58
|
stmt.executeUpdate(createSushiIndex);
|
|
59
|
|
|
60
|
stmt.close();
|
|
61
|
ConnectDB.getConnection().close();
|
|
62
|
log.info("Sushi Tables Created");
|
|
63
|
} catch (Exception e) {
|
|
64
|
log.error("Failed to create tables: " + e);
|
|
65
|
throw new Exception("Failed to create tables: " + e.toString(), e);
|
|
66
|
}
|
|
67
|
}
|
|
68
|
|
|
69
|
private void createTmpTables() throws Exception {
|
|
70
|
try {
|
|
71
|
|
|
72
|
Statement stmt = ConnectDB.getConnection().createStatement();
|
|
73
|
String sqlCreateTableSushiLog = "CREATE TABLE IF NOT EXISTS sushilogtmp(source TEXT, repository TEXT, rid TEXT, date TEXT, metric_type TEXT, count INT, PRIMARY KEY(source, repository, rid, date, metric_type));";
|
|
74
|
stmt.executeUpdate(sqlCreateTableSushiLog);
|
|
75
|
|
|
76
|
//stmt.executeUpdate("CREATE TABLE IF NOT EXISTS public.sushilog AS TABLE sushilog;");
|
|
77
|
//String sqlCopyPublicSushiLog = "INSERT INTO sushilog SELECT * FROM public.sushilog;";
|
|
78
|
//stmt.executeUpdate(sqlCopyPublicSushiLog);
|
|
79
|
String sqlcreateRuleSushiLog = "CREATE OR REPLACE RULE ignore_duplicate_inserts AS "
|
|
80
|
+ " ON INSERT TO sushilogtmp "
|
|
81
|
+ " WHERE (EXISTS ( SELECT sushilog.source, sushilog.repository,"
|
|
82
|
+ "sushilog.rid, sushilog.date "
|
|
83
|
+ "FROM sushilog "
|
|
84
|
+ "WHERE sushilog.source = new.source AND sushilog.repository = new.repository AND sushilog.rid = new.rid AND sushilog.date = new.date AND sushilog.metric_type = new.metric_type)) DO INSTEAD NOTHING;";
|
|
85
|
stmt.executeUpdate(sqlcreateRuleSushiLog);
|
|
86
|
|
|
87
|
stmt.close();
|
|
88
|
ConnectDB.getConnection().close();
|
|
89
|
log.info("Sushi Tmp Tables Created");
|
|
90
|
} catch (Exception e) {
|
|
91
|
log.error("Failed to create tables: " + e);
|
|
92
|
throw new Exception("Failed to create tables: " + e.toString(), e);
|
|
93
|
}
|
|
94
|
}
|
|
95
|
|
|
96
|
public void irusStats() throws Exception {
|
|
97
|
Statement stmt = ConnectDB.getConnection().createStatement();
|
|
98
|
ConnectDB.getConnection().setAutoCommit(false);
|
|
99
|
|
|
100
|
//String sql = "INSERT INTO sushi_result_downloads SELECT s.source, d.id AS repository, ro.id, s.date, s.count FROM sushilog s, datasource_oids d, result_oids ro WHERE s.repository=d.orid AND s.oai=ro.orid AND metric_type='ft_total'";
|
|
101
|
//String sql = "SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count INTO downloads_stats FROM sushilog s, datasource_oids d, result_oids ro WHERE s.repository=d.orid AND s.oai=ro.orid AND metric_type='ft_total'";
|
|
102
|
//String sql = "INSERT INTO downloads_stats SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count FROM sushilog s, datasource_oids d, result_oids ro WHERE s.repository=d.orid AND s.oai=ro.orid AND metric_type='ft_total';";
|
|
103
|
String sql = "INSERT INTO downloads_stats SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count, '0' FROM sushilogtmp s, public.datasource_oids d, public.result_oids ro WHERE s.repository=d.orid AND s.rid=ro.orid AND metric_type='ft_total' AND s.source='IRUS-UK';";
|
|
104
|
stmt.executeUpdate(sql);
|
|
105
|
|
|
106
|
sql = "Insert into sushilog select * from sushilogtmp;";
|
|
107
|
stmt.executeUpdate(sql);
|
|
108
|
|
|
109
|
sql = "drop table sushilogtmp;";
|
|
110
|
stmt.executeUpdate(sql);
|
|
111
|
|
|
112
|
ConnectDB.getConnection().commit();
|
|
113
|
ConnectDB.getConnection().close();
|
|
114
|
}
|
|
115
|
|
|
116
|
public void processIrusRRReport() throws Exception {
|
|
117
|
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM");
|
|
118
|
//String reportUrl = "https://irus.jisc.ac.uk" + "/api/sushilite/v1_7/GetReport/?Report=RR1&Release=4&RequestorID=OpenAIRE&BeginDate=2016-01&EndDate=" + simpleDateFormat.format(new Date()) + "&RepositoryIdentifier=&ItemDataType=&NewJiscBand=&Granularity=Monthly&Callback=";
|
|
119
|
String reportUrl = irusUKURL+"GetReport/?Report=RR1&Release=4&RequestorID=OpenAIRE&BeginDate=2016-01&EndDate=" + simpleDateFormat.format(new Date()) + "&RepositoryIdentifier=&ItemDataType=&NewJiscBand=&Granularity=Monthly&Callback=";
|
|
120
|
|
|
121
|
log.info("Getting Irus report: " + reportUrl);
|
|
122
|
|
|
123
|
String text = getJson(reportUrl, "", "");
|
|
124
|
|
|
125
|
log.info("Report: " + text);
|
|
126
|
|
|
127
|
JSONParser parser = new JSONParser();
|
|
128
|
JSONObject jsonObject = (JSONObject) parser.parse(text);
|
|
129
|
jsonObject = (JSONObject) jsonObject.get("ReportResponse");
|
|
130
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
|
131
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
|
132
|
jsonObject = (JSONObject) jsonObject.get("Customer");
|
|
133
|
JSONArray jsonArray = (JSONArray) jsonObject.get("ReportItems");
|
|
134
|
int i = 0;
|
|
135
|
for (Object aJsonArray : jsonArray) {
|
|
136
|
JSONObject jsonObjectRow = (JSONObject) aJsonArray;
|
|
137
|
JSONArray itemIdentifier = (JSONArray) jsonObjectRow.get("ItemIdentifier");
|
|
138
|
for (Object identifier : itemIdentifier) {
|
|
139
|
JSONObject opendoar = (JSONObject) identifier;
|
|
140
|
if (opendoar.get("Type").toString().equals("OpenDOAR")) {
|
|
141
|
//System.out.println(i + ": " + opendoar.get("Value").toString());
|
|
142
|
log.info(i + ": " + opendoar.get("Value").toString());
|
|
143
|
i++;
|
|
144
|
processIrusIRReport(opendoar.get("Value").toString());
|
|
145
|
break;
|
|
146
|
}
|
|
147
|
}
|
|
148
|
//break;
|
|
149
|
}
|
|
150
|
}
|
|
151
|
|
|
152
|
private void processIrusIRReport(String opendoar) throws Exception {
|
|
153
|
System.out.println(opendoar);
|
|
154
|
ConnectDB.getConnection().setAutoCommit(false);
|
|
155
|
|
|
156
|
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM");
|
|
157
|
|
|
158
|
Calendar start = Calendar.getInstance();
|
|
159
|
start.set(Calendar.YEAR, 2016);
|
|
160
|
start.set(Calendar.MONTH, Calendar.JANUARY);
|
|
161
|
//start.setTime(simpleDateFormat.parse("2016-01"));
|
|
162
|
|
|
163
|
Calendar end = Calendar.getInstance();
|
|
164
|
end.add(Calendar.DAY_OF_MONTH, -1);
|
|
165
|
|
|
166
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
|
|
167
|
PreparedStatement st = ConnectDB.getConnection().prepareStatement("SELECT max(date) FROM sushilog WHERE repository=?;");
|
|
168
|
st.setString(1, "opendoar____::" + opendoar);
|
|
169
|
ResultSet rs_date = st.executeQuery();
|
|
170
|
while (rs_date.next()) {
|
|
171
|
if (rs_date.getString(1) != null && !rs_date.getString(1).equals("null") && !rs_date.getString(1).equals("")) {
|
|
172
|
start.setTime(sdf.parse(rs_date.getString(1)));
|
|
173
|
}
|
|
174
|
}
|
|
175
|
rs_date.close();
|
|
176
|
PreparedStatement preparedStatement = ConnectDB.getConnection().prepareStatement("INSERT INTO sushilogtmp (source, repository, rid, date, metric_type, count) VALUES (?,?,?,?,?,?)");
|
|
177
|
int batch_size = 0;
|
|
178
|
|
|
179
|
while (start.before(end)) {
|
|
180
|
//log.info("date: " + simpleDateFormat.format(start.getTime()));
|
|
181
|
String reportUrl = this.irusUKURL+"GetReport/?Report=IR1&Release=4&RequestorID=OpenAIRE&BeginDate=" + simpleDateFormat.format(start.getTime()) + "&EndDate=" + simpleDateFormat.format(start.getTime()) + "&RepositoryIdentifier=opendoar%3A" + opendoar + "&ItemIdentifier=&ItemDataType=&hasDOI=&Granularity=Monthly&Callback=";
|
|
182
|
start.add(Calendar.MONTH, 1);
|
|
183
|
|
|
184
|
String text = getJson(reportUrl, "", "");
|
|
185
|
if (text == null) {
|
|
186
|
continue;
|
|
187
|
}
|
|
188
|
|
|
189
|
JSONParser parser = new JSONParser();
|
|
190
|
JSONObject jsonObject = (JSONObject) parser.parse(text);
|
|
191
|
jsonObject = (JSONObject) jsonObject.get("ReportResponse");
|
|
192
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
|
193
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
|
194
|
jsonObject = (JSONObject) jsonObject.get("Customer");
|
|
195
|
JSONArray jsonArray = (JSONArray) jsonObject.get("ReportItems");
|
|
196
|
if (jsonArray == null) {
|
|
197
|
continue;
|
|
198
|
}
|
|
199
|
String oai = "";
|
|
200
|
for (Object aJsonArray : jsonArray) {
|
|
201
|
JSONObject jsonObjectRow = (JSONObject) aJsonArray;
|
|
202
|
JSONArray itemIdentifier = (JSONArray) jsonObjectRow.get("ItemIdentifier");
|
|
203
|
for (Object identifier : itemIdentifier) {
|
|
204
|
JSONObject oaiPmh = (JSONObject) identifier;
|
|
205
|
if (oaiPmh.get("Type").toString().equals("OAI")) {
|
|
206
|
oai = oaiPmh.get("Value").toString();
|
|
207
|
//System.out.println("OAI: " + oai);
|
|
208
|
break;
|
|
209
|
}
|
|
210
|
}
|
|
211
|
|
|
212
|
JSONArray itemPerformance = (JSONArray) jsonObjectRow.get("ItemPerformance");
|
|
213
|
String period;
|
|
214
|
String type;
|
|
215
|
String count;
|
|
216
|
for (Object perf : itemPerformance) {
|
|
217
|
JSONObject performance = (JSONObject) perf;
|
|
218
|
JSONObject periodObj = (JSONObject) performance.get("Period");
|
|
219
|
period = periodObj.get("Begin").toString();
|
|
220
|
JSONObject instanceObj = (JSONObject) performance.get("Instance");
|
|
221
|
type = instanceObj.get("MetricType").toString();
|
|
222
|
count = instanceObj.get("Count").toString();
|
|
223
|
//System.out.println(oai + " : " + period + " : " + count);
|
|
224
|
|
|
225
|
preparedStatement.setString(1, "IRUS-UK");
|
|
226
|
preparedStatement.setString(2, "opendoar____::" + opendoar);
|
|
227
|
preparedStatement.setString(3, oai);
|
|
228
|
preparedStatement.setString(4, period);
|
|
229
|
preparedStatement.setString(5, type);
|
|
230
|
preparedStatement.setInt(6, Integer.parseInt(count));
|
|
231
|
preparedStatement.addBatch();
|
|
232
|
batch_size++;
|
|
233
|
if (batch_size == 10000) {
|
|
234
|
preparedStatement.executeBatch();
|
|
235
|
ConnectDB.getConnection().commit();
|
|
236
|
batch_size = 0;
|
|
237
|
}
|
|
238
|
}
|
|
239
|
//break;
|
|
240
|
}
|
|
241
|
//break;
|
|
242
|
}
|
|
243
|
|
|
244
|
preparedStatement.executeBatch();
|
|
245
|
ConnectDB.getConnection().commit();
|
|
246
|
ConnectDB.getConnection().close();
|
|
247
|
}
|
|
248
|
|
|
249
|
public void processIrusIRReport(String opendoar, String startDate) throws Exception {
|
|
250
|
ConnectDB.getConnection().setAutoCommit(false);
|
|
251
|
|
|
252
|
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM");
|
|
253
|
|
|
254
|
Calendar start = Calendar.getInstance();
|
|
255
|
start.set(Calendar.YEAR, 2016);
|
|
256
|
start.set(Calendar.MONTH, Calendar.JANUARY);
|
|
257
|
//start.setTime(simpleDateFormat.parse("2016-01"));
|
|
258
|
|
|
259
|
Calendar end = Calendar.getInstance();
|
|
260
|
end.add(Calendar.DAY_OF_MONTH, -1);
|
|
261
|
|
|
262
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
|
|
263
|
start.setTime(sdf.parse(startDate));
|
|
264
|
|
|
265
|
String createTablesQuery = "-- Table: shadow.sushilog" + opendoar + "\n"
|
|
266
|
+ "\n"
|
|
267
|
+ "-- DROP TABLE shadow.sushilog" + opendoar + ";\n"
|
|
268
|
+ "\n"
|
|
269
|
+ "CREATE TABLE shadow.sushilog" + opendoar + "\n"
|
|
270
|
+ "(\n"
|
|
271
|
+ " source text COLLATE pg_catalog.\"default\" NOT NULL,\n"
|
|
272
|
+ " repository text COLLATE pg_catalog.\"default\" NOT NULL,\n"
|
|
273
|
+ " rid text COLLATE pg_catalog.\"default\" NOT NULL,\n"
|
|
274
|
+ " date text COLLATE pg_catalog.\"default\" NOT NULL,\n"
|
|
275
|
+ " metric_type text COLLATE pg_catalog.\"default\" NOT NULL,\n"
|
|
276
|
+ " count integer,\n"
|
|
277
|
+ " CONSTRAINT sushilog" + opendoar + "_pkey PRIMARY KEY (source, repository, rid, date, metric_type)\n"
|
|
278
|
+ " USING INDEX TABLESPACE index_storage\n"
|
|
279
|
+ ")\n"
|
|
280
|
+ "\n"
|
|
281
|
+ "TABLESPACE pg_default;\n"
|
|
282
|
+ "\n"
|
|
283
|
+ "ALTER TABLE shadow.sushilog" + opendoar + "\n"
|
|
284
|
+ " OWNER to sqoop;\n"
|
|
285
|
+ "\n"
|
|
286
|
+ "-- Rule: ignore_duplicate_inserts ON shadow.sushilog" + opendoar + "\n"
|
|
287
|
+ "\n"
|
|
288
|
+ "-- DROP Rule ignore_duplicate_inserts ON shadow.sushilog" + opendoar + ";\n"
|
|
289
|
+ "\n"
|
|
290
|
+ "CREATE OR REPLACE RULE ignore_duplicate_inserts AS\n"
|
|
291
|
+ " ON INSERT TO shadow.sushilog" + opendoar + "\n"
|
|
292
|
+ " WHERE (EXISTS ( SELECT sushilog" + opendoar + ".source,\n"
|
|
293
|
+ " sushilog" + opendoar + ".repository,\n"
|
|
294
|
+ " sushilog" + opendoar + ".rid,\n"
|
|
295
|
+ " sushilog" + opendoar + ".date\n"
|
|
296
|
+ " FROM sushilog" + opendoar + "\n"
|
|
297
|
+ " WHERE sushilog" + opendoar + ".source = new.source AND sushilog" + opendoar + ".repository = new.repository AND sushilog" + opendoar + ".rid = new.rid AND sushilog" + opendoar + ".date = new.date AND sushilog" + opendoar + ".metric_type = new.metric_type))\n"
|
|
298
|
+ " DO INSTEAD\n"
|
|
299
|
+ "NOTHING;";
|
|
300
|
|
|
301
|
Statement stCreateTables = ConnectDB.getConnection().createStatement();
|
|
302
|
stCreateTables.execute(createTablesQuery);
|
|
303
|
ConnectDB.getConnection().commit();
|
|
304
|
|
|
305
|
PreparedStatement preparedStatement = ConnectDB.getConnection().prepareStatement("INSERT INTO sushilog" + opendoar + " (source, repository, rid, date, metric_type, count) VALUES (?,?,?,?,?,?)");
|
|
306
|
int batch_size = 0;
|
|
307
|
|
|
308
|
while (start.before(end)) {
|
|
309
|
//log.info("date: " + simpleDateFormat.format(start.getTime()));
|
|
310
|
String reportUrl = "https://irus.jisc.ac.uk/api/sushilite/v1_7/GetReport/?Report=IR1&Release=4&RequestorID=OpenAIRE&BeginDate=" + simpleDateFormat.format(start.getTime()) + "&EndDate=2019-10-31&RepositoryIdentifier=opendoar%3A" + opendoar + "&ItemIdentifier=&ItemDataType=&hasDOI=&Granularity=Monthly&Callback=";
|
|
311
|
start.add(Calendar.MONTH, 1);
|
|
312
|
|
|
313
|
String text = getJson(reportUrl, "", "");
|
|
314
|
if (text == null) {
|
|
315
|
continue;
|
|
316
|
}
|
|
317
|
|
|
318
|
JSONParser parser = new JSONParser();
|
|
319
|
JSONObject jsonObject = (JSONObject) parser.parse(text);
|
|
320
|
jsonObject = (JSONObject) jsonObject.get("ReportResponse");
|
|
321
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
|
322
|
jsonObject = (JSONObject) jsonObject.get("Report");
|
|
323
|
jsonObject = (JSONObject) jsonObject.get("Customer");
|
|
324
|
JSONArray jsonArray = (JSONArray) jsonObject.get("ReportItems");
|
|
325
|
if (jsonArray == null) {
|
|
326
|
continue;
|
|
327
|
}
|
|
328
|
String oai = "";
|
|
329
|
for (Object aJsonArray : jsonArray) {
|
|
330
|
JSONObject jsonObjectRow = (JSONObject) aJsonArray;
|
|
331
|
JSONArray itemIdentifier = (JSONArray) jsonObjectRow.get("ItemIdentifier");
|
|
332
|
for (Object identifier : itemIdentifier) {
|
|
333
|
JSONObject oaiPmh = (JSONObject) identifier;
|
|
334
|
if (oaiPmh.get("Type").toString().equals("OAI")) {
|
|
335
|
oai = oaiPmh.get("Value").toString();
|
|
336
|
//System.out.println("OAI: " + oai);
|
|
337
|
break;
|
|
338
|
}
|
|
339
|
}
|
|
340
|
|
|
341
|
JSONArray itemPerformance = (JSONArray) jsonObjectRow.get("ItemPerformance");
|
|
342
|
String period;
|
|
343
|
String type;
|
|
344
|
String count;
|
|
345
|
for (Object perf : itemPerformance) {
|
|
346
|
JSONObject performance = (JSONObject) perf;
|
|
347
|
JSONObject periodObj = (JSONObject) performance.get("Period");
|
|
348
|
period = periodObj.get("Begin").toString();
|
|
349
|
JSONObject instanceObj = (JSONObject) performance.get("Instance");
|
|
350
|
type = instanceObj.get("MetricType").toString();
|
|
351
|
count = instanceObj.get("Count").toString();
|
|
352
|
//System.out.println(oai + " : " + period + " : " + count);
|
|
353
|
|
|
354
|
preparedStatement.setString(1, "IRUS-UK");
|
|
355
|
preparedStatement.setString(2, "opendoar____::" + opendoar);
|
|
356
|
preparedStatement.setString(3, oai);
|
|
357
|
preparedStatement.setString(4, period);
|
|
358
|
preparedStatement.setString(5, type);
|
|
359
|
preparedStatement.setInt(6, Integer.parseInt(count));
|
|
360
|
preparedStatement.addBatch();
|
|
361
|
batch_size++;
|
|
362
|
if (batch_size == 10000) {
|
|
363
|
preparedStatement.executeBatch();
|
|
364
|
ConnectDB.getConnection().commit();
|
|
365
|
batch_size = 0;
|
|
366
|
}
|
|
367
|
}
|
|
368
|
//break;
|
|
369
|
}
|
|
370
|
//break;
|
|
371
|
}
|
|
372
|
|
|
373
|
preparedStatement.executeBatch();
|
|
374
|
ConnectDB.getConnection().commit();
|
|
375
|
ConnectDB.getConnection().close();
|
|
376
|
}
|
|
377
|
|
|
378
|
private String getJson(String url, String username, String password) throws Exception {
|
|
379
|
//String cred=username+":"+password;
|
|
380
|
//String encoded = new sun.misc.BASE64Encoder().encode (cred.getBytes());
|
|
381
|
try {
|
|
382
|
URL website = new URL(url);
|
|
383
|
URLConnection connection = website.openConnection();
|
|
384
|
//connection.setRequestProperty ("Authorization", "Basic "+encoded);
|
|
385
|
StringBuilder response;
|
|
386
|
try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
|
|
387
|
response = new StringBuilder();
|
|
388
|
String inputLine;
|
|
389
|
while ((inputLine = in.readLine()) != null) {
|
|
390
|
response.append(inputLine);
|
|
391
|
response.append("\n");
|
|
392
|
}
|
|
393
|
}
|
|
394
|
return response.toString();
|
|
395
|
} catch (Exception e) {
|
|
396
|
log.error("Failed to get URL", e);
|
|
397
|
return null;
|
|
398
|
}
|
|
399
|
}
|
|
400
|
}
|