1
|
package eu.dnetlib.usagestats.export;
|
2
|
|
3
|
import java.io.*;
|
4
|
//import java.io.BufferedReader;
|
5
|
//import java.io.InputStreamReader;
|
6
|
import java.net.URL;
|
7
|
import java.net.URLConnection;
|
8
|
import java.sql.ResultSet;
|
9
|
import java.text.SimpleDateFormat;
|
10
|
import java.util.Calendar;
|
11
|
|
12
|
import java.sql.Connection;
|
13
|
import java.sql.DriverManager;
|
14
|
import java.sql.PreparedStatement;
|
15
|
import java.sql.Statement;
|
16
|
|
17
|
import org.json.simple.JSONArray;
|
18
|
import org.json.simple.JSONObject;
|
19
|
import org.json.simple.parser.JSONParser;
|
20
|
|
21
|
import org.apache.log4j.Logger;
|
22
|
|
23
|
/**
|
24
|
* Created by tsampikos on 27/6/2017.
|
25
|
*/
|
26
|
public class SarcStats {
|
27
|
|
28
|
private String dbUrl;
|
29
|
private String dbSchema;
|
30
|
private String dbUserName;
|
31
|
private String dbPassword;
|
32
|
|
33
|
private Connection conn = null;
|
34
|
private Statement stmt = null;
|
35
|
|
36
|
private final Logger log = Logger.getLogger(this.getClass());
|
37
|
|
38
|
public SarcStats() throws Exception {
|
39
|
if (ConnectDB.DB_CONNECTION.isClosed()) {
|
40
|
connectDB();
|
41
|
}
|
42
|
createTables();
|
43
|
}
|
44
|
|
45
|
private void connectDB() throws Exception {
|
46
|
try {
|
47
|
ConnectDB connectDB = new ConnectDB();
|
48
|
} catch (Exception e) {
|
49
|
log.error("Connect to db failed: " + e);
|
50
|
throw new Exception("Failed to connect to db: " + e.toString(), e);
|
51
|
}
|
52
|
}
|
53
|
|
54
|
|
55
|
private void createTables() throws Exception {
|
56
|
try {
|
57
|
|
58
|
stmt = ConnectDB.DB_CONNECTION.createStatement();
|
59
|
String sqlCreateTableSushiLog = "CREATE TABLE IF NOT EXISTS sushilog(source TEXT, repository TEXT, rid TEXT, date TEXT, metric_type TEXT, count INT, PRIMARY KEY(source, repository, rid, date, metric_type));";
|
60
|
stmt.executeUpdate(sqlCreateTableSushiLog);
|
61
|
|
62
|
//String sqlCopyPublicSushiLog="INSERT INTO sushilog SELECT * FROM public.sushilog;";
|
63
|
//stmt.executeUpdate(sqlCopyPublicSushiLog);
|
64
|
String sqlcreateRuleSushiLog = "CREATE OR REPLACE RULE ignore_duplicate_inserts AS "
|
65
|
+ " ON INSERT TO sushilog "
|
66
|
+ " WHERE (EXISTS ( SELECT sushilog.source, sushilog.repository,"
|
67
|
+ "sushilog.rid, sushilog.date "
|
68
|
+ "FROM sushilog "
|
69
|
+ "WHERE sushilog.source = new.source AND sushilog.repository = new.repository AND sushilog.rid = new.rid AND sushilog.date = new.date AND sushilog.metric_type = new.metric_type)) DO INSTEAD NOTHING;";
|
70
|
stmt.executeUpdate(sqlcreateRuleSushiLog);
|
71
|
|
72
|
stmt.close();
|
73
|
ConnectDB.DB_CONNECTION.close();
|
74
|
log.info("Sushi Tables Created");
|
75
|
} catch (Exception e) {
|
76
|
log.error("Failed to create tables: " + e);
|
77
|
throw new Exception("Failed to create tables: " + e.toString(), e);
|
78
|
}
|
79
|
}
|
80
|
|
81
|
public void processSarc() throws Exception {
|
82
|
processARReport("https://revistas.rcaap.pt/motricidade/sushiLite/v1_7/", "1646-107X");
|
83
|
processARReport("http://revistas.rcaap.pt/antropologicas/sushiLite/v1_7/", "0873-819X");
|
84
|
processARReport("https://revistas.rcaap.pt/interaccoes/sushiLite/v1_7/", "1646-2335");
|
85
|
processARReport("http://revistas.rcaap.pt/cct/sushiLite/v1_7/", "2182-3030");
|
86
|
processARReport("http://actapediatrica.spp.pt/sushiLite/v1_7/", "0873-9781");
|
87
|
//processARReport("http://revistas.rcaap.pt/boletimspm/sushiLite/v1_7/");
|
88
|
processARReport("https://revistas.rcaap.pt/sociologiapp/sushiLite/v1_7/", "0873-6529");
|
89
|
processARReport("http://revistas.rcaap.pt/finisterra/sushiLite/v1_7/", "0430-5027");
|
90
|
processARReport("http://revistas.rcaap.pt/sisyphus/sushiLite/v1_7/", "2182-8474");
|
91
|
processARReport("https://revistas.rcaap.pt/anestesiologia/sushiLite/v1_7/", "0871-6099");
|
92
|
processARReport("https://revistas.rcaap.pt/rpe/sushiLite/v1_7/", "0871-9187");
|
93
|
processARReport("https://revistas.rcaap.pt/psilogos/sushiLite/v1_7/", "1646-091X");
|
94
|
processARReport("https://revistas.rcaap.pt/juridica/sushiLite/v1_7/", "2183-5799");
|
95
|
processARReport("http://revistas.rcaap.pt/ecr/sushiLite/v1_7/", "1647-2098");
|
96
|
processARReport("https://revistas.rcaap.pt/nascercrescer/sushiLite/v1_7/", "0872-0754");
|
97
|
processARReport("http://revistas.rcaap.pt/cea/sushiLite/v1_7/", "1645-3794");
|
98
|
processARReport("http://revistas.rcaap.pt/proelium/sushiLite/v1_7/", "1645-8826");
|
99
|
processARReport("http://revistas.rcaap.pt/millenium/sushiLite/v1_7/", "0873-3015");
|
100
|
|
101
|
//processARReport("http://www.bad.pt/publicacoes/index.php/cadernos/sushiLite/v1_7/");
|
102
|
}
|
103
|
|
104
|
public void sarcStats() throws Exception {
|
105
|
if (ConnectDB.DB_CONNECTION.isClosed()) {
|
106
|
connectDB();
|
107
|
}
|
108
|
|
109
|
stmt = ConnectDB.DB_CONNECTION.createStatement();
|
110
|
ConnectDB.DB_CONNECTION.setAutoCommit(false);
|
111
|
|
112
|
//String sql = "SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count, '0' INTO downloads_stats FROM sushilog s, datasource_oids d, result_oids ro WHERE s.repository=d.orid AND s.rid=ro.orid AND metric_type='ft_total'";
|
113
|
String sql = "INSERT INTO downloads_stats SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count, '0' FROM sushilog s, datasource_oids d, datasource_results dr, result_pids ro WHERE d.orid LIKE '%' || s.repository || '%' AND dr.id=d.id AND dr.result=ro.id AND s.rid=ro.pid AND ro.type='doi' AND metric_type='ft_total' AND s.source='SARC-OJS';";
|
114
|
stmt.executeUpdate(sql);
|
115
|
|
116
|
stmt.close();
|
117
|
ConnectDB.DB_CONNECTION.commit();
|
118
|
ConnectDB.DB_CONNECTION.close();
|
119
|
}
|
120
|
|
121
|
public void processARReport(String url, String issn) throws Exception {
|
122
|
log.info("Processing SARC! issn: " + issn + " with url: " + url);
|
123
|
if (ConnectDB.DB_CONNECTION.isClosed()) {
|
124
|
connectDB();
|
125
|
}
|
126
|
|
127
|
ConnectDB.DB_CONNECTION.setAutoCommit(false);
|
128
|
|
129
|
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM");
|
130
|
|
131
|
Calendar start = Calendar.getInstance();
|
132
|
start.set(Calendar.YEAR, 2016);
|
133
|
start.set(Calendar.MONTH, Calendar.JANUARY);
|
134
|
//start.setTime(simpleDateFormat.parse("2016-01"));
|
135
|
|
136
|
Calendar end = Calendar.getInstance();
|
137
|
end.add(Calendar.DAY_OF_MONTH, -1);
|
138
|
|
139
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
|
140
|
PreparedStatement st = conn.prepareStatement("SELECT max(date) FROM public.sushilog WHERE repository=?;");
|
141
|
st.setString(1, issn);
|
142
|
ResultSet rs_date = st.executeQuery();
|
143
|
while (rs_date.next()) {
|
144
|
if (rs_date.getString(1) != null && !rs_date.getString(1).equals("null") && !rs_date.getString(1).equals("")) {
|
145
|
start.setTime(sdf.parse(rs_date.getString(1)));
|
146
|
}
|
147
|
}
|
148
|
rs_date.close();
|
149
|
|
150
|
PreparedStatement preparedStatement = conn.prepareStatement("INSERT INTO sushilog (source, repository, rid, date, metric_type, count) VALUES (?,?,?,?,?,?)");
|
151
|
int batch_size = 0;
|
152
|
|
153
|
while (start.before(end)) {
|
154
|
//String reportUrl = "http://irus.mimas.ac.uk/api/sushilite/v1_7/GetReport/?Report=IR1&Release=4&RequestorID=OpenAIRE&BeginDate=" + simpleDateFormat.format(start.getTime()) + "&EndDate=" + simpleDateFormat.format(start.getTime()) + "&RepositoryIdentifier=opendoar%3A" + opendoar + "&ItemIdentifier=&ItemDataType=&hasDOI=&Granularity=Monthly&Callback=";
|
155
|
String reportUrl = url + "GetReport/?Report=AR1&Format=json&BeginDate=" + simpleDateFormat.format(start.getTime()) + "&EndDate=" + simpleDateFormat.format(start.getTime());
|
156
|
//System.out.println(reportUrl);
|
157
|
start.add(Calendar.MONTH, 1);
|
158
|
|
159
|
String text = getJson(reportUrl, "", "");
|
160
|
if (text == null) {
|
161
|
continue;
|
162
|
}
|
163
|
|
164
|
/*
|
165
|
PrintWriter wr = new PrintWriter(new FileWriter("logs/" + simpleDateFormat.format(start.getTime()) + ".json"));
|
166
|
wr.print(text);
|
167
|
wr.close();
|
168
|
*/
|
169
|
JSONParser parser = new JSONParser();
|
170
|
JSONObject jsonObject = (JSONObject) parser.parse(text);
|
171
|
jsonObject = (JSONObject) jsonObject.get("sc:ReportResponse");
|
172
|
jsonObject = (JSONObject) jsonObject.get("sc:Report");
|
173
|
if (jsonObject == null) {
|
174
|
continue;
|
175
|
}
|
176
|
jsonObject = (JSONObject) jsonObject.get("c:Report");
|
177
|
jsonObject = (JSONObject) jsonObject.get("c:Customer");
|
178
|
Object obj = jsonObject.get("c:ReportItems");
|
179
|
JSONArray jsonArray = new JSONArray();
|
180
|
if (obj instanceof JSONObject) {
|
181
|
jsonArray.add(obj);
|
182
|
} else {
|
183
|
jsonArray = (JSONArray) obj;
|
184
|
//jsonArray = (JSONArray) jsonObject.get("c:ReportItems");
|
185
|
}
|
186
|
if (jsonArray == null) {
|
187
|
continue;
|
188
|
}
|
189
|
|
190
|
String rid = "";
|
191
|
for (Object aJsonArray : jsonArray) {
|
192
|
JSONObject jsonObjectRow = (JSONObject) aJsonArray;
|
193
|
JSONArray itemIdentifier = new JSONArray();
|
194
|
obj = jsonObjectRow.get("c:ItemIdentifier");
|
195
|
if (obj instanceof JSONObject) {
|
196
|
itemIdentifier.add(obj);
|
197
|
} else {
|
198
|
//JSONArray itemIdentifier = (JSONArray) jsonObjectRow.get("c:ItemIdentifier");
|
199
|
itemIdentifier = (JSONArray) obj;
|
200
|
}
|
201
|
for (Object identifier : itemIdentifier) {
|
202
|
JSONObject doi = (JSONObject) identifier;
|
203
|
if (doi.get("c:Type").toString().equals("DOI")) {
|
204
|
rid = doi.get("c:Value").toString();
|
205
|
//System.out.println("DOI: " + rid);
|
206
|
break;
|
207
|
}
|
208
|
}
|
209
|
if (rid.isEmpty()) {
|
210
|
continue;
|
211
|
}
|
212
|
|
213
|
JSONObject itemPerformance = (JSONObject) jsonObjectRow.get("c:ItemPerformance");
|
214
|
//for (Object perf : itemPerformance) {
|
215
|
JSONObject performance = (JSONObject) itemPerformance;
|
216
|
JSONObject periodObj = (JSONObject) performance.get("c:Period");
|
217
|
String period = periodObj.get("c:Begin").toString();
|
218
|
JSONObject instanceObj = (JSONObject) performance.get("c:Instance");
|
219
|
String type = instanceObj.get("c:MetricType").toString();
|
220
|
String count = instanceObj.get("c:Count").toString();
|
221
|
//System.out.println(rid + " : " + period + " : " + count);
|
222
|
|
223
|
preparedStatement.setString(1, "SARC-OJS");
|
224
|
preparedStatement.setString(2, issn);
|
225
|
//preparedStatement.setString(2, url);
|
226
|
preparedStatement.setString(3, rid);
|
227
|
preparedStatement.setString(4, period);
|
228
|
preparedStatement.setString(5, type);
|
229
|
preparedStatement.setInt(6, Integer.parseInt(count));
|
230
|
preparedStatement.addBatch();
|
231
|
batch_size++;
|
232
|
if (batch_size == 10000) {
|
233
|
preparedStatement.executeBatch();
|
234
|
conn.commit();
|
235
|
batch_size = 0;
|
236
|
}
|
237
|
//}
|
238
|
|
239
|
//break;
|
240
|
}
|
241
|
//break;
|
242
|
}
|
243
|
|
244
|
preparedStatement.executeBatch();
|
245
|
ConnectDB.DB_CONNECTION.commit();
|
246
|
ConnectDB.DB_CONNECTION.close();
|
247
|
}
|
248
|
|
249
|
private String getJson(String url, String username, String password) throws Exception {
|
250
|
//String cred=username+":"+password;
|
251
|
//String encoded = new sun.misc.BASE64Encoder().encode (cred.getBytes());
|
252
|
try {
|
253
|
URL website = new URL(url);
|
254
|
URLConnection connection = website.openConnection();
|
255
|
//connection.setRequestProperty ("Authorization", "Basic "+encoded);
|
256
|
StringBuilder response;
|
257
|
try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
|
258
|
response = new StringBuilder();
|
259
|
String inputLine;
|
260
|
while ((inputLine = in.readLine()) != null) {
|
261
|
response.append(inputLine);
|
262
|
response.append("\n");
|
263
|
}
|
264
|
}
|
265
|
return response.toString();
|
266
|
} catch (Exception e) {
|
267
|
log.error("Failed to get URL: " + e);
|
268
|
//System.out.println("Failed to get URL: " + e);
|
269
|
return null;
|
270
|
//throw new Exception("Failed to get URL: " + e.toString(), e);
|
271
|
}
|
272
|
}
|
273
|
|
274
|
/*
|
275
|
private void flushString(String data, String destination) throws Exception {
|
276
|
FSDataOutputStream fin;
|
277
|
try {
|
278
|
FileSystem fs = FileSystem.get(new Configuration());
|
279
|
fin = fs.create(new Path(destination), true);
|
280
|
fin.write(data.getBytes());
|
281
|
fin.close();
|
282
|
} catch (Exception e) {
|
283
|
log.error("Failed to write exported data to a file : ", e);
|
284
|
throw new Exception("Failed to write exported data to a file : " + e.toString(), e);
|
285
|
}
|
286
|
}
|
287
|
*/
|
288
|
}
|