1
|
package eu.dnetlib.usagestats.export;
|
2
|
|
3
|
import java.io.*;
|
4
|
//import java.io.BufferedReader;
|
5
|
//import java.io.InputStreamReader;
|
6
|
import java.net.URL;
|
7
|
import java.net.URLConnection;
|
8
|
import java.sql.ResultSet;
|
9
|
import java.text.SimpleDateFormat;
|
10
|
import java.util.Date;
|
11
|
import java.util.Calendar;
|
12
|
|
13
|
import java.sql.Connection;
|
14
|
import java.sql.DriverManager;
|
15
|
import java.sql.PreparedStatement;
|
16
|
import java.sql.Statement;
|
17
|
|
18
|
import org.json.simple.JSONArray;
|
19
|
import org.json.simple.JSONObject;
|
20
|
import org.json.simple.parser.JSONParser;
|
21
|
|
22
|
import org.apache.log4j.Logger;
|
23
|
/**
|
24
|
* Created by tsampikos on 27/6/2017.
|
25
|
*/
|
26
|
|
27
|
public class SarcStats {
|
28
|
|
29
|
private final String dbUrl;
|
30
|
private final String dbSchema;
|
31
|
private final String dbUserName;
|
32
|
private final String dbPassword;
|
33
|
|
34
|
private Connection conn = null;
|
35
|
private Statement stmt = null;
|
36
|
|
37
|
private final Logger log = Logger.getLogger(this.getClass());
|
38
|
|
39
|
public SarcStats(String dbUrl, String dbUsername, String dbPassword) throws Exception {
|
40
|
this.dbUrl = dbUrl;
|
41
|
this.dbSchema = "shadow";
|
42
|
this.dbUserName = dbUsername;
|
43
|
this.dbPassword = dbPassword;
|
44
|
|
45
|
connectDB();
|
46
|
createTables();
|
47
|
}
|
48
|
|
49
|
private void connectDB() throws Exception {
|
50
|
try {
|
51
|
Class.forName("org.postgresql.Driver");
|
52
|
conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword);
|
53
|
|
54
|
stmt = conn.createStatement();
|
55
|
String sqlSetSearchPath = "SET search_path TO " + dbSchema + ";";
|
56
|
stmt.executeUpdate(sqlSetSearchPath);
|
57
|
|
58
|
//log.info("Opened database successfully")
|
59
|
|
60
|
} catch (Exception e) {
|
61
|
//log.error("Connect to db failed: " + e);
|
62
|
throw new Exception("Failed to connect to db: " + e.toString(), e);
|
63
|
}
|
64
|
}
|
65
|
|
66
|
private void createTables() throws Exception {
|
67
|
try {
|
68
|
|
69
|
stmt = conn.createStatement();
|
70
|
String sqlCreateTableSushiLog = "CREATE TABLE IF NOT EXISTS sushilog(source TEXT, repository TEXT, rid TEXT, date TEXT, metric_type TEXT, count INT, PRIMARY KEY(source, repository, rid, date, metric_type));";
|
71
|
stmt.executeUpdate(sqlCreateTableSushiLog);
|
72
|
|
73
|
//String sqlCopyPublicSushiLog="INSERT INTO sushilog SELECT * FROM public.sushilog;";
|
74
|
//stmt.executeUpdate(sqlCopyPublicSushiLog);
|
75
|
|
76
|
String sqlcreateRuleSushiLog = "CREATE OR REPLACE RULE ignore_duplicate_inserts AS " +
|
77
|
" ON INSERT TO sushilog " +
|
78
|
" WHERE (EXISTS ( SELECT sushilog.source, sushilog.repository," +
|
79
|
"sushilog.rid, sushilog.date " +
|
80
|
"FROM sushilog " +
|
81
|
"WHERE sushilog.source = new.source AND sushilog.repository = new.repository AND sushilog.rid = new.rid AND sushilog.date = new.date AND sushilog.metric_type = new.metric_type)) DO INSTEAD NOTHING;";
|
82
|
stmt.executeUpdate(sqlcreateRuleSushiLog);
|
83
|
|
84
|
stmt.close();
|
85
|
conn.close();
|
86
|
log.info("Sushi Tables Created");
|
87
|
} catch (Exception e) {
|
88
|
log.error("Failed to create tables: " + e);
|
89
|
throw new Exception("Failed to create tables: " + e.toString(), e);
|
90
|
}
|
91
|
}
|
92
|
|
93
|
|
94
|
public void processSarc() throws Exception {
|
95
|
processARReport("https://revistas.rcaap.pt/motricidade/sushiLite/v1_7/", "1646-107X");
|
96
|
processARReport("http://revistas.rcaap.pt/antropologicas/sushiLite/v1_7/", "0873-819X");
|
97
|
processARReport("https://revistas.rcaap.pt/interaccoes/sushiLite/v1_7/", "1646-2335");
|
98
|
processARReport("http://revistas.rcaap.pt/cct/sushiLite/v1_7/", "2182-3030");
|
99
|
processARReport("http://actapediatrica.spp.pt/sushiLite/v1_7/", "0873-9781");
|
100
|
//processARReport("http://revistas.rcaap.pt/boletimspm/sushiLite/v1_7/");
|
101
|
processARReport("https://revistas.rcaap.pt/sociologiapp/sushiLite/v1_7/", "0873-6529");
|
102
|
processARReport("http://revistas.rcaap.pt/finisterra/sushiLite/v1_7/", "0430-5027");
|
103
|
processARReport("http://revistas.rcaap.pt/sisyphus/sushiLite/v1_7/", "2182-8474");
|
104
|
processARReport("https://revistas.rcaap.pt/anestesiologia/sushiLite/v1_7/", "0871-6099");
|
105
|
processARReport("https://revistas.rcaap.pt/rpe/sushiLite/v1_7/", "0871-9187");
|
106
|
processARReport("https://revistas.rcaap.pt/psilogos/sushiLite/v1_7/", "1646-091X");
|
107
|
processARReport("https://revistas.rcaap.pt/juridica/sushiLite/v1_7/", "2183-5799");
|
108
|
processARReport("http://revistas.rcaap.pt/ecr/sushiLite/v1_7/", "1647-2098");
|
109
|
processARReport("https://revistas.rcaap.pt/nascercrescer/sushiLite/v1_7/", "0872-0754");
|
110
|
processARReport("http://revistas.rcaap.pt/cea/sushiLite/v1_7/", "1645-3794");
|
111
|
processARReport("http://revistas.rcaap.pt/proelium/sushiLite/v1_7/", "1645-8826");
|
112
|
processARReport("http://revistas.rcaap.pt/millenium/sushiLite/v1_7/", "0873-3015");
|
113
|
|
114
|
//processARReport("http://www.bad.pt/publicacoes/index.php/cadernos/sushiLite/v1_7/");
|
115
|
}
|
116
|
|
117
|
public void sarcStats() throws Exception {
|
118
|
if (conn.isClosed())
|
119
|
connectDB();
|
120
|
|
121
|
stmt = conn.createStatement();
|
122
|
conn.setAutoCommit(false);
|
123
|
|
124
|
//String sql = "SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count, '0' INTO downloads_stats FROM sushilog s, datasource_oids d, result_oids ro WHERE s.repository=d.orid AND s.rid=ro.orid AND metric_type='ft_total'";
|
125
|
String sql = "INSERT INTO downloads_stats SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count, '0' FROM sushilog s, datasource_oids d, datasource_results dr, result_pids ro WHERE d.orid LIKE '%' || s.repository || '%' AND dr.id=d.id AND dr.result=ro.id AND s.rid=ro.pid AND ro.type='doi' AND metric_type='ft_total' AND s.source='SARC-OJS';";
|
126
|
stmt.executeUpdate(sql);
|
127
|
|
128
|
|
129
|
stmt.close();
|
130
|
conn.commit();
|
131
|
conn.close();
|
132
|
}
|
133
|
|
134
|
public void processARReport(String url, String issn) throws Exception {
|
135
|
log.info("Processing SARC! issn: " + issn + " with url: " + url);
|
136
|
if (conn.isClosed())
|
137
|
connectDB();
|
138
|
|
139
|
conn.setAutoCommit(false);
|
140
|
|
141
|
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM");
|
142
|
|
143
|
Calendar start = Calendar.getInstance();
|
144
|
start.set(Calendar.YEAR, 2016);
|
145
|
start.set(Calendar.MONTH, Calendar.JANUARY);
|
146
|
//start.setTime(simpleDateFormat.parse("2016-01"));
|
147
|
|
148
|
Calendar end = Calendar.getInstance();
|
149
|
end.add(Calendar.DAY_OF_MONTH, -1);
|
150
|
|
151
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
|
152
|
PreparedStatement st = conn.prepareStatement("SELECT max(date) FROM public.sushilog WHERE repository=?;");
|
153
|
st.setString(1, issn);
|
154
|
ResultSet rs_date = st.executeQuery();
|
155
|
while(rs_date.next()){
|
156
|
if(rs_date.getString(1) != null && !rs_date.getString(1).equals("null") && !rs_date.getString(1).equals("")) {
|
157
|
start.setTime(sdf.parse(rs_date.getString(1)));
|
158
|
}
|
159
|
}
|
160
|
rs_date.close();
|
161
|
|
162
|
PreparedStatement preparedStatement = conn.prepareStatement("INSERT INTO sushilog (source, repository, rid, date, metric_type, count) VALUES (?,?,?,?,?,?)");
|
163
|
int batch_size = 0;
|
164
|
|
165
|
while(start.before(end)){
|
166
|
//String reportUrl = "http://irus.mimas.ac.uk/api/sushilite/v1_7/GetReport/?Report=IR1&Release=4&RequestorID=OpenAIRE&BeginDate=" + simpleDateFormat.format(start.getTime()) + "&EndDate=" + simpleDateFormat.format(start.getTime()) + "&RepositoryIdentifier=opendoar%3A" + opendoar + "&ItemIdentifier=&ItemDataType=&hasDOI=&Granularity=Monthly&Callback=";
|
167
|
String reportUrl = url + "GetReport/?Report=AR1&Format=json&BeginDate=" + simpleDateFormat.format(start.getTime()) + "&EndDate=" + simpleDateFormat.format(start.getTime());
|
168
|
//System.out.println(reportUrl);
|
169
|
start.add(Calendar.MONTH, 1);
|
170
|
|
171
|
String text = getJson(reportUrl, "", "");
|
172
|
if(text == null || text.trim().length()==0){
|
173
|
continue;
|
174
|
}
|
175
|
|
176
|
/*
|
177
|
PrintWriter wr = new PrintWriter(new FileWriter("logs/" + simpleDateFormat.format(start.getTime()) + ".json"));
|
178
|
wr.print(text);
|
179
|
wr.close();
|
180
|
*/
|
181
|
|
182
|
JSONParser parser = new JSONParser();
|
183
|
JSONObject jsonObject = (JSONObject) parser.parse(text);
|
184
|
jsonObject = (JSONObject) jsonObject.get("sc:ReportResponse");
|
185
|
jsonObject = (JSONObject) jsonObject.get("sc:Report");
|
186
|
if(jsonObject == null){
|
187
|
continue;
|
188
|
}
|
189
|
jsonObject = (JSONObject) jsonObject.get("c:Report");
|
190
|
jsonObject = (JSONObject) jsonObject.get("c:Customer");
|
191
|
Object obj = jsonObject.get("c:ReportItems");
|
192
|
JSONArray jsonArray = new JSONArray();
|
193
|
if(obj instanceof JSONObject){
|
194
|
jsonArray.add(obj);
|
195
|
}
|
196
|
else{
|
197
|
jsonArray = (JSONArray) obj;
|
198
|
//jsonArray = (JSONArray) jsonObject.get("c:ReportItems");
|
199
|
}
|
200
|
if(jsonArray == null){
|
201
|
continue;
|
202
|
}
|
203
|
|
204
|
String rid = "";
|
205
|
for (Object aJsonArray : jsonArray) {
|
206
|
JSONObject jsonObjectRow = (JSONObject) aJsonArray;
|
207
|
JSONArray itemIdentifier = new JSONArray();
|
208
|
obj = jsonObjectRow.get("c:ItemIdentifier");
|
209
|
if(obj instanceof JSONObject){
|
210
|
itemIdentifier.add(obj);
|
211
|
}
|
212
|
else{
|
213
|
//JSONArray itemIdentifier = (JSONArray) jsonObjectRow.get("c:ItemIdentifier");
|
214
|
itemIdentifier = (JSONArray) obj;
|
215
|
}
|
216
|
for (Object identifier : itemIdentifier) {
|
217
|
JSONObject doi = (JSONObject) identifier;
|
218
|
if(doi.get("c:Type").toString().equals("DOI")){
|
219
|
rid = doi.get("c:Value").toString();
|
220
|
//System.out.println("DOI: " + rid);
|
221
|
break;
|
222
|
}
|
223
|
}
|
224
|
if(rid.isEmpty()){
|
225
|
continue;
|
226
|
}
|
227
|
|
228
|
JSONObject itemPerformance = (JSONObject) jsonObjectRow.get("c:ItemPerformance");
|
229
|
//for (Object perf : itemPerformance) {
|
230
|
JSONObject performance = (JSONObject) itemPerformance;
|
231
|
JSONObject periodObj = (JSONObject) performance.get("c:Period");
|
232
|
String period = periodObj.get("c:Begin").toString();
|
233
|
JSONObject instanceObj = (JSONObject) performance.get("c:Instance");
|
234
|
String type = instanceObj.get("c:MetricType").toString();
|
235
|
String count = instanceObj.get("c:Count").toString();
|
236
|
//System.out.println(rid + " : " + period + " : " + count);
|
237
|
|
238
|
preparedStatement.setString(1, "SARC-OJS");
|
239
|
preparedStatement.setString(2, issn);
|
240
|
//preparedStatement.setString(2, url);
|
241
|
preparedStatement.setString(3, rid);
|
242
|
preparedStatement.setString(4, period);
|
243
|
preparedStatement.setString(5, type);
|
244
|
preparedStatement.setInt(6, Integer.parseInt(count));
|
245
|
preparedStatement.addBatch();
|
246
|
batch_size++;
|
247
|
if(batch_size == 10000){
|
248
|
preparedStatement.executeBatch();
|
249
|
conn.commit();
|
250
|
batch_size = 0;
|
251
|
}
|
252
|
//}
|
253
|
|
254
|
//break;
|
255
|
}
|
256
|
//break;
|
257
|
}
|
258
|
|
259
|
preparedStatement.executeBatch();
|
260
|
conn.commit();
|
261
|
conn.close();
|
262
|
}
|
263
|
|
264
|
private String getJson(String url,String username, String password) throws Exception {
|
265
|
//String cred=username+":"+password;
|
266
|
//String encoded = new sun.misc.BASE64Encoder().encode (cred.getBytes());
|
267
|
try {
|
268
|
URL website = new URL(url);
|
269
|
URLConnection connection = website.openConnection();
|
270
|
//connection.setRequestProperty ("Authorization", "Basic "+encoded);
|
271
|
StringBuilder response;
|
272
|
try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
|
273
|
response = new StringBuilder();
|
274
|
String inputLine;
|
275
|
while ((inputLine = in.readLine()) != null) {
|
276
|
response.append(inputLine);
|
277
|
response.append("\n");
|
278
|
}
|
279
|
}
|
280
|
return response.toString();
|
281
|
}catch (Exception e){
|
282
|
log.error("Failed to get URL: " + e);
|
283
|
//System.out.println("Failed to get URL: " + e);
|
284
|
return null;
|
285
|
//throw new Exception("Failed to get URL: " + e.toString(), e);
|
286
|
}
|
287
|
}
|
288
|
|
289
|
/*
|
290
|
private void flushString(String data, String destination) throws Exception {
|
291
|
FSDataOutputStream fin;
|
292
|
try {
|
293
|
FileSystem fs = FileSystem.get(new Configuration());
|
294
|
fin = fs.create(new Path(destination), true);
|
295
|
fin.write(data.getBytes());
|
296
|
fin.close();
|
297
|
} catch (Exception e) {
|
298
|
log.error("Failed to write exported data to a file : ", e);
|
299
|
throw new Exception("Failed to write exported data to a file : " + e.toString(), e);
|
300
|
}
|
301
|
}
|
302
|
*/
|
303
|
}
|