1
|
package eu.dnetlib.usagestats.export;
|
2
|
|
3
|
import java.io.*;
|
4
|
//import java.io.BufferedReader;
|
5
|
//import java.io.InputStreamReader;
|
6
|
import java.net.URL;
|
7
|
import java.net.URLConnection;
|
8
|
import java.sql.ResultSet;
|
9
|
import java.text.SimpleDateFormat;
|
10
|
import java.util.Calendar;
|
11
|
import java.sql.PreparedStatement;
|
12
|
import java.sql.Statement;
|
13
|
import org.json.simple.JSONArray;
|
14
|
import org.json.simple.JSONObject;
|
15
|
import org.json.simple.parser.JSONParser;
|
16
|
|
17
|
import org.apache.log4j.Logger;
|
18
|
|
19
|
/**
|
20
|
* Created by dpie
|
21
|
*/
|
22
|
public class SarcStats {
|
23
|
|
24
|
private Statement stmt = null;
|
25
|
|
26
|
private final Logger log = Logger.getLogger(this.getClass());
|
27
|
|
28
|
public SarcStats() throws Exception {
|
29
|
createTables();
|
30
|
}
|
31
|
|
32
|
private void createTables() throws Exception {
|
33
|
try {
|
34
|
|
35
|
stmt = ConnectDB.getConnection().createStatement();
|
36
|
String sqlCreateTableSushiLog = "CREATE TABLE IF NOT EXISTS sushilog(source TEXT, repository TEXT, rid TEXT, date TEXT, metric_type TEXT, count INT, PRIMARY KEY(source, repository, rid, date, metric_type));";
|
37
|
stmt.executeUpdate(sqlCreateTableSushiLog);
|
38
|
|
39
|
//String sqlCopyPublicSushiLog="INSERT INTO sushilog SELECT * FROM public.sushilog;";
|
40
|
//stmt.executeUpdate(sqlCopyPublicSushiLog);
|
41
|
String sqlcreateRuleSushiLog = "CREATE OR REPLACE RULE ignore_duplicate_inserts AS "
|
42
|
+ " ON INSERT TO sushilog "
|
43
|
+ " WHERE (EXISTS ( SELECT sushilog.source, sushilog.repository,"
|
44
|
+ "sushilog.rid, sushilog.date "
|
45
|
+ "FROM sushilog "
|
46
|
+ "WHERE sushilog.source = new.source AND sushilog.repository = new.repository AND sushilog.rid = new.rid AND sushilog.date = new.date AND sushilog.metric_type = new.metric_type)) DO INSTEAD NOTHING;";
|
47
|
stmt.executeUpdate(sqlcreateRuleSushiLog);
|
48
|
String createSushiIndex = "create index if not exists sushilog_duplicates on sushilog(source, repository, rid, date, metric_type);";
|
49
|
stmt.executeUpdate(createSushiIndex);
|
50
|
|
51
|
stmt.close();
|
52
|
ConnectDB.getConnection().close();
|
53
|
log.info("Sushi Tables Created");
|
54
|
} catch (Exception e) {
|
55
|
log.error("Failed to create tables: " + e);
|
56
|
throw new Exception("Failed to create tables: " + e.toString(), e);
|
57
|
}
|
58
|
}
|
59
|
|
60
|
public void processSarc() throws Exception {
|
61
|
processARReport("https://revistas.rcaap.pt/motricidade/sushiLite/v1_7/", "1646-107X");
|
62
|
processARReport("https://revistas.rcaap.pt/antropologicas/sushiLite/v1_7/", "0873-819X");
|
63
|
processARReport("https://revistas.rcaap.pt/interaccoes/sushiLite/v1_7/", "1646-2335");
|
64
|
processARReport("https://revistas.rcaap.pt/cct/sushiLite/v1_7/", "2182-3030");
|
65
|
processARReport("https://actapediatrica.spp.pt/sushiLite/v1_7/", "0873-9781");
|
66
|
processARReport("https://revistas.rcaap.pt/sociologiapp/sushiLite/v1_7/", "0873-6529");
|
67
|
processARReport("https://revistas.rcaap.pt/finisterra/sushiLite/v1_7/", "0430-5027");
|
68
|
processARReport("https://revistas.rcaap.pt/sisyphus/sushiLite/v1_7/", "2182-8474");
|
69
|
processARReport("https://revistas.rcaap.pt/anestesiologia/sushiLite/v1_7/", "0871-6099");
|
70
|
processARReport("https://revistas.rcaap.pt/rpe/sushiLite/v1_7/", "0871-9187");
|
71
|
processARReport("https://revistas.rcaap.pt/psilogos/sushiLite/v1_7/", "1646-091X");
|
72
|
processARReport("https://revistas.rcaap.pt/juridica/sushiLite/v1_7/", "2183-5799");
|
73
|
processARReport("https://revistas.rcaap.pt/ecr/sushiLite/v1_7/", "1647-2098");
|
74
|
processARReport("https://revistas.rcaap.pt/nascercrescer/sushiLite/v1_7/", "0872-0754");
|
75
|
processARReport("https://revistas.rcaap.pt/cea/sushiLite/v1_7/", "1645-3794");
|
76
|
processARReport("https://revistas.rcaap.pt/proelium/sushiLite/v1_7/", "1645-8826");
|
77
|
processARReport("https://revistas.rcaap.pt/millenium/sushiLite/v1_7/", "0873-3015");
|
78
|
}
|
79
|
|
80
|
public void sarcStats() throws Exception {
|
81
|
stmt = ConnectDB.getConnection().createStatement();
|
82
|
ConnectDB.getConnection().setAutoCommit(false);
|
83
|
|
84
|
//String sql = "SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count, '0' INTO downloads_stats FROM sushilog s, datasource_oids d, result_oids ro WHERE s.repository=d.orid AND s.rid=ro.orid AND metric_type='ft_total'";
|
85
|
String sql = "INSERT INTO downloads_stats SELECT s.source, d.id AS repository_id, ro.id as result_id, extract('year' from s.date::date) ||'/'|| LPAD(CAST(extract('month' from s.date::date) AS VARCHAR), 2, '0') as date, s.count, '0' FROM sushilog s, public.datasource_oids d, public.datasource_results dr, public.result_pids ro WHERE d.orid LIKE '%' || s.repository || '%' AND dr.id=d.id AND dr.result=ro.id AND s.rid=ro.pid AND ro.type='doi' AND metric_type='ft_total' AND s.source='SARC-OJS';";
|
86
|
stmt.executeUpdate(sql);
|
87
|
|
88
|
stmt.close();
|
89
|
ConnectDB.getConnection().commit();
|
90
|
ConnectDB.getConnection().close();
|
91
|
}
|
92
|
|
93
|
public void processARReport(String url, String issn) throws Exception {
|
94
|
log.info("Processing SARC! issn: " + issn + " with url: " + url);
|
95
|
ConnectDB.getConnection().setAutoCommit(false);
|
96
|
|
97
|
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM");
|
98
|
|
99
|
Calendar start = Calendar.getInstance();
|
100
|
start.set(Calendar.YEAR, 2016);
|
101
|
start.set(Calendar.MONTH, Calendar.JANUARY);
|
102
|
//start.setTime(simpleDateFormat.parse("2016-01"));
|
103
|
|
104
|
Calendar end = Calendar.getInstance();
|
105
|
end.add(Calendar.DAY_OF_MONTH, -1);
|
106
|
|
107
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
|
108
|
PreparedStatement st = ConnectDB.getConnection().prepareStatement("SELECT max(date) FROM sushilog WHERE repository=?;");
|
109
|
st.setString(1, issn);
|
110
|
ResultSet rs_date = st.executeQuery();
|
111
|
while (rs_date.next()) {
|
112
|
if (rs_date.getString(1) != null && !rs_date.getString(1).equals("null") && !rs_date.getString(1).equals("")) {
|
113
|
start.setTime(sdf.parse(rs_date.getString(1)));
|
114
|
}
|
115
|
}
|
116
|
rs_date.close();
|
117
|
|
118
|
PreparedStatement preparedStatement = ConnectDB.getConnection().prepareStatement("INSERT INTO sushilog (source, repository, rid, date, metric_type, count) VALUES (?,?,?,?,?,?)");
|
119
|
int batch_size = 0;
|
120
|
|
121
|
while (start.before(end)) {
|
122
|
//String reportUrl = "http://irus.mimas.ac.uk/api/sushilite/v1_7/GetReport/?Report=IR1&Release=4&RequestorID=OpenAIRE&BeginDate=" + simpleDateFormat.format(start.getTime()) + "&EndDate=" + simpleDateFormat.format(start.getTime()) + "&RepositoryIdentifier=opendoar%3A" + opendoar + "&ItemIdentifier=&ItemDataType=&hasDOI=&Granularity=Monthly&Callback=";
|
123
|
String reportUrl = url + "GetReport/?Report=AR1&Format=json&BeginDate=" + simpleDateFormat.format(start.getTime()) + "&EndDate=" + simpleDateFormat.format(start.getTime());
|
124
|
//System.out.println(reportUrl);
|
125
|
start.add(Calendar.MONTH, 1);
|
126
|
|
127
|
String text = getJson(reportUrl);
|
128
|
if (text == null) {
|
129
|
continue;
|
130
|
}
|
131
|
|
132
|
/*
|
133
|
PrintWriter wr = new PrintWriter(new FileWriter("logs/" + simpleDateFormat.format(start.getTime()) + ".json"));
|
134
|
wr.print(text);
|
135
|
wr.close();
|
136
|
*/
|
137
|
JSONParser parser = new JSONParser();
|
138
|
JSONObject jsonObject = (JSONObject) parser.parse(text);
|
139
|
jsonObject = (JSONObject) jsonObject.get("sc:ReportResponse");
|
140
|
jsonObject = (JSONObject) jsonObject.get("sc:Report");
|
141
|
if (jsonObject == null) {
|
142
|
continue;
|
143
|
}
|
144
|
jsonObject = (JSONObject) jsonObject.get("c:Report");
|
145
|
jsonObject = (JSONObject) jsonObject.get("c:Customer");
|
146
|
Object obj = jsonObject.get("c:ReportItems");
|
147
|
JSONArray jsonArray = new JSONArray();
|
148
|
if (obj instanceof JSONObject) {
|
149
|
jsonArray.add(obj);
|
150
|
} else {
|
151
|
jsonArray = (JSONArray) obj;
|
152
|
//jsonArray = (JSONArray) jsonObject.get("c:ReportItems");
|
153
|
}
|
154
|
if (jsonArray == null) {
|
155
|
continue;
|
156
|
}
|
157
|
|
158
|
String rid = "";
|
159
|
for (Object aJsonArray : jsonArray) {
|
160
|
JSONObject jsonObjectRow = (JSONObject) aJsonArray;
|
161
|
JSONArray itemIdentifier = new JSONArray();
|
162
|
obj = jsonObjectRow.get("c:ItemIdentifier");
|
163
|
if (obj instanceof JSONObject) {
|
164
|
itemIdentifier.add(obj);
|
165
|
} else {
|
166
|
//JSONArray itemIdentifier = (JSONArray) jsonObjectRow.get("c:ItemIdentifier");
|
167
|
itemIdentifier = (JSONArray) obj;
|
168
|
}
|
169
|
for (Object identifier : itemIdentifier) {
|
170
|
JSONObject doi = (JSONObject) identifier;
|
171
|
if (doi.get("c:Type").toString().equals("DOI")) {
|
172
|
rid = doi.get("c:Value").toString();
|
173
|
//System.out.println("DOI: " + rid);
|
174
|
break;
|
175
|
}
|
176
|
}
|
177
|
if (rid.isEmpty()) {
|
178
|
continue;
|
179
|
}
|
180
|
|
181
|
JSONObject itemPerformance = (JSONObject) jsonObjectRow.get("c:ItemPerformance");
|
182
|
//for (Object perf : itemPerformance) {
|
183
|
JSONObject performance = (JSONObject) itemPerformance;
|
184
|
JSONObject periodObj = (JSONObject) performance.get("c:Period");
|
185
|
String period = periodObj.get("c:Begin").toString();
|
186
|
JSONObject instanceObj = (JSONObject) performance.get("c:Instance");
|
187
|
String type = instanceObj.get("c:MetricType").toString();
|
188
|
String count = instanceObj.get("c:Count").toString();
|
189
|
//System.out.println(rid + " : " + period + " : " + count);
|
190
|
|
191
|
preparedStatement.setString(1, "SARC-OJS");
|
192
|
preparedStatement.setString(2, issn);
|
193
|
//preparedStatement.setString(2, url);
|
194
|
preparedStatement.setString(3, rid);
|
195
|
preparedStatement.setString(4, period);
|
196
|
preparedStatement.setString(5, type);
|
197
|
preparedStatement.setInt(6, Integer.parseInt(count));
|
198
|
preparedStatement.addBatch();
|
199
|
batch_size++;
|
200
|
if (batch_size == 10000) {
|
201
|
preparedStatement.executeBatch();
|
202
|
ConnectDB.getConnection().commit();
|
203
|
batch_size = 0;
|
204
|
}
|
205
|
//}
|
206
|
|
207
|
//break;
|
208
|
}
|
209
|
//break;
|
210
|
}
|
211
|
|
212
|
preparedStatement.executeBatch();
|
213
|
ConnectDB.getConnection().commit();
|
214
|
ConnectDB.getConnection().close();
|
215
|
}
|
216
|
|
217
|
private String getJson(String url) {
|
218
|
//String cred=username+":"+password;
|
219
|
//String encoded = new sun.misc.BASE64Encoder().encode (cred.getBytes());
|
220
|
try {
|
221
|
URL website = new URL(url);
|
222
|
URLConnection connection = website.openConnection();
|
223
|
//connection.setRequestProperty ("Authorization", "Basic "+encoded);
|
224
|
StringBuilder response;
|
225
|
try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
|
226
|
response = new StringBuilder();
|
227
|
String inputLine;
|
228
|
while ((inputLine = in.readLine()) != null) {
|
229
|
response.append(inputLine);
|
230
|
response.append("\n");
|
231
|
}
|
232
|
}
|
233
|
return response.toString();
|
234
|
} catch (Exception e) {
|
235
|
log.error("Failed to get URL: " + e);
|
236
|
//System.out.println("Failed to get URL: " + e);
|
237
|
return null;
|
238
|
//throw new Exception("Failed to get URL: " + e.toString(), e);
|
239
|
}
|
240
|
}
|
241
|
}
|