D-Net

package eu.dnetlib.usagestats.export;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.fs.FileSystem;

import org.apache.log4j.Logger;

import java.io.*;

import java.net.URL;

import java.net.URLConnection;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.Statement;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.Calendar;

public class PiwikDownloadLogs {

    private final String piwikUrl;

    private Date startDate;

    private final String tokenAuth;

    /*

       The Piwik's API method 

     */

    private final String APImethod = "?module=API&method=Live.getLastVisitsDetails";

    private final String format = "&format=json";

    private final Logger log = Logger.getLogger(this.getClass());

    public PiwikDownloadLogs(String piwikUrl, String tokenAuth) {

        this.piwikUrl = piwikUrl;

        this.tokenAuth = tokenAuth;

    }

    private String getPiwikLogUrl() {

        return "https://" + piwikUrl + "/";

    }

    private String getJson(String url) throws Exception {

        try {

            URL website = new URL(url);

            URLConnection connection = website.openConnection();

            //connection.setRequestProperty ("Authorization", "Basic "+encoded);

            StringBuilder response;

            try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {

                response = new StringBuilder();

                String inputLine;

                while ((inputLine = in.readLine()) != null) {

                    response.append(inputLine);

                    response.append("\n");

                }

            }

            return response.toString();

        } catch (Exception e) {

            log.error("Failed to get URL: " + e);

            throw new Exception("Failed to get URL: " + e.toString(), e);

        }

    }

    public void GetOpenAIRELogs(String repoLogsPath, String portalLogPath, String portalMatomoID) throws Exception {

        Statement statement = ConnectDB.getConnection().createStatement();

        ResultSet rs = statement.executeQuery("SELECT distinct piwik_id from public.datasource order by piwik_id;");

        PreparedStatement st = ConnectDB.getConnection().prepareStatement("SELECT max(timestamp) FROM piwiklog WHERE source=? HAVING max(timestamp) is not null;");

        while (rs.next()) {

            int siteId = rs.getInt(1);

            st.setInt(1, siteId);

            ResultSet rs_date = st.executeQuery();

            while (rs_date.next()) {

                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");

                Date dateMax = sdf.parse(rs_date.getString(1));

                //Date dateMax = sdf.parse("2020-02-01");

                Calendar start = Calendar.getInstance();

                start.setTime(dateMax);

                Calendar end = Calendar.getInstance();

                end.add(Calendar.DAY_OF_MONTH, -1);

                for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {

                    log.info("Downloading logs for repoid " + siteId + " and for " + sdf.format(date));

                    String period = "&period=day&date=" + sdf.format(date);

                    String outFolder = "";

                    //portal siteId = 109;

                    if (siteId == Integer.parseInt(portalMatomoID)) {

                        outFolder = portalLogPath;

                    } else {

                        outFolder = repoLogsPath;

                    }

                    FileSystem fs = FileSystem.get(new Configuration());

                    FSDataOutputStream fin = fs.create(new Path(outFolder + "/" + siteId + "_Piwiklog" + sdf.format((date)) + ".json"), true);

                    String baseApiUrl = getPiwikLogUrl() + APImethod + "&idSite=" + siteId + period + format + "&expanded=5&filter_limit=1000&token_auth=" + tokenAuth;

                    String content = "";

                    int i = 0;

                    while (!content.equals("[]\n")) {

                        String apiUrl = baseApiUrl;

                        if (i > 0) {

                            apiUrl += "&filter_offset=" + (i * 1000);

                        }

                        content = getJson(apiUrl);

                        fin.write(content.getBytes());

                        i++;

                    }

                    fin.close();

                }

            }

        }

    }

}