D-Net

package eu.dnetlib.usagestats.export;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.fs.FileSystem;

import org.apache.log4j.Logger;

import java.io.*;

import java.net.URL;

import java.net.URLConnection;

import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.Statement;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.Calendar;

public class PiwikDownloadLogs {

    private final String piwikUsername;

    private final String piwikPassword;

    private final String httpProtocol;

    private final String piwikUrl;

    private final Date startDate;

    private final String tokenAuth;

    private final String logsPath;

    private final String dbUrl;

    private final String dbUserName;

    private final String dbPassword;

    /*

       The Piwik's API method 

    */

    private final String APImethod = "?module=API&method=Live.getLastVisitsDetails";

    private final String format = "&format=json";

    private final Logger log = Logger.getLogger(this.getClass());

    public PiwikDownloadLogs(String username, String password, String tokenAuth, String httpProtocol, String piwikURl, String sDate, String logsPath, String dbUrl, String dbUsername, String dbPassword) throws Exception{

        this.piwikUsername = username;

        this.piwikPassword = password;

        this.httpProtocol = httpProtocol;

        this.piwikUrl = piwikURl;

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");

        this.startDate = sdf.parse(sDate);

        this.tokenAuth = tokenAuth;

        this.logsPath = logsPath;

        this.dbUrl = dbUrl;

        this.dbUserName = dbUsername;

        this.dbPassword = dbPassword;

    }

    private String getPiwikLogUrl(){

        return httpProtocol + "://" + piwikUrl + "/";

    }

    private String getJson(String url,String username, String password) throws Exception {

        //String cred=username+":"+password;

        //String encoded = new sun.misc.BASE64Encoder().encode (cred.getBytes());

        try {

            URL website = new URL(url);

            URLConnection connection = website.openConnection();

            //connection.setRequestProperty ("Authorization", "Basic "+encoded);

            StringBuilder response;

            try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {

                response = new StringBuilder();

                String inputLine;

                while ((inputLine = in.readLine()) != null) {

                    response.append(inputLine);

                    response.append("\n");

                }

            }

            return response.toString();

        }catch (Exception e){

            log.error("Failed to get URL: " + e);

            throw new Exception("Failed to get URL: " + e.toString(), e);

        }

    }

    public void getPiwikLogs() throws Exception{

        GetPortalLogs();

        GetRepositoriesLogs();

    }

    private void GetPortalLogs() throws Exception{

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");

        Calendar start = Calendar.getInstance();

        start.setTime(startDate);

        Calendar end = Calendar.getInstance();

        end.add(Calendar.DAY_OF_MONTH, -1);

        //end.setTime(getFinalDate());

        try{

            log.info("downloading logs for site with piwik_id: 5");

            Class.forName("org.postgresql.Driver");

            Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword);

            PreparedStatement st = conn.prepareStatement("SELECT max(timestamp) FROM public.piwiklog WHERE source='5' HAVING max(timestamp) is not null;");

            ResultSet rs_date = st.executeQuery();

            while(rs_date.next()){

                start.setTime(sdf.parse(rs_date.getString(1)));

            }

            rs_date.close();

            conn.close();

            for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {

                String period="&period=day&date="+sdf.format(date);

                log.info("Downloading logs for " + sdf.format(date));

                FileSystem fs = FileSystem.get(new Configuration());

                FSDataOutputStream fin = fs.create(new Path(logsPath + "portallog/" + "5_Piwiklog"+sdf.format((date))+".json"), true);

                String baseApiUrl = getPiwikLogUrl() + APImethod + "&idSite=5" + period + format + "&expanded=5&filter_limit=1000&token_auth=" + tokenAuth;

                String content = "";

                int i=0;

                while(!content.equals("[]\n")) {

                    String apiUrl = baseApiUrl;

                    if (i > 0)

                        apiUrl += "&filter_offset=" + (i*1000);

                    content = getJson(apiUrl, piwikUsername, piwikPassword);

                    fin.write(content.getBytes());

                    i++;

                }

                fin.close();

//

//

//

//

//

//

//                String apiUrl=getPiwikLogUrl()+APImethod+"&idSite=5"+period+format+"&expanded=5&filter_limit=1000&token_auth="+tokenAuth;

//                String content = getJson(apiUrl,piwikUsername,piwikPassword);

//

//                //for (int i=1;i<10;i++){

//                int i = 1;

//                while(true) {

//                    String apiUrlnew=apiUrl+"&filter_offset="+i*1000;

//                    String contentNew = getJson(apiUrlnew,piwikUsername,piwikUsername);

//                    content += contentNew;

//                    i++;

//                    if(contentNew.equals("[]\n")){

//                        break;

//                    }

//                }

//                flushString(content, logsPath + "portallog/" + "5_Piwiklog"+sdf.format((date))+".json");

            }

        } catch (Exception e) {

            log.error("Failed to get portal logs", e);

            throw new Exception("Failed to get portal logs: " + e.toString(), e);

        }

    }

    private void GetRepositoriesLogs() throws Exception{

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");

        Calendar start = Calendar.getInstance();

        start.setTime(startDate);

        Calendar end = Calendar.getInstance();

        end.add(Calendar.DAY_OF_MONTH, -1);

        //end.setTime(getFinalDate());

        Class.forName("org.postgresql.Driver");

        Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword);

        Statement statement = conn.createStatement();

        ResultSet rs = statement.executeQuery("SELECT distinct piwik_id from shadow.datasource where piwik_id is not null and piwik_id!='5' order by piwik_id;");

        while(rs.next()){

            int siteId = rs.getInt(1);

            PreparedStatement st = conn.prepareStatement("SELECT max(timestamp) FROM public.piwiklog WHERE source=?;");

            start.setTime(startDate);

            log.info("downloading logs for site with piwik_id: " + siteId);

            st.setInt(1, siteId);

            ResultSet rs_date = st.executeQuery();

            while(rs_date.next()){

                //log.info("source: " + siteId + " - date: " + rs_date.getString(1));

                if(rs_date.getString(1) == null || rs_date.getString(1).equals("null") || rs_date.getString(1).equals("")) {

//                        start = Calendar.getInstance();

//                        start.add(Calendar.MONTH, -1);

                    // DO NOTHING USE this.startDate!!!

                }

                else {

                    start.setTime(sdf.parse(rs_date.getString(1)));

                }

            }

            rs_date.close();

            for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {

                log.info("Downloading logs for " + sdf.format(date));

                String period="&period=day&date="+sdf.format(date);

                FileSystem fs = FileSystem.get(new Configuration());

                String baseApiUrl = getPiwikLogUrl() + APImethod + "&idSite=" + siteId + period + format + "&expanded=5&filter_limit=1000&token_auth=" + tokenAuth;

                String content = "";

                int i=0;

                while(!content.equals("[]\n")) {

                    FSDataOutputStream fin = fs.create(new Path(logsPath + "repolog/" + siteId + "_Piwiklog"+sdf.format((date)) + "_" + i + ".json"), true);

                    String apiUrl = baseApiUrl;

                    if (i > 0)

                        apiUrl += "&filter_offset=" + (i*1000);

                    content = getJson(apiUrl, piwikUsername, piwikPassword);

                    fin.write(content.getBytes());

                    fin.close();

                    i++;

                }

            }

        }

        rs.close();

        conn.close();

    }

//    private void flushString(String data, String destination) throws Exception {

//        FSDataOutputStream fin;

//        try {

//            FileSystem fs = FileSystem.get(new Configuration());

//            fin = fs.create(new Path(destination), true);

//            fin.write(data.getBytes());

//            fin.close();

//        } catch (Exception e) {

//            log.error("Failed  to write exported data to a file : ", e);

//            throw new Exception("Failed  to write exported data to a file : " + e.toString(), e);

//        }

//    }

}