D-Net

package eu.dnetlib.usagestats.export;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.fs.FileSystem;

import org.apache.log4j.Logger;

import java.io.*;

import java.net.URL;

import java.net.URLConnection;

import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.Statement;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.Calendar;

public class PiwikDownloadLogs {

    private final String piwikUsername;

    private final String piwikPassword;

    private final String httpProtocol;

    private final String piwikUrl;

    private final Date startDate;

    private final String tokenAuth;

    private final String logsPath;

    private final String dbUrl;

    private final String dbUserName;

    private final String dbPassword;

    /*

       The Piwik's API method 

    */

    private final String APImethod = "?module=API&method=Live.getLastVisitsDetails";

    private final String format = "&format=json";

    private final Logger log = Logger.getLogger(this.getClass());

    public PiwikDownloadLogs(String username, String password, String tokenAuth, String httpProtocol, String piwikURl, String sDate, String logsPath, String dbUrl, String dbUsername, String dbPassword) throws Exception{

        this.piwikUsername = username;

        this.piwikPassword = password;

        this.httpProtocol = httpProtocol;

        this.piwikUrl = piwikURl;

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");

        this.startDate = sdf.parse(sDate);

        this.tokenAuth = tokenAuth;

        this.logsPath = logsPath;

        this.dbUrl = dbUrl;

        this.dbUserName = dbUsername;

        this.dbPassword = dbPassword;

    }

    private String getPiwikLogUrl(){

        return httpProtocol + "://" + piwikUrl + "/";

    }

    private String getJson(String url,String username, String password) throws Exception {

        //String cred=username+":"+password;

        //String encoded = new sun.misc.BASE64Encoder().encode (cred.getBytes());

        try {

            URL website = new URL(url);

            URLConnection connection = website.openConnection();

            //connection.setRequestProperty ("Authorization", "Basic "+encoded);

            StringBuilder response;

            try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {

                response = new StringBuilder();

                String inputLine;

                while ((inputLine = in.readLine()) != null) {

                    response.append(inputLine);

                    response.append("\n");

                }

            }

            return response.toString();

        }catch (Exception e){

            log.error("Failed to get URL: " + e);

            throw new Exception("Failed to get URL: " + e.toString(), e);

        }

    }

    public void getPiwikLogs() throws Exception{

        GetPortalLogs();

        GetRepositoriesLogs();

    }

    private void GetPortalLogs() throws Exception{

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");

        Calendar start = Calendar.getInstance();

        start.setTime(startDate);

        Calendar end = Calendar.getInstance();

        end.add(Calendar.DAY_OF_MONTH, -1);

        //end.setTime(getFinalDate());

        try{

            log.info("downloading logs for site with piwik_id: 5");

            Class.forName("org.postgresql.Driver");

            Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword);

            PreparedStatement st = conn.prepareStatement("SELECT max(timestamp) FROM public.piwiklog WHERE source='5';");

            ResultSet rs_date = st.executeQuery();

            while(rs_date.next()){

                start.setTime(sdf.parse(rs_date.getString(1)));

            }

            rs_date.close();

            conn.close();

            for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {

                String period="&period=day&date="+sdf.format(date);

                String apiUrl=getPiwikLogUrl()+APImethod+"&idSite=5"+period+format+"&expanded=5&filter_limit=1000&token_auth="+tokenAuth;

                String content = getJson(apiUrl,piwikUsername,piwikPassword);

                //for (int i=1;i<10;i++){

                int i = 1;

                while(true) {

                    String apiUrlnew=apiUrl+"&filter_offset="+i*1000;

                    String contentNew = getJson(apiUrlnew,piwikUsername,piwikUsername);

                    content += contentNew;

                    i++;

                    if(contentNew.equals("[]\n")){

                        break;

                    }

                }

                flushString(content, logsPath + "portallog/" + "5_Piwiklog"+sdf.format((date))+".json");

            }

        } catch (Exception e) {

            log.error(e);

            throw new Exception("Failed to get portal logs: " + e.toString(), e);

        }

    }

    private void GetRepositoriesLogs() throws Exception{

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");

        Calendar start = Calendar.getInstance();

        start.setTime(startDate);

        Calendar end = Calendar.getInstance();

        end.add(Calendar.DAY_OF_MONTH, -1);

        //end.setTime(getFinalDate());

        try{

            Class.forName("org.postgresql.Driver");

            Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword);

            Statement statement = conn.createStatement();

            ResultSet rs = statement.executeQuery("SELECT distinct piwik_id from shadow.datasource where piwik_id is not null and piwik_id!='5' order by piwik_id;");

            while(rs.next()){

                start.setTime(startDate);

                int siteId = rs.getInt(1);

                log.info("downloading logs for site with piwik_id: " + siteId);

                PreparedStatement st = conn.prepareStatement("SELECT max(timestamp) FROM public.piwiklog WHERE source=?;");

                st.setInt(1, siteId);

                ResultSet rs_date = st.executeQuery();

                while(rs_date.next()){

                    //log.info("source: " + siteId + " - date: " + rs_date.getString(1));

                    if(rs_date.getString(1) == null || rs_date.getString(1).equals("null") || rs_date.getString(1).equals("")) {

                        start = Calendar.getInstance();

                        start.add(Calendar.MONTH, -6);

                    }

                    else {

                        start.setTime(sdf.parse(rs_date.getString(1)));

                    }

                }

                rs_date.close();

                for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {

                    String period="&period=day&date="+sdf.format(date);

                    String apiUrl=getPiwikLogUrl()+APImethod+"&idSite="+siteId+period+format+"&expanded=5&filter_limit=1000&token_auth="+tokenAuth;

                    String content = getJson(apiUrl,piwikUsername,piwikPassword);

                    //for (int i=1;i<10;i++){

                    int i=1;

                    while(true) {

                        String apiUrlnew=apiUrl+"&filter_offset="+i*1000;

                        String contentNew = getJson(apiUrlnew,piwikUsername,piwikPassword);

                        content += contentNew;

                        i++;

                        if(contentNew.equals("[]\n")){

                            break;

                        }

                    }

                    flushString(content, logsPath + "repolog/" + siteId + "_Piwiklog"+sdf.format((date))+".json");

                }

            }

            rs.close();

            conn.close();

        } catch (Exception e) {

            log.error(e);

            throw new Exception("Failed to get repository logs: " + e.toString(), e);

        }

    }

    private void flushString(String data, String destination) throws Exception {

        FSDataOutputStream fin;

        try {

            FileSystem fs = FileSystem.get(new Configuration());

            fin = fs.create(new Path(destination), true);

            fin.write(data.getBytes());

            fin.close();

        } catch (Exception e) {

            log.error("Failed  to write exported data to a file : ", e);

            throw new Exception("Failed  to write exported data to a file : " + e.toString(), e);

        }

    }

}