D-Net

/*

 * To change this license header, choose License Headers in Project Properties.

 * To change this template file, choose Tools | Templates

 * and open the template in the editor.

 */

package eu.dnetlib.usagestats.export;

/**

 *

 * @author dpie

 */

import java.io.InputStream;

import java.sql.Connection;

import java.sql.DriverManager;

import java.util.Properties;

import org.apache.log4j.Logger;

public class ConnectDB {

    public static Connection DB_CONNECTION;

    private final String dbURL;

    private final String dbUsername;

    private final String dbPassword;

    private final Logger log = Logger.getLogger(this.getClass());

    public ConnectDB() throws Exception {

        Properties prop = new Properties();

        InputStream propertiesInputStream = executeWorkflowNew.class.getClassLoader().getResourceAsStream("usagestats.properties");

        prop.load(propertiesInputStream);

        Class.forName(prop.getProperty("db.driverClassName"));

        this.dbURL = prop.getProperty("db.Url");

        this.dbUsername = prop.getProperty("db.username");

        this.dbPassword = prop.getProperty("db.password");

        this.connectDB();

    }

    private void connectDB() throws Exception {

        try {

            Class.forName("org.postgresql.Driver");

            DB_CONNECTION = DriverManager.getConnection(dbURL, dbUsername, dbPassword);

            log.info("Opened database successfully");

        } catch (Exception e) {

            log.error("Connect to db failed: " + e);

            throw new Exception("Failed to connect to db: " + e.toString(), e);

        }

    }

}

/*

 * To change this license header, choose License Headers in Project Properties.

 * To change this template file, choose Tools | Templates

 * and open the template in the editor.

 */

package eu.dnetlib.usagestats.export;

/**

 *

 * @author dpie

 */

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.MalformedURLException;

import java.net.URL;

import java.nio.charset.Charset;

import java.util.ArrayList;

import org.json.JSONException;

import org.json.simple.JSONArray;

import org.json.simple.parser.JSONParser;

import org.json.simple.parser.ParseException;

public class ReadCounterRobotsList {

    private ArrayList robotsPatterns = new ArrayList();

    private String COUNTER_ROBOTS_URL;

    public ReadCounterRobotsList(String url) throws IOException, JSONException, ParseException {

        COUNTER_ROBOTS_URL = url;

        robotsPatterns = readRobotsPartners(COUNTER_ROBOTS_URL);

    }

    private ArrayList readRobotsPartners(String url) throws MalformedURLException, IOException, ParseException {

        InputStream is = new URL(url).openStream();

        JSONParser parser = new JSONParser();

        BufferedReader reader = new BufferedReader(new InputStreamReader(is, Charset.forName("ISO-8859-1")));

        JSONArray jsonArray = (JSONArray) parser.parse(reader);

        for (Object aJsonArray : jsonArray) {

            org.json.simple.JSONObject jsonObjectRow = (org.json.simple.JSONObject) aJsonArray;

            robotsPatterns.add(jsonObjectRow.get("pattern").toString().replace("\\", "\\\\"));

        }

        return robotsPatterns;

    }

    public ArrayList getRobotsPatterns(){

        return robotsPatterns;

    }

}

/*

 * To change this license header, choose License Headers in Project Properties.

 * To change this template file, choose Tools | Templates

 * and open the template in the editor.

 */

package eu.dnetlib.usagestats.export;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileReader;

import java.io.IOException;

import java.io.InputStream;

import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.util.ArrayList;

import java.util.Iterator;

import java.util.Properties;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

/**

 *

 * @author dpie

 */

public class executeWorkflow {

    public static void main(String args[]) throws ClassNotFoundException, SQLException, Exception {

        UsageStatsExporter usagestatsExport =new UsageStatsExporter();

        usagestatsExport.export();

    }

}

#logging.config=log4j.properties

db.driverClassName=org.postgresql.Driver

#spring.datasource.url=jdbc:postgresql://88.197.53.70:5432/stats

#spring.datasource.url=jdbc:postgresql://statsdb-beta.openaire.eu:5432/stats

db.Url=jdbc:postgresql://88.197.53.70:5432/stats2

#db.Url=jdbc:postgresql://statsdb-prod.openaire.eu/stats

db.username=sqoop

db.password=sqoop

matomo.AuthToken=703bd17d845acdaf795e01bb1e0895b9

matomo.BaseUrl=analytics.openaire.eu

repo.LogPath=/Users/dpie/Desktop/Repologs/

portal.LogPath=/Users/dpie/Desktop/Portallogs/

portal.MatomoID=109

COUNTER.robots.Url=https://raw.githubusercontent.com/atmire/COUNTER-Robots/master/COUNTER_Robots_list.json

IRUS_UK.BaseUrl=https://irus.jisc.ac.uk/api/sushilite/v1_7/

#server.compression.enabled=true

#compression.max_number_of_records=1000

#usagestats.redis.hostname=localhost

#usagestats.redis.port=6379

#spring.jackson.serialization.INDENT_OUTPUT=true

#download.folder=/Users/dpie/DownloadSushiLite

#sushi-lite.server=http://localhost:8080