D-Net

'use strict';

let express = require('express');

let app = express();

let mcache = require('memory-cache');

const request = require('superagent');

const prom = require('prom-client');

const URL = require('url');

const expireShort = 2 * 60 * 1000; //2mins

const expireLong = 24 * 60 * 60 * 1000; //24 hours

const cacheMaxSize = 500;

const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent",

    "/provision/mvc/vocabularies/", "/pages?page_route=",

    "/allmetrics", "/countryusagestats/", "/openaire/info",

    "/api/communities/", "/openaire/contexts/"];

let cors = require('cors');

app.use(cors());

const register = new prom.Registry();

prom.collectDefaultMetrics({register: register});

const responses = new prom.Counter({

    name: 'cache_http_responses_total',

    help: 'A counter for cache response codes for every API request.',

    labelNames: ['scheme', 'target', 'code'],

    registers: [register]

});

const entries = new prom.Gauge({

    name: 'cache_used_entries',

    help: 'A counter to count cache entries',

    registers: [register]

});

const histogram = new prom.Histogram({

    name: 'cache_http_request_duration_seconds',

    help: 'A Histogram for cache. Providing information about a cache request and load latency in seconds.',

    labelNames: ['scheme', 'target', 'cache'],

    registers: [register],

    buckets: [0.1, 0.2, 0.5, 1, 2]

});

let cache = () => {

    return (req, res, next) => {

        if (req.query.url) {

            if (mcache.memsize() > cacheMaxSize) {

                console.log("Max cache size reached!" + cacheMaxSize);

                clearCache();

            }

            let key = '__express__' + req.query.url;

            let longCache = checkForLongCachedRequests(req.query.url);

            let cachedBody = mcache.get(key);

            const url = new URL.parse(req.query.url);

            const target = url.host + '/' + url.pathname.split('/')[1];

            const scheme = url.protocol.replace(':', '');

            if (cachedBody) {

                const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'});

                res.send(JSON.parse(cachedBody));

                responses.inc({scheme: scheme, target: target, code: res.statusCode});

                end();

            } else {

                const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'});

                res.sendResponse = res.send;

                res.send = (body) => {

                    let alreadyCached = !!mcache.get(key);

                    entries.set(mcache.memsize());

                    if (!alreadyCached) {

                        responses.inc({scheme: scheme, target: target, code: res.statusCode});

                        end();

                    }

                    if (res.statusCode === 200) {

                        // console.log("Expire in " +(longCache?expireLong:expireShort) + " " +req.query.url);

                        mcache.put(key, body, longCache ? expireLong : expireShort, () => {

                            entries.set(mcache.memsize());

                        });

                    }

                    res.sendResponse(body);

                };

                next();

            }

        } else {

            next();

        }

    };

};

app.get('/clear', (req, res) => {

    let c = mcache.size();

    const url = req.query.url;

    let message = "";

    if (url) {

        let key = '__express__' + req.query.url;

        mcache.del(key);

        message = "Delete entry with key " + url;

        entries.set(mcache.size());

    } else {

        clearCache();

        message = "Delete " + c + " entries. Now there are: " + mcache.size()

    }

    res.header("Access-Control-Allow-Headers", "Origin, Content-Type,  Content-Length");

    res.header("Access-Control-Allow-Methods", "GET,  OPTIONS");

    res.header("Access-Control-Allow-Methods", "GET,  OPTIONS");

    res.header("Content-Type", "application/json");

    res.status(200).send(getResponse(200, message));

});

app.get('/metrics', (req, res) => {

    res.set('Content-Type', register.contentType);

    res.end(register.metrics());

});

app.get('/get', cache(), cors(), (req, res) => {

    setTimeout(() => {

        const url = (req.query) ? req.query.url : null;

        if (!url) {

            res.status(404).send(getResponse(404, "Not Found ")) //not found

        } else {

            request.get(url, function (err, response) {

                // res.header("Access-Control-Allow-Origin", "http://localhost:3000");

                res.header("Access-Control-Allow-Headers", "Origin, Content-Type,  Content-Length");

                res.header("Access-Control-Allow-Methods", "GET,  OPTIONS");

                res.header("Access-Control-Allow-Methods", "GET,  OPTIONS");

                res.header("Content-Type", "application/json");

                if (!response && err) {

                    res.status(500).send(getResponse(500, "An error occurred for " + url))

                } else {

                    res.status(response.status).send(response.body);

                }

            })

        }

    })

});

app.use((req, res) => {

    res.status(404).send(getResponse(404, "Not Found")); //not found

});

app.listen((process.env.PORT) ? process.env.PORT : 3000, function () {

    console.log(`Example app listening on port ${(process.env.PORT) ? process.env.PORT : 3000}!`)

    //run the timer

    resetAtMidnight();

});

function getResponse(code, message) {

    var response = {};

    response["code"] = code;

    response["message"] = message;

    return response;

}

function clearCache() {

    console.log("cache is cleared!");

    mcache.clear();

    entries.set(mcache.size());

}

function checkForLongCachedRequests(url) {

    let long = false;

    longCachingRequests.forEach(partUrl => {

        if (url.indexOf(partUrl) !== -1) {

            long = true;

        }

    });

    return long;

}

function resetAtMidnight() {

    console.log("Run Reset timer");

    var now = new Date();

    var night = new Date(

        now.getFullYear(),

        now.getMonth(),

        now.getDate() + 1, //  the next day, ...

        0, 0, 0 // ...at 00:00:00 hours

    );

    var msToMidnight = night.getTime() - now.getTime();

    setTimeout(function () {

        clearCache();              //      <-- This is the function being called at midnight.

        resetAtMidnight();    //      Then, reset again next midnight.

    }, msToMidnight);

}