| 1 |
56738
|
argiro.kok
|
'use strict';
|
| 2 |
49707
|
argiro.kok
|
|
| 3 |
56738
|
argiro.kok
|
let express = require('express');
|
| 4 |
|
|
let app = express();
|
| 5 |
|
|
let mcache = require('memory-cache');
|
| 6 |
49707
|
argiro.kok
|
const request = require('superagent');
|
| 7 |
58225
|
k.triantaf
|
const prom = require('prom-client');
|
| 8 |
58699
|
argiro.kok
|
const URL = require('url');
|
| 9 |
59001
|
argiro.kok
|
const expireShort = 2 * 60 * 1000; //2mins
|
| 10 |
|
|
const expireLong = 24 * 60 * 60 * 1000; //24 hours
|
| 11 |
61217
|
k.triantaf
|
const cacheMaxSize = 500;
|
| 12 |
59649
|
konstantin
|
const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent",
|
| 13 |
61217
|
k.triantaf
|
"/provision/mvc/vocabularies/", "/pages?page_route=",
|
| 14 |
|
|
"/allmetrics", "/countryusagestats/", "/openaire/info",
|
| 15 |
|
|
"/api/communities/", "/openaire/contexts/"];
|
| 16 |
58225
|
k.triantaf
|
let cors = require('cors');
|
| 17 |
|
|
app.use(cors());
|
| 18 |
49714
|
argiro.kok
|
|
| 19 |
58225
|
k.triantaf
|
const register = new prom.Registry();
|
| 20 |
58478
|
k.triantaf
|
prom.collectDefaultMetrics({register: register});
|
| 21 |
49714
|
argiro.kok
|
|
| 22 |
58225
|
k.triantaf
|
const responses = new prom.Counter({
|
| 23 |
|
|
name: 'cache_http_responses_total',
|
| 24 |
|
|
help: 'A counter for cache response codes for every API request.',
|
| 25 |
58495
|
k.triantaf
|
labelNames: ['scheme', 'target', 'code'],
|
| 26 |
58225
|
k.triantaf
|
registers: [register]
|
| 27 |
|
|
});
|
| 28 |
49714
|
argiro.kok
|
|
| 29 |
58495
|
k.triantaf
|
const entries = new prom.Gauge({
|
| 30 |
58478
|
k.triantaf
|
name: 'cache_used_entries',
|
| 31 |
|
|
help: 'A counter to count cache entries',
|
| 32 |
|
|
registers: [register]
|
| 33 |
|
|
});
|
| 34 |
|
|
|
| 35 |
58225
|
k.triantaf
|
const histogram = new prom.Histogram({
|
| 36 |
|
|
name: 'cache_http_request_duration_seconds',
|
| 37 |
58478
|
k.triantaf
|
help: 'A Histogram for cache. Providing information about a cache request and load latency in seconds.',
|
| 38 |
58495
|
k.triantaf
|
labelNames: ['scheme', 'target', 'cache'],
|
| 39 |
58225
|
k.triantaf
|
registers: [register],
|
| 40 |
|
|
buckets: [0.1, 0.2, 0.5, 1, 2]
|
| 41 |
|
|
});
|
| 42 |
49707
|
argiro.kok
|
|
| 43 |
59001
|
argiro.kok
|
let cache = () => {
|
| 44 |
58225
|
k.triantaf
|
return (req, res, next) => {
|
| 45 |
61217
|
k.triantaf
|
if (req.query.url) {
|
| 46 |
|
|
if (mcache.memsize() > cacheMaxSize) {
|
| 47 |
|
|
console.log("Max cache size reached!" + cacheMaxSize);
|
| 48 |
59001
|
argiro.kok
|
clearCache();
|
| 49 |
|
|
}
|
| 50 |
58495
|
k.triantaf
|
let key = '__express__' + req.query.url;
|
| 51 |
59001
|
argiro.kok
|
let longCache = checkForLongCachedRequests(req.query.url);
|
| 52 |
58495
|
k.triantaf
|
let cachedBody = mcache.get(key);
|
| 53 |
58699
|
argiro.kok
|
const url = new URL.parse(req.query.url);
|
| 54 |
58495
|
k.triantaf
|
const target = url.host + '/' + url.pathname.split('/')[1];
|
| 55 |
|
|
const scheme = url.protocol.replace(':', '');
|
| 56 |
|
|
if (cachedBody) {
|
| 57 |
|
|
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'});
|
| 58 |
|
|
res.send(JSON.parse(cachedBody));
|
| 59 |
|
|
responses.inc({scheme: scheme, target: target, code: res.statusCode});
|
| 60 |
|
|
end();
|
| 61 |
|
|
} else {
|
| 62 |
|
|
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'});
|
| 63 |
|
|
res.sendResponse = res.send;
|
| 64 |
|
|
res.send = (body) => {
|
| 65 |
|
|
let alreadyCached = !!mcache.get(key);
|
| 66 |
59194
|
k.triantaf
|
entries.set(mcache.memsize());
|
| 67 |
61217
|
k.triantaf
|
if (!alreadyCached) {
|
| 68 |
58495
|
k.triantaf
|
responses.inc({scheme: scheme, target: target, code: res.statusCode});
|
| 69 |
|
|
end();
|
| 70 |
|
|
}
|
| 71 |
|
|
if (res.statusCode === 200) {
|
| 72 |
59001
|
argiro.kok
|
// console.log("Expire in " +(longCache?expireLong:expireShort) + " " +req.query.url);
|
| 73 |
61217
|
k.triantaf
|
mcache.put(key, body, longCache ? expireLong : expireShort, () => {
|
| 74 |
59194
|
k.triantaf
|
entries.set(mcache.memsize());
|
| 75 |
58495
|
k.triantaf
|
});
|
| 76 |
|
|
}
|
| 77 |
|
|
res.sendResponse(body);
|
| 78 |
|
|
};
|
| 79 |
|
|
next();
|
| 80 |
|
|
}
|
| 81 |
58225
|
k.triantaf
|
} else {
|
| 82 |
|
|
next();
|
| 83 |
|
|
}
|
| 84 |
|
|
};
|
| 85 |
58226
|
k.triantaf
|
};
|
| 86 |
49707
|
argiro.kok
|
|
| 87 |
59001
|
argiro.kok
|
app.get('/clear', (req, res) => {
|
| 88 |
|
|
let c = mcache.size();
|
| 89 |
|
|
const url = req.query.url;
|
| 90 |
61217
|
k.triantaf
|
let message = "";
|
| 91 |
|
|
if (url) {
|
| 92 |
59001
|
argiro.kok
|
let key = '__express__' + req.query.url;
|
| 93 |
|
|
mcache.del(key);
|
| 94 |
61217
|
k.triantaf
|
message = "Delete entry with key " + url;
|
| 95 |
59001
|
argiro.kok
|
entries.set(mcache.size());
|
| 96 |
61217
|
k.triantaf
|
} else {
|
| 97 |
59001
|
argiro.kok
|
clearCache();
|
| 98 |
61217
|
k.triantaf
|
message = "Delete " + c + " entries. Now there are: " + mcache.size()
|
| 99 |
|
|
}
|
| 100 |
59001
|
argiro.kok
|
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
|
| 101 |
|
|
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
| 102 |
|
|
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
| 103 |
|
|
res.header("Content-Type", "application/json");
|
| 104 |
61217
|
k.triantaf
|
res.status(200).send(getResponse(200, message));
|
| 105 |
49714
|
argiro.kok
|
|
| 106 |
59001
|
argiro.kok
|
});
|
| 107 |
58225
|
k.triantaf
|
app.get('/metrics', (req, res) => {
|
| 108 |
|
|
res.set('Content-Type', register.contentType);
|
| 109 |
|
|
res.end(register.metrics());
|
| 110 |
|
|
});
|
| 111 |
|
|
|
| 112 |
59001
|
argiro.kok
|
app.get('/get', cache(), cors(), (req, res) => {
|
| 113 |
|
|
setTimeout(() => {
|
| 114 |
|
|
const url = (req.query) ? req.query.url : null;
|
| 115 |
|
|
if (!url) {
|
| 116 |
|
|
res.status(404).send(getResponse(404, "Not Found ")) //not found
|
| 117 |
|
|
} else {
|
| 118 |
|
|
request.get(url, function (err, response) {
|
| 119 |
|
|
// res.header("Access-Control-Allow-Origin", "http://localhost:3000");
|
| 120 |
|
|
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
|
| 121 |
|
|
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
| 122 |
|
|
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
| 123 |
|
|
res.header("Content-Type", "application/json");
|
| 124 |
|
|
if (!response && err) {
|
| 125 |
|
|
res.status(500).send(getResponse(500, "An error occurred for " + url))
|
| 126 |
|
|
} else {
|
| 127 |
|
|
res.status(response.status).send(response.body);
|
| 128 |
|
|
}
|
| 129 |
|
|
})
|
| 130 |
|
|
}
|
| 131 |
|
|
})
|
| 132 |
|
|
});
|
| 133 |
|
|
|
| 134 |
|
|
|
| 135 |
49707
|
argiro.kok
|
app.use((req, res) => {
|
| 136 |
58225
|
k.triantaf
|
res.status(404).send(getResponse(404, "Not Found")); //not found
|
| 137 |
56738
|
argiro.kok
|
});
|
| 138 |
49707
|
argiro.kok
|
|
| 139 |
58225
|
k.triantaf
|
app.listen((process.env.PORT) ? process.env.PORT : 3000, function () {
|
| 140 |
59002
|
argiro.kok
|
console.log(`Example app listening on port ${(process.env.PORT) ? process.env.PORT : 3000}!`)
|
| 141 |
|
|
//run the timer
|
| 142 |
|
|
resetAtMidnight();
|
| 143 |
56738
|
argiro.kok
|
});
|
| 144 |
52744
|
argiro.kok
|
|
| 145 |
58225
|
k.triantaf
|
function getResponse(code, message) {
|
| 146 |
|
|
var response = {};
|
| 147 |
|
|
response["code"] = code;
|
| 148 |
|
|
response["message"] = message;
|
| 149 |
|
|
return response;
|
| 150 |
61217
|
k.triantaf
|
}
|
| 151 |
52744
|
argiro.kok
|
|
| 152 |
61217
|
k.triantaf
|
function clearCache() {
|
| 153 |
59002
|
argiro.kok
|
console.log("cache is cleared!");
|
| 154 |
59001
|
argiro.kok
|
mcache.clear();
|
| 155 |
|
|
entries.set(mcache.size());
|
| 156 |
|
|
}
|
| 157 |
61217
|
k.triantaf
|
|
| 158 |
|
|
function checkForLongCachedRequests(url) {
|
| 159 |
|
|
let long = false;
|
| 160 |
59001
|
argiro.kok
|
longCachingRequests.forEach(partUrl => {
|
| 161 |
61217
|
k.triantaf
|
if (url.indexOf(partUrl) !== -1) {
|
| 162 |
59001
|
argiro.kok
|
long = true;
|
| 163 |
|
|
}
|
| 164 |
|
|
});
|
| 165 |
|
|
return long;
|
| 166 |
|
|
}
|
| 167 |
61217
|
k.triantaf
|
|
| 168 |
59001
|
argiro.kok
|
function resetAtMidnight() {
|
| 169 |
59002
|
argiro.kok
|
console.log("Run Reset timer");
|
| 170 |
59001
|
argiro.kok
|
var now = new Date();
|
| 171 |
|
|
var night = new Date(
|
| 172 |
|
|
now.getFullYear(),
|
| 173 |
|
|
now.getMonth(),
|
| 174 |
61217
|
k.triantaf
|
now.getDate() + 1, // the next day, ...
|
| 175 |
|
|
0, 0, 0 // ...at 00:00:00 hours
|
| 176 |
59001
|
argiro.kok
|
);
|
| 177 |
|
|
var msToMidnight = night.getTime() - now.getTime();
|
| 178 |
61217
|
k.triantaf
|
setTimeout(function () {
|
| 179 |
59001
|
argiro.kok
|
clearCache(); // <-- This is the function being called at midnight.
|
| 180 |
|
|
resetAtMidnight(); // Then, reset again next midnight.
|
| 181 |
|
|
}, msToMidnight);
|
| 182 |
|
|
}
|