From 220d1a17480eb85ea3f4cbbbe2ed0db645c9c40c Mon Sep 17 00:00:00 2001 From: Shweelan Samson Date: Sun, 9 Sep 2018 21:16:05 +0300 Subject: [PATCH] Implement cluster and exceptions handler --- config.json | 4 ++ package.json | 2 +- server/cluster.js | 110 ++++++++++++++++++++++++++++++++++++++++++++++ server/logger.js | 8 ++-- server/server.js | 59 ++++++++++++++++++++----- webpack.config.js | 10 ++++- 6 files changed, 175 insertions(+), 18 deletions(-) create mode 100644 server/cluster.js diff --git a/config.json b/config.json index 1217810..39ffa28 100644 --- a/config.json +++ b/config.json @@ -1,5 +1,7 @@ { "server": { + "workers": 4, + "interface": "0.0.0.0", "port": 3000, "nbi": "http://127.0.0.1:7557/", "mongodbConnectionUrl": "mongodb://127.0.0.1:27017", @@ -284,6 +286,8 @@ ] }, "logger": { + "logFile": "", + "accessLogFile": "", "logFormat": "simple", "accessLogFormat": "simple" } diff --git a/package.json b/package.json index 31186a4..fca4ee2 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "test": "echo \"Error: no test specified\" && exit 1", "lint": "eslint --fix common/*.js common/validators/*.js server/*.js client/*.js client/components/*.js;prettier --write *.js common/*.js common/validators/*.js server/*.js client/*.js client/components/*.js client/css/*.css", "build": "npm run lint;webpack;postcss -c postcss.config.js client/css/app.css -o dist/public/app.css", - "start": "cd dist && node server" + "start": "cd dist && node cluster" }, "author": "Zaid Abdulla ", "dependencies": { diff --git a/server/cluster.js b/server/cluster.js new file mode 100644 index 0000000..f77434e --- /dev/null +++ b/server/cluster.js @@ -0,0 +1,110 @@ +"use strict"; + +const cluster = require("cluster"); +const config = require("./config"); +const logger = require("./logger"); + +const VERSION = require("../package.json")["version"]; + +let respawnTimestamp = 0; +let tooManyCrashesTimestamp = 0; + +function restartWorker(worker, code, signal) { + const msg = { + message: "Worker died", + pid: worker.process.pid + }; + + if (code != null) msg.exitCode = code; + + if (signal != null) msg.signal = signal; + + logger.error(msg); + + const now = Date.now(); + respawnTimestamp = Math.max(now, respawnTimestamp + 2000); + if (respawnTimestamp === now) { + tooManyCrashesTimestamp = now; + cluster.fork(); + return; + } + + if (now - tooManyCrashesTimestamp > 60000) { + process.exitCode = 1; + cluster.removeListener("exit", restartWorker); + for (const pid in cluster.workers) cluster.workers[pid].kill(); + + logger.error({ + message: "Too many crashes, exiting", + pid: process.pid + }); + return; + } + + setTimeout(() => { + if (process.exitCode) return; + cluster.fork(); + }, respawnTimestamp - now); +} + +function start() { + logger.init(VERSION); + + logger.info({ + message: "GenieACS UI starting", + pid: process.pid, + version: VERSION + }); + + cluster.on("listening", (worker, address) => { + logger.info({ + message: "Worker listening", + pid: worker.process.pid, + address: address.address, + port: address.port + }); + }); + + cluster.on("exit", restartWorker); + + process.on("SIGINT", () => { + logger.info({ + message: "Received signal SIGINT, exiting", + pid: process.pid + }); + + cluster.removeListener("exit", restartWorker); + }); + + process.on("SIGTERM", () => { + logger.info({ + message: "Received signal SIGTERM, exiting", + pid: process.pid + }); + + cluster.removeListener("exit", restartWorker); + for (const pid in cluster.workers) cluster.workers[pid].kill(); + }); + + cluster.setupMaster({ exec: "./server" }); + + let workerCount = config.server.workers; + + if (!workerCount) workerCount = Math.max(2, require("os").cpus().length); + + for (let i = 0; i < workerCount; ++i) cluster.fork(); +} + +process.on("uncaughtException", err => { + logger.error({ + message: "Uncaught exception in master process, exiting", + exception: err, + pid: process.pid + }); + process.exitCode = 1; + cluster.removeListener("exit", restartWorker); + for (const pid in cluster.workers) cluster.workers[pid].kill(); +}); + +// Starting GenieACS UI +start(); diff --git a/server/logger.js b/server/logger.js index 7e388e8..aab3345 100644 --- a/server/logger.js +++ b/server/logger.js @@ -72,14 +72,14 @@ function reopen() { setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref(); } -function init(service, version) { +function init(version) { defaultMeta.hostname = os.hostname(); defaultMeta.pid = process.pid; - defaultMeta.name = `genieacs-ui-${service}`; + defaultMeta.name = "genieacs-ui"; defaultMeta.version = version; - LOG_FILE = config.logger[`${service}LogFile`]; - ACCESS_LOG_FILE = config.logger[`${service}AccessLogFile`]; + LOG_FILE = config.logger.logFile; + ACCESS_LOG_FILE = config.logger.accessLogFile; if (LOG_FILE) { logStream = fs.createWriteStream(null, { fd: fs.openSync(LOG_FILE, "a") }); diff --git a/server/server.js b/server/server.js index d0a8894..e759443 100644 --- a/server/server.js +++ b/server/server.js @@ -1,5 +1,7 @@ "use strict"; +const http = require("http"); +const cluster = require("cluster"); const Koa = require("koa"); const Router = require("koa-router"); const koaStatic = require("koa-static"); @@ -13,6 +15,10 @@ const api = require("./api"); const Authorizer = require("../common/authorizer"); const expression = require("../common/expression"); const logger = require("./logger"); +const db = require("./db"); + +const VERSION = require("../package.json")["version"]; +logger.init("server", VERSION); const koa = new Koa(); const router = new Router(); @@ -20,14 +26,6 @@ const router = new Router(); const JWT_SECRET = config.server.jwtSecret; const JWT_COOKIE = "genieacs-ui-jwt"; -const VERSION = require("../package.json")["version"]; - -logger.info({ - message: "GenieACS UI starting", - pid: process.pid, - version: VERSION -}); - function getPermissionSets(roles) { const allPermissions = config.permissions; const permissionSets = roles.map(role => { @@ -149,10 +147,47 @@ koa.use( koa.use(router.routes()); koa.use(koaStatic("./public")); -koa.listen(config.server.port, "0.0.0.0", () => { - logger.info({ - message: "Worker litening", - port: config.server.port, +let server = http + .createServer(koa.callback()) + .listen(config.server.port, config.server.interface); + +function closeServer(timeout) { + return new Promise(resolve => { + setTimeout(() => { + if (!resolve) return; + // Ignore HTTP requests from alive connections + server.removeListener("request", koa.callback()); + server.setTimeout(1); + + setTimeout(resolve, 1000); + }, timeout).unref(); + + // prevent new sockets to connect and close server eventually. + server.close(resolve); + }); +} + +function exit() { + setTimeout(() => { + process.exit(1); + }, 30000).unref(); + + closeServer(20000).then(() => { + db.disconnect(); + if (cluster.worker) cluster.worker.disconnect(); + logger.close(); + }); +} + +process.on("uncaughtException", err => { + logger.error({ + message: "Uncaught exception", + exception: err, pid: process.pid }); + process.exitCode = 1; + exit(); }); + +process.on("SIGINT", exit); +process.on("SIGTERM", exit); diff --git a/webpack.config.js b/webpack.config.js index 4de8a8b..d2aee54 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -8,7 +8,7 @@ function transformPackageJson(content) { const pkg = JSON.parse(content); delete pkg["devDependencies"]; pkg["scripts"] = { - start: "node server" + start: "node cluster" }; return JSON.stringify(pkg, null, 2); } @@ -39,6 +39,14 @@ module.exports = [ ) ] }, + { + mode: mode, + entry: "./server/cluster.js", + target: "node", + output: { + filename: "cluster.js" + } + }, { mode: mode, entry: "./client/app.js",