Implement cluster and exceptions handler

config.json

@@ -1,5 +1,7 @@
 {
   "server": {
+    "workers": 4,
+    "interface": "0.0.0.0",
     "port": 3000,
     "nbi": "http://127.0.0.1:7557/",
     "mongodbConnectionUrl": "mongodb://127.0.0.1:27017",
@@ -284,6 +286,8 @@
     ]
   },
   "logger": {
+    "logFile": "",
+    "accessLogFile": "",
     "logFormat": "simple",
     "accessLogFormat": "simple"
   }

package.json

@@ -6,7 +6,7 @@
     "test": "echo \"Error: no test specified\" && exit 1",
     "lint": "eslint --fix common/*.js common/validators/*.js server/*.js client/*.js client/components/*.js;prettier --write *.js common/*.js common/validators/*.js server/*.js client/*.js client/components/*.js client/css/*.css",
     "build": "npm run lint;webpack;postcss -c postcss.config.js client/css/app.css -o dist/public/app.css",
-    "start": "cd dist && node server"
+    "start": "cd dist && node cluster"
   },
   "author": "Zaid Abdulla <zaid@genieacs.com>",
   "dependencies": {

server/cluster.js

@@ -0,0 +1,110 @@
+"use strict";
+
+const cluster = require("cluster");
+const config = require("./config");
+const logger = require("./logger");
+
+const VERSION = require("../package.json")["version"];
+
+let respawnTimestamp = 0;
+let tooManyCrashesTimestamp = 0;
+
+function restartWorker(worker, code, signal) {
+  const msg = {
+    message: "Worker died",
+    pid: worker.process.pid
+  };
+
+  if (code != null) msg.exitCode = code;
+
+  if (signal != null) msg.signal = signal;
+
+  logger.error(msg);
+
+  const now = Date.now();
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    tooManyCrashesTimestamp = now;
+    cluster.fork();
+    return;
+  }
+
+  if (now - tooManyCrashesTimestamp > 60000) {
+    process.exitCode = 1;
+    cluster.removeListener("exit", restartWorker);
+    for (const pid in cluster.workers) cluster.workers[pid].kill();
+
+    logger.error({
+      message: "Too many crashes, exiting",
+      pid: process.pid
+    });
+    return;
+  }
+
+  setTimeout(() => {
+    if (process.exitCode) return;
+    cluster.fork();
+  }, respawnTimestamp - now);
+}
+
+function start() {
+  logger.init(VERSION);
+
+  logger.info({
+    message: "GenieACS UI starting",
+    pid: process.pid,
+    version: VERSION
+  });
+
+  cluster.on("listening", (worker, address) => {
+    logger.info({
+      message: "Worker listening",
+      pid: worker.process.pid,
+      address: address.address,
+      port: address.port
+    });
+  });
+
+  cluster.on("exit", restartWorker);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid
+    });
+
+    cluster.removeListener("exit", restartWorker);
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid
+    });
+
+    cluster.removeListener("exit", restartWorker);
+    for (const pid in cluster.workers) cluster.workers[pid].kill();
+  });
+
+  cluster.setupMaster({ exec: "./server" });
+
+  let workerCount = config.server.workers;
+
+  if (!workerCount) workerCount = Math.max(2, require("os").cpus().length);
+
+  for (let i = 0; i < workerCount; ++i) cluster.fork();
+}
+
+process.on("uncaughtException", err => {
+  logger.error({
+    message: "Uncaught exception in master process, exiting",
+    exception: err,
+    pid: process.pid
+  });
+  process.exitCode = 1;
+  cluster.removeListener("exit", restartWorker);
+  for (const pid in cluster.workers) cluster.workers[pid].kill();
+});
+
+// Starting GenieACS UI
+start();

server/logger.js

@@ -72,14 +72,14 @@ function reopen() {
     setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
 }
 
-function init(service, version) {
+function init(version) {
   defaultMeta.hostname = os.hostname();
   defaultMeta.pid = process.pid;
-  defaultMeta.name = `genieacs-ui-${service}`;
+  defaultMeta.name = "genieacs-ui";
   defaultMeta.version = version;
 
-  LOG_FILE = config.logger[`${service}LogFile`];
+  LOG_FILE = config.logger.logFile;
-  ACCESS_LOG_FILE = config.logger[`${service}AccessLogFile`];
+  ACCESS_LOG_FILE = config.logger.accessLogFile;
 
   if (LOG_FILE) {
     logStream = fs.createWriteStream(null, { fd: fs.openSync(LOG_FILE, "a") });

server/server.js

@@ -1,5 +1,7 @@
 "use strict";
 
+const http = require("http");
+const cluster = require("cluster");
 const Koa = require("koa");
 const Router = require("koa-router");
 const koaStatic = require("koa-static");
@@ -13,6 +15,10 @@ const api = require("./api");
 const Authorizer = require("../common/authorizer");
 const expression = require("../common/expression");
 const logger = require("./logger");
+const db = require("./db");
+
+const VERSION = require("../package.json")["version"];
+logger.init("server", VERSION);
 
 const koa = new Koa();
 const router = new Router();
@@ -20,14 +26,6 @@ const router = new Router();
 const JWT_SECRET = config.server.jwtSecret;
 const JWT_COOKIE = "genieacs-ui-jwt";
 
-const VERSION = require("../package.json")["version"];
-
-logger.info({
-  message: "GenieACS UI starting",
-  pid: process.pid,
-  version: VERSION
-});
-
 function getPermissionSets(roles) {
   const allPermissions = config.permissions;
   const permissionSets = roles.map(role => {
@@ -149,10 +147,47 @@ koa.use(
 koa.use(router.routes());
 koa.use(koaStatic("./public"));
 
-koa.listen(config.server.port, "0.0.0.0", () => {
+let server = http
-  logger.info({
+  .createServer(koa.callback())
-    message: "Worker litening",
+  .listen(config.server.port, config.server.interface);
-    port: config.server.port,
+
+function closeServer(timeout) {
+  return new Promise(resolve => {
+    setTimeout(() => {
+      if (!resolve) return;
+      // Ignore HTTP requests from alive connections
+      server.removeListener("request", koa.callback());
+      server.setTimeout(1);
+
+      setTimeout(resolve, 1000);
+    }, timeout).unref();
+
+    // prevent new sockets to connect and close server eventually.
+    server.close(resolve);
+  });
+}
+
+function exit() {
+  setTimeout(() => {
+    process.exit(1);
+  }, 30000).unref();
+
+  closeServer(20000).then(() => {
+    db.disconnect();
+    if (cluster.worker) cluster.worker.disconnect();
+    logger.close();
+  });
+}
+
+process.on("uncaughtException", err => {
+  logger.error({
+    message: "Uncaught exception",
+    exception: err,
     pid: process.pid
   });
+  process.exitCode = 1;
+  exit();
 });
+
+process.on("SIGINT", exit);
+process.on("SIGTERM", exit);

webpack.config.js

@@ -8,7 +8,7 @@ function transformPackageJson(content) {
   const pkg = JSON.parse(content);
   delete pkg["devDependencies"];
   pkg["scripts"] = {
-    start: "node server"
+    start: "node cluster"
   };
   return JSON.stringify(pkg, null, 2);
 }
@@ -39,6 +39,14 @@ module.exports = [
       )
     ]
   },
+  {
+    mode: mode,
+    entry: "./server/cluster.js",
+    target: "node",
+    output: {
+      filename: "cluster.js"
+    }
+  },
   {
     mode: mode,
     entry: "./client/app.js",