diff --git a/lib/cluster.js b/lib/cluster.js index 19bafa9..5ce19fc 100644 --- a/lib/cluster.js +++ b/lib/cluster.js @@ -24,6 +24,8 @@ const config = require("./config"); const VERSION = require("../package.json").version; +let respawnTimestamp = 0; +let tooManyCrashesTimestamp = 0; function getDependencyVersions() { let dependencies = [`node@${process.versions.node}`]; @@ -67,7 +69,32 @@ function restartWorker(worker, code, signal) { msg.signal = signal; logger.error(msg); - cluster.fork(); + + const now = Date.now(); + respawnTimestamp = Math.max(now, respawnTimestamp + 2000); + if (respawnTimestamp === now) { + tooManyCrashesTimestamp = now; + cluster.fork(); + return + } + + if (now - tooManyCrashesTimestamp > 60000) { + process.exitCode = 1; + cluster.removeListener("exit", restartWorker); + for (let pid in cluster.workers) + cluster.workers[pid].kill(); + + logger.error({ + message: "Too many crashes, exiting", + pid: process.pid + }); + return; + } + + setTimeout(function() { + if (process.exitCode) return; + cluster.fork(); + }, respawnTimestamp - now); } @@ -140,11 +167,10 @@ process.on("uncaughtException", function(err) { exception: err, pid: process.pid }); + process.exitCode = 1; cluster.removeListener("exit", restartWorker); for (let pid in cluster.workers) cluster.workers[pid].kill(); - - logger.close(); }); diff --git a/lib/server.coffee b/lib/server.coffee index e23cdf3..89e1872 100644 --- a/lib/server.coffee +++ b/lib/server.coffee @@ -70,13 +70,11 @@ exit = () -> ) , 30000).unref() - cluster.worker?.disconnect() - closeServer(20000, () -> db.disconnect() cache.disconnect() extensions.killAll() - logger.close() + cluster.worker?.disconnect() )