Exit main process after too many exceptions

lib/cluster.js

@@ -24,6 +24,8 @@ const config = require("./config");
 
 const VERSION = require("../package.json").version;
 
+let respawnTimestamp = 0;
+let tooManyCrashesTimestamp = 0;
 
 function getDependencyVersions() {
   let dependencies = [`node@${process.versions.node}`];
@@ -67,7 +69,32 @@ function restartWorker(worker, code, signal) {
     msg.signal = signal;
 
   logger.error(msg);
-  cluster.fork();
+
+  const now = Date.now();
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    tooManyCrashesTimestamp = now;
+    cluster.fork();
+    return
+  }
+
+  if (now - tooManyCrashesTimestamp > 60000) {
+    process.exitCode = 1;
+    cluster.removeListener("exit", restartWorker);
+    for (let pid in cluster.workers)
+      cluster.workers[pid].kill();
+
+    logger.error({
+      message: "Too many crashes, exiting",
+      pid: process.pid
+    });
+    return;
+  }
+
+  setTimeout(function() {
+    if (process.exitCode) return;
+    cluster.fork();
+  }, respawnTimestamp - now);
 }
 
 
@@ -140,11 +167,10 @@ process.on("uncaughtException", function(err) {
     exception: err,
     pid: process.pid
   });
+  process.exitCode = 1;
   cluster.removeListener("exit", restartWorker);
   for (let pid in cluster.workers)
     cluster.workers[pid].kill();
-
-  logger.close();
 });
 
 

lib/server.coffee

@@ -70,13 +70,11 @@ exit = () ->
     )
   , 30000).unref()
 
-  cluster.worker?.disconnect()
-
   closeServer(20000, () ->
     db.disconnect()
     cache.disconnect()
     extensions.killAll()
-    logger.close()
+    cluster.worker?.disconnect()
   )