Fix process shutdown issues

lib/cluster.ts

@@ -22,7 +22,7 @@ import { cpus } from "os";
 import * as logger from "./logger";
 
 let respawnTimestamp = 0;
-let tooManyCrashesTimestamp = 0;
+let crashes: number[] = [];
 
 function fork(): cluster.Worker {
   const w = cluster.fork();
@@ -51,14 +51,21 @@ function restartWorker(worker, code, signal): void {
   logger.error(msg);
 
   const now = Date.now();
-  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
-  if (respawnTimestamp === now) {
-    tooManyCrashesTimestamp = now;
-    fork();
-    return;
-  }
+  crashes.push(now);
 
-  if (now - tooManyCrashesTimestamp > 60000) {
+  let min1 = 0,
+    min2 = 0,
+    min3 = 0;
+
+  crashes = crashes.filter(n => {
+    if (n > now - 60000) ++min1;
+    else if (n > now - 120000) ++min2;
+    else if (n > now - 180000) ++min3;
+    else return false;
+    return true;
+  });
+
+  if (min1 > 5 && min2 > 5 && min3 > 5) {
     process.exitCode = 1;
     cluster.removeListener("exit", restartWorker);
     for (const pid in cluster.workers) cluster.workers[pid].kill();
@@ -70,6 +77,12 @@ function restartWorker(worker, code, signal): void {
     return;
   }
 
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    fork();
+    return;
+  }
+
   setTimeout(() => {
     if (process.exitCode) return;
     fork();