From 1c9aa493c61252e522e47f65bfdc409a377a1b59 Mon Sep 17 00:00:00 2001
From: Zaid Abdulla <zaid@genieacs.com>
Date: Mon, 9 Sep 2019 02:14:12 -0700
Subject: [PATCH] Fix occasional exceptions on SIGINT and SIGTERM

---
 bin/genieacs-cwmp | 34 +++++++++++++++++++---------------
 bin/genieacs-fs   | 34 +++++++++++++++++++---------------
 bin/genieacs-nbi  | 34 +++++++++++++++++++---------------
 bin/genieacs-ui   | 38 +++++++++++++++++++++-----------------
 lib/cluster.ts    | 18 +++++++++++++++---
 lib/server.ts     |  3 ++-
 6 files changed, 95 insertions(+), 66 deletions(-)

diff --git a/bin/genieacs-cwmp b/bin/genieacs-cwmp
index 7e75c1c..458f674 100755
--- a/bin/genieacs-cwmp
+++ b/bin/genieacs-cwmp
@@ -94,21 +94,7 @@ if (!cluster.worker) {
       .catch(exitWorkerUngracefully);
   });
 
-  process.on("SIGINT", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
+  const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
       server.start(
         SERVICE_PORT,
@@ -124,4 +110,22 @@ if (!cluster.worker) {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
 }
diff --git a/bin/genieacs-fs b/bin/genieacs-fs
index 4612894..b84b583 100755
--- a/bin/genieacs-fs
+++ b/bin/genieacs-fs
@@ -90,21 +90,7 @@ if (!cluster.worker) {
       .catch(exitWorkerUngracefully);
   });
 
-  process.on("SIGINT", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
+  const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
       server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
     })
@@ -113,4 +99,22 @@ if (!cluster.worker) {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
 }
diff --git a/bin/genieacs-nbi b/bin/genieacs-nbi
index 6c740ce..ed9b37a 100755
--- a/bin/genieacs-nbi
+++ b/bin/genieacs-nbi
@@ -94,21 +94,7 @@ if (!cluster.worker) {
       .catch(exitWorkerUngracefully);
   });
 
-  process.on("SIGINT", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
+  const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
       server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
     })
@@ -117,4 +103,22 @@ if (!cluster.worker) {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
 }
diff --git a/bin/genieacs-ui b/bin/genieacs-ui
index 93fe15c..ec933ac 100755
--- a/bin/genieacs-ui
+++ b/bin/genieacs-ui
@@ -99,28 +99,12 @@ if (!cluster.worker) {
       .catch(exitWorkerUngracefully);
   });
 
-  process.on("SIGINT", () => {
-    stopping = true;
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    stopping = true;
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
   const _listener = (req, res): void => {
     if (stopping) res.setHeader("Connection", "close");
     listener(req, res);
   };
 
-  Promise.all([db2.connect(), cache.connect()])
+  const initPromise = Promise.all([db2.connect(), cache.connect()])
     .then(() => {
       server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
     })
@@ -129,4 +113,24 @@ if (!cluster.worker) {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
 }
diff --git a/lib/cluster.ts b/lib/cluster.ts
index 6cd96fd..b223ec4 100644
--- a/lib/cluster.ts
+++ b/lib/cluster.ts
@@ -24,6 +24,18 @@ import * as logger from "./logger";
 let respawnTimestamp = 0;
 let tooManyCrashesTimestamp = 0;
 
+function fork(): cluster.Worker {
+  const w = cluster.fork();
+  w.on("error", (err: NodeJS.ErrnoException) => {
+    // Avoid exception when attempting to kill the process just as it's exiting
+    if (err.code !== "EPIPE") throw err;
+    setTimeout(() => {
+      if (!w.isDead()) throw err;
+    }, 50);
+  });
+  return w;
+}
+
 function restartWorker(worker, code, signal): void {
   const msg = {
     message: "Worker died",
@@ -42,7 +54,7 @@ function restartWorker(worker, code, signal): void {
   respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
   if (respawnTimestamp === now) {
     tooManyCrashesTimestamp = now;
-    cluster.fork();
+    fork();
     return;
   }
 
@@ -60,7 +72,7 @@ function restartWorker(worker, code, signal): void {
 
   setTimeout(() => {
     if (process.exitCode) return;
-    cluster.fork();
+    fork();
   }, respawnTimestamp - now);
 }
 
@@ -84,7 +96,7 @@ export function start(workerCount, servicePort, serviceAddress): void {
 
   if (!workerCount) workerCount = Math.max(2, cpus().length);
 
-  for (let i = 0; i < workerCount; ++i) cluster.fork();
+  for (let i = 0; i < workerCount; ++i) fork();
 }
 
 export function stop(): void {
diff --git a/lib/server.ts b/lib/server.ts
index c2bda37..d5aae82 100644
--- a/lib/server.ts
+++ b/lib/server.ts
@@ -46,7 +46,8 @@ function closeServer(timeout, callback): void {
 
     const cb = callback;
     callback = null;
-    cb();
+    // Allow some time for connection close events to fire
+    setTimeout(cb, 50);
   });
 }