Fix process shutdown issues

bin/genieacs-cwmp

@@ -35,10 +35,13 @@ const SERVICE_ADDRESS = config.get("CWMP_INTERFACE");
 const SERVICE_PORT = config.get("CWMP_PORT");
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
-  cache.disconnect();
-  extensions.killAll();
-  cluster.worker.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {

bin/genieacs-fs

@@ -34,9 +34,12 @@ const SERVICE_ADDRESS = config.get("FS_INTERFACE");
 const SERVICE_PORT = config.get("FS_PORT");
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
-  cache.disconnect();
-  cluster.worker.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    cache.disconnect(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {

bin/genieacs-nbi

@@ -35,10 +35,13 @@ const SERVICE_ADDRESS = config.get("NBI_INTERFACE");
 const SERVICE_PORT = config.get("NBI_PORT");
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
-  cache.disconnect();
-  extensions.killAll();
-  cluster.worker.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {

bin/genieacs-ui

@@ -36,11 +36,14 @@ const SERVICE_ADDRESS = config.get("UI_INTERFACE");
 const SERVICE_PORT = config.get("UI_PORT");
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
-  db2.disconnect();
-  cache.disconnect();
-  extensions.killAll();
-  cluster.worker.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    db2.disconnect(),
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {

docs/installation-guide.rst

@@ -119,7 +119,6 @@ Configure systemd
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
       ExecStart=/usr/bin/genieacs-cwmp
-      KillMode=process
 
       [Install]
       WantedBy=default.target
@@ -142,7 +141,6 @@ Configure systemd
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
       ExecStart=/usr/bin/genieacs-nbi
-      KillMode=process
 
       [Install]
       WantedBy=default.target
@@ -165,7 +163,6 @@ Configure systemd
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
       ExecStart=/usr/bin/genieacs-fs
-      KillMode=process
 
       [Install]
       WantedBy=default.target
@@ -188,7 +185,6 @@ Configure systemd
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
       ExecStart=/usr/bin/genieacs-ui
-      KillMode=process
 
       [Install]
       WantedBy=default.target

lib/cache.ts

@@ -22,16 +22,16 @@ import * as config from "./config";
 
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let mongoClient: MongoClient;
+let clientPromise: Promise<MongoClient>;
 let mongoCollection: Collection;
 let mongoTimeOffset = 0;
 
 export async function connect(): Promise<void> {
   const MONGODB_CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
-  mongoClient = await MongoClient.connect(MONGODB_CONNECTION_URL, {
+  clientPromise = MongoClient.connect(MONGODB_CONNECTION_URL, {
     useNewUrlParser: true
   });
-  const db = mongoClient.db();
+  const db = (await clientPromise).db();
   mongoCollection = db.collection("cache");
   await mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
   const now = Date.now();
@@ -40,7 +40,7 @@ export async function connect(): Promise<void> {
 }
 
 export async function disconnect(): Promise<void> {
-  if (mongoClient) await mongoClient.close();
+  if (clientPromise) await (await clientPromise).close();
 }
 
 export async function get(key): Promise<any> {

lib/cluster.ts

@@ -22,7 +22,7 @@ import { cpus } from "os";
 import * as logger from "./logger";
 
 let respawnTimestamp = 0;
-let tooManyCrashesTimestamp = 0;
+let crashes: number[] = [];
 
 function fork(): cluster.Worker {
   const w = cluster.fork();
@@ -51,14 +51,21 @@ function restartWorker(worker, code, signal): void {
   logger.error(msg);
 
   const now = Date.now();
-  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
-  if (respawnTimestamp === now) {
-    tooManyCrashesTimestamp = now;
-    fork();
-    return;
-  }
+  crashes.push(now);
 
-  if (now - tooManyCrashesTimestamp > 60000) {
+  let min1 = 0,
+    min2 = 0,
+    min3 = 0;
+
+  crashes = crashes.filter(n => {
+    if (n > now - 60000) ++min1;
+    else if (n > now - 120000) ++min2;
+    else if (n > now - 180000) ++min3;
+    else return false;
+    return true;
+  });
+
+  if (min1 > 5 && min2 > 5 && min3 > 5) {
     process.exitCode = 1;
     cluster.removeListener("exit", restartWorker);
     for (const pid in cluster.workers) cluster.workers[pid].kill();
@@ -70,6 +77,12 @@ function restartWorker(worker, code, signal): void {
     return;
   }
 
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    fork();
+    return;
+  }
+
   setTimeout(() => {
     if (process.exitCode) return;
     fork();

lib/db.ts

@@ -44,13 +44,15 @@ export let tasksCollection: Collection,
   usersCollection: Collection,
   configCollection: Collection;
 
+let clientPromise: Promise<MongoClient>;
 export let client: MongoClient;
 
 export async function connect(): Promise<void> {
-  client = await MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
     useNewUrlParser: true
   });
 
+  client = await clientPromise;
   const db = client.db();
 
   tasksCollection = db.collection("tasks");
@@ -70,7 +72,7 @@ export async function connect(): Promise<void> {
 }
 
 export async function disconnect(): Promise<void> {
-  if (client) await client.close();
+  if (clientPromise) await (await clientPromise).close();
 }
 
 // Optimize projection by removing overlaps

lib/ui/db.ts

@@ -26,7 +26,7 @@ import { QueryOptions } from "../types";
 
 const CACHE_TTL = 300000;
 
-let _clientPromise = null;
+let clientPromise: Promise<MongoClient>;
 
 const RESOURCE_COLLECTION = {
   files: "fs.files"
@@ -40,8 +40,8 @@ function ensureIndexes(client): void {
 }
 
 function getClient(): Promise<MongoClient> {
-  if (!_clientPromise) {
-    _clientPromise = new Promise((resolve, reject) => {
+  if (!clientPromise) {
+    clientPromise = new Promise((resolve, reject) => {
       const CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
       MongoClient.connect(
         CONNECTION_URL,
@@ -55,7 +55,7 @@ function getClient(): Promise<MongoClient> {
     });
   }
 
-  return _clientPromise;
+  return clientPromise;
 }
 
 export function cache<T>(key, valueGetter: () => Promise<T>, ttl): Promise<T> {
@@ -405,10 +405,6 @@ export function deleteTask(id): Promise<void> {
   return deleteResource("tasks", id);
 }
 
-export function disconnect(): void {
-  if (_clientPromise) {
-    _clientPromise.then(client => {
-      client.close();
-    });
-  }
+export async function disconnect(): Promise<void> {
+  if (clientPromise) await (await clientPromise).close();
 }