From ed0aa8ae7a115a32b9c189cbfb0c3150c1d255fe Mon Sep 17 00:00:00 2001 From: Zaid Abdulla Date: Sun, 22 Sep 2019 02:06:58 -0700 Subject: [PATCH] Fix process shutdown issues --- bin/genieacs-cwmp | 11 +++++++---- bin/genieacs-fs | 9 ++++++--- bin/genieacs-nbi | 11 +++++++---- bin/genieacs-ui | 13 ++++++++----- docs/installation-guide.rst | 4 ---- lib/cache.ts | 8 ++++---- lib/cluster.ts | 29 +++++++++++++++++++++-------- lib/db.ts | 6 ++++-- lib/ui/db.ts | 16 ++++++---------- 9 files changed, 63 insertions(+), 44 deletions(-) diff --git a/bin/genieacs-cwmp b/bin/genieacs-cwmp index 458f674..1d7d881 100755 --- a/bin/genieacs-cwmp +++ b/bin/genieacs-cwmp @@ -35,10 +35,13 @@ const SERVICE_ADDRESS = config.get("CWMP_INTERFACE"); const SERVICE_PORT = config.get("CWMP_PORT"); function exitWorkerGracefully(): void { - db.disconnect(); - cache.disconnect(); - extensions.killAll(); - cluster.worker.disconnect(); + setTimeout(exitWorkerUngracefully, 5000).unref(); + Promise.all([ + db.disconnect(), + cache.disconnect(), + extensions.killAll(), + cluster.worker.disconnect() + ]).catch(exitWorkerUngracefully); } function exitWorkerUngracefully(): void { diff --git a/bin/genieacs-fs b/bin/genieacs-fs index b84b583..affbd3d 100755 --- a/bin/genieacs-fs +++ b/bin/genieacs-fs @@ -34,9 +34,12 @@ const SERVICE_ADDRESS = config.get("FS_INTERFACE"); const SERVICE_PORT = config.get("FS_PORT"); function exitWorkerGracefully(): void { - db.disconnect(); - cache.disconnect(); - cluster.worker.disconnect(); + setTimeout(exitWorkerUngracefully, 5000).unref(); + Promise.all([ + db.disconnect(), + cache.disconnect(), + cluster.worker.disconnect() + ]).catch(exitWorkerUngracefully); } function exitWorkerUngracefully(): void { diff --git a/bin/genieacs-nbi b/bin/genieacs-nbi index ed9b37a..512d3cc 100755 --- a/bin/genieacs-nbi +++ b/bin/genieacs-nbi @@ -35,10 +35,13 @@ const SERVICE_ADDRESS = config.get("NBI_INTERFACE"); const SERVICE_PORT = config.get("NBI_PORT"); function exitWorkerGracefully(): void { - db.disconnect(); - cache.disconnect(); - extensions.killAll(); - cluster.worker.disconnect(); + setTimeout(exitWorkerUngracefully, 5000).unref(); + Promise.all([ + db.disconnect(), + cache.disconnect(), + extensions.killAll(), + cluster.worker.disconnect() + ]).catch(exitWorkerUngracefully); } function exitWorkerUngracefully(): void { diff --git a/bin/genieacs-ui b/bin/genieacs-ui index ec933ac..e7d2506 100755 --- a/bin/genieacs-ui +++ b/bin/genieacs-ui @@ -36,11 +36,14 @@ const SERVICE_ADDRESS = config.get("UI_INTERFACE"); const SERVICE_PORT = config.get("UI_PORT"); function exitWorkerGracefully(): void { - db.disconnect(); - db2.disconnect(); - cache.disconnect(); - extensions.killAll(); - cluster.worker.disconnect(); + setTimeout(exitWorkerUngracefully, 5000).unref(); + Promise.all([ + db.disconnect(), + db2.disconnect(), + cache.disconnect(), + extensions.killAll(), + cluster.worker.disconnect() + ]).catch(exitWorkerUngracefully); } function exitWorkerUngracefully(): void { diff --git a/docs/installation-guide.rst b/docs/installation-guide.rst index 9a09fca..810ab8a 100644 --- a/docs/installation-guide.rst +++ b/docs/installation-guide.rst @@ -119,7 +119,6 @@ Configure systemd User=genieacs EnvironmentFile=/opt/genieacs/genieacs.env ExecStart=/usr/bin/genieacs-cwmp - KillMode=process [Install] WantedBy=default.target @@ -142,7 +141,6 @@ Configure systemd User=genieacs EnvironmentFile=/opt/genieacs/genieacs.env ExecStart=/usr/bin/genieacs-nbi - KillMode=process [Install] WantedBy=default.target @@ -165,7 +163,6 @@ Configure systemd User=genieacs EnvironmentFile=/opt/genieacs/genieacs.env ExecStart=/usr/bin/genieacs-fs - KillMode=process [Install] WantedBy=default.target @@ -188,7 +185,6 @@ Configure systemd User=genieacs EnvironmentFile=/opt/genieacs/genieacs.env ExecStart=/usr/bin/genieacs-ui - KillMode=process [Install] WantedBy=default.target diff --git a/lib/cache.ts b/lib/cache.ts index f7d25f1..1a01b71 100644 --- a/lib/cache.ts +++ b/lib/cache.ts @@ -22,16 +22,16 @@ import * as config from "./config"; const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL"); -let mongoClient: MongoClient; +let clientPromise: Promise; let mongoCollection: Collection; let mongoTimeOffset = 0; export async function connect(): Promise { const MONGODB_CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL"); - mongoClient = await MongoClient.connect(MONGODB_CONNECTION_URL, { + clientPromise = MongoClient.connect(MONGODB_CONNECTION_URL, { useNewUrlParser: true }); - const db = mongoClient.db(); + const db = (await clientPromise).db(); mongoCollection = db.collection("cache"); await mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }); const now = Date.now(); @@ -40,7 +40,7 @@ export async function connect(): Promise { } export async function disconnect(): Promise { - if (mongoClient) await mongoClient.close(); + if (clientPromise) await (await clientPromise).close(); } export async function get(key): Promise { diff --git a/lib/cluster.ts b/lib/cluster.ts index b223ec4..12f0559 100644 --- a/lib/cluster.ts +++ b/lib/cluster.ts @@ -22,7 +22,7 @@ import { cpus } from "os"; import * as logger from "./logger"; let respawnTimestamp = 0; -let tooManyCrashesTimestamp = 0; +let crashes: number[] = []; function fork(): cluster.Worker { const w = cluster.fork(); @@ -51,14 +51,21 @@ function restartWorker(worker, code, signal): void { logger.error(msg); const now = Date.now(); - respawnTimestamp = Math.max(now, respawnTimestamp + 2000); - if (respawnTimestamp === now) { - tooManyCrashesTimestamp = now; - fork(); - return; - } + crashes.push(now); - if (now - tooManyCrashesTimestamp > 60000) { + let min1 = 0, + min2 = 0, + min3 = 0; + + crashes = crashes.filter(n => { + if (n > now - 60000) ++min1; + else if (n > now - 120000) ++min2; + else if (n > now - 180000) ++min3; + else return false; + return true; + }); + + if (min1 > 5 && min2 > 5 && min3 > 5) { process.exitCode = 1; cluster.removeListener("exit", restartWorker); for (const pid in cluster.workers) cluster.workers[pid].kill(); @@ -70,6 +77,12 @@ function restartWorker(worker, code, signal): void { return; } + respawnTimestamp = Math.max(now, respawnTimestamp + 2000); + if (respawnTimestamp === now) { + fork(); + return; + } + setTimeout(() => { if (process.exitCode) return; fork(); diff --git a/lib/db.ts b/lib/db.ts index 2324990..b50062d 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -44,13 +44,15 @@ export let tasksCollection: Collection, usersCollection: Collection, configCollection: Collection; +let clientPromise: Promise; export let client: MongoClient; export async function connect(): Promise { - client = await MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), { + clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), { useNewUrlParser: true }); + client = await clientPromise; const db = client.db(); tasksCollection = db.collection("tasks"); @@ -70,7 +72,7 @@ export async function connect(): Promise { } export async function disconnect(): Promise { - if (client) await client.close(); + if (clientPromise) await (await clientPromise).close(); } // Optimize projection by removing overlaps diff --git a/lib/ui/db.ts b/lib/ui/db.ts index c478b11..57e89b4 100644 --- a/lib/ui/db.ts +++ b/lib/ui/db.ts @@ -26,7 +26,7 @@ import { QueryOptions } from "../types"; const CACHE_TTL = 300000; -let _clientPromise = null; +let clientPromise: Promise; const RESOURCE_COLLECTION = { files: "fs.files" @@ -40,8 +40,8 @@ function ensureIndexes(client): void { } function getClient(): Promise { - if (!_clientPromise) { - _clientPromise = new Promise((resolve, reject) => { + if (!clientPromise) { + clientPromise = new Promise((resolve, reject) => { const CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL"); MongoClient.connect( CONNECTION_URL, @@ -55,7 +55,7 @@ function getClient(): Promise { }); } - return _clientPromise; + return clientPromise; } export function cache(key, valueGetter: () => Promise, ttl): Promise { @@ -405,10 +405,6 @@ export function deleteTask(id): Promise { return deleteResource("tasks", id); } -export function disconnect(): void { - if (_clientPromise) { - _clientPromise.then(client => { - client.close(); - }); - } +export async function disconnect(): Promise { + if (clientPromise) await (await clientPromise).close(); }