Cache files to improve genieacs-fs performance

This also fixes a memory leak issue when the client disconnects before
completing the download. I suspect the leak had something to do with
using pipe() but I'm not sure if it's a bug in MongoDB driver or that
we were lacking proper error handling. Anyway, the pipe is no longer
leaking :)

bin/genieacs-fs.ts

@@ -94,7 +94,7 @@ if (!cluster.worker) {
 
   const _listener = (req, res): void => {
     if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
+    void listener(req, res);
   };
 
   const initPromise = db

lib/common/memoize.ts

@@ -47,9 +47,18 @@ export default function memoize<T extends (...args: any[]) => any>(func: T): T {
     if (cache1.has(key)) return cache1.get(key);
 
     let r;
-    if (cache2.has(key)) r = cache2.get(key);
-    else r = func(...args);
-    cache1.set(key, r);
+    if (cache2.has(key)) {
+      cache1.set(key, (r = cache2.get(key)));
+    } else {
+      cache1.set(key, (r = func(...args)));
+      // Evict rejected promises
+      if (r instanceof Promise) {
+        r.catch(() => {
+          cache1.delete(key);
+          cache2.delete(key);
+        });
+      }
+    }
     return r;
   }) as any;
 }

lib/fs.ts

@@ -24,6 +24,7 @@ import { Collection, GridFSBucket } from "mongodb";
 import { onConnect } from "./db";
 import * as logger from "./logger";
 import { getRequestOrigin } from "./forwarded";
+import memoize from "./common/memoize";
 
 let filesCollection: Collection;
 let filesBucket: GridFSBucket;
@@ -33,12 +34,36 @@ onConnect(async (db) => {
   filesBucket = new GridFSBucket(db);
 });
 
-export function listener(
+const getFile = memoize(
+  (md5: string, size: number, filename: string): Promise<Buffer> => {
+    return new Promise((resolve, reject) => {
+      const buffer = Buffer.allocUnsafe(size);
+      let i = 0;
+      const downloadStream = filesBucket.openDownloadStreamByName(filename);
+      downloadStream.on("error", reject);
+      downloadStream.on("data", (data: Buffer) => {
+        data.copy(buffer, i);
+        i += data.length;
+      });
+      downloadStream.on("end", () => {
+        if (i !== size) reject(new Error("File size mismatch"));
+        else resolve(buffer);
+      });
+    });
+  }
+);
+
+export async function listener(
   request: IncomingMessage,
   response: ServerResponse
-): void {
+): Promise<void> {
+  if (request.method !== "GET") {
+    response.writeHead(405, { Allow: "GET" });
+    response.end("405 Method Not Allowed");
+    return;
+  }
+
   const urlParts = url.parse(request.url, true);
-  if (request.method === "GET") {
   const filename = querystring.unescape(urlParts.pathname.substring(1));
 
   const log = {
@@ -47,8 +72,7 @@ export function listener(
     remoteAddress: getRequestOrigin(request).remoteAddress,
   };
 
-    filesCollection.findOne({ _id: filename }, (err, file) => {
-      if (err) throw err;
+  const file = await filesCollection.findOne({ _id: filename });
 
   if (!file) {
     response.writeHead(404);
@@ -58,18 +82,13 @@ export function listener(
     return;
   }
 
+  const buffer = await getFile(file.md5, file.length, filename);
+
   response.writeHead(200, {
     "Content-Type": file.contentType || "application/octet-stream",
     "Content-Length": file.length,
   });
 
-      const downloadStream = filesBucket.openDownloadStreamByName(filename);
-      downloadStream.pipe(response);
-
+  response.end(buffer);
   logger.accessInfo(log);
-    });
-  } else {
-    response.writeHead(405, { Allow: "GET" });
-    response.end("405 Method Not Allowed");
-  }
 }