1.2.1

Fixing this vulnerability https://npmjs.com/advisories/1555

.eslintrc.json

@@ -21,6 +21,7 @@
     "@typescript-eslint/no-explicit-any": 0,
     "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
     "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
+    "@typescript-eslint/no-floating-promises": "error",
     "require-atomic-updates": 0,
     "no-prototype-builtins": 0
   }

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Change Log
 
+## 1.2.1 (2020-09-08)
+- Fix bug causing faults to not be displayed in the UI.
+- Fix bug where deleting objects does not get reflected immediately in the UI.
+- Improve conversion between filters written in the expression format and
+MongoDB queries. There should now be fewer edge cases where the two are not
+equisatisfiable.
+
 ## 1.2.0 (2020-09-01)
 
 - Support GetParameterAttributes and SetParameterAttributes TR-069 methods.

bin/genieacs-cwmp.ts

@@ -43,7 +43,7 @@ function exitWorkerGracefully(): void {
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }

bin/genieacs-nbi.ts

@@ -43,7 +43,7 @@ function exitWorkerGracefully(): void {
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }

bin/genieacs-ui.ts

@@ -45,7 +45,7 @@ function exitWorkerGracefully(): void {
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -99,7 +99,9 @@ if (!cluster.worker) {
 
   const _listener = (req, res): void => {
     if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
+    listener(req, res).catch((err) => {
+      throw err;
+    });
   };
 
   const initPromise = Promise.all([db2.connect(), cache.connect()])

build/build.ts

@@ -81,7 +81,7 @@ const externals = [
   "codemirror/mode/yaml/yaml",
   "ipaddr.js",
   "jsbi",
-  "yebool",
+  "espresso-iisojs",
 ];
 
 function rmDirSync(dirPath): void {
@@ -297,7 +297,7 @@ async function generateFrontendJs(): Promise<void> {
   const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
   const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
 
-  const inlineDeps = ["mithril", "parsimmon", "jsbi", "yebool"];
+  const inlineDeps = ["mithril", "parsimmon", "jsbi", "espresso-iisojs"];
   const bundle = await rollup({
     input: inputFile,
     external: externals.filter((e) => !inlineDeps.includes(e)),
@@ -306,10 +306,14 @@ async function generateFrontendJs(): Promise<void> {
       {
         name: "",
         resolveId: function (importee, importer) {
-          if (importee.endsWith("/bigint"))
+          if (importee.endsWith("/bigint")) {
             return this.resolve(importee + "-jsbi", importer);
-          else if (importee === "yebool")
-            return this.resolve("yebool/dist/yebool-jsbi.mjs", importer);
+          } else if (importee === "espresso-iisojs") {
+            return this.resolve(
+              "espresso-iisojs/dist/espresso-iisojs-jsbi.mjs",
+              importer
+            );
+          }
           return null;
         },
       },

lib/common/boolean-expression.ts

@@ -18,7 +18,7 @@
  */
 
 import * as BI from "./bigint";
-import { espresso, complement, tautology } from "yebool";
+import { espresso, complement, tautology } from "espresso-iisojs";
 import { Expression } from "../types";
 import { map } from "./expression-parser";
 import { and, evaluateCallback } from "./expression";

lib/extensions.ts

@@ -58,6 +58,7 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
             });
           }
 
+          // eslint-disable-next-line @typescript-eslint/no-floating-promises
           kill(processes[scriptName]);
           delete processes[scriptName];
         }

lib/local-cache.ts

@@ -22,7 +22,7 @@ import * as crypto from "crypto";
 import * as config from "./config";
 import * as db from "./db";
 import * as cache from "./cache";
-import { mongoQueryToFilter } from "./mongodb-functions";
+import { convertOldPrecondition } from "./mongodb-functions";
 import * as logger from "./logger";
 import * as scheduling from "./scheduling";
 import * as expression from "./common/expression";
@@ -173,7 +173,9 @@ async function fetchPresets(): Promise<Preset[]> {
       try {
         precondition = parse(preset["precondition"]);
       } catch (error) {
-        precondition = mongoQueryToFilter(JSON.parse(preset["precondition"]));
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"])
+        );
       }
 
       // Simplify expression

lib/mongodb-functions.ts

@@ -19,7 +19,7 @@
 
 import { ObjectID } from "mongodb";
 import { map, parse, stringify, parseList } from "./common/expression-parser";
-import { likePatternToRegExp, evaluate } from "./common/expression";
+import { likePatternToRegExp } from "./common/expression";
 import { Expression, Fault, Task } from "./types";
 
 const isArray = Array.isArray;
@@ -44,12 +44,10 @@ export function processDeviceFilter(filter: Expression): Expression {
       exp[1][1].startsWith("Tags.")
     ) {
       const t = exp[1][1].slice(5);
-      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
-      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
-      else if (exp[0] === "=" && exp[2] === true)
-        return ["=", ["PARAM", "_tags"], t];
-      else if (exp[0] === "<>" && exp[2] !== true)
-        return ["=", ["PARAM", "_tags"], t];
+      if (exp[0] === "IS NULL") return ["_tags", t, false];
+      else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
+      else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
+      else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
     } else if (
       [
         "=",
@@ -117,140 +115,131 @@ export function processFaultsFilter(filter: Expression): Expression {
 }
 
 export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
-  const ops = {
-    OR: 0,
-    AND: 0,
-    NOT: 0,
-    "=": 1,
-    "<>": 1,
-    ">": 1,
-    ">=": 1,
-    "<": 1,
-    "<=": 1,
-    LIKE: 1,
-    "NOT LIKE": 1,
-    "IS NULL": 1,
-    "IS NOT NULL": 1,
-  };
-
-  function recursive(filter, negate, res = {}): Record<string, unknown> {
+  function recursive(filter, negate): Record<string, unknown> {
     const op = filter[0];
 
-    if (ops[op] === 0) {
-      if (op === "AND") filter = filter.filter((f) => f !== true);
-      if (op === "OR") filter = filter.filter((f) => f !== false);
-      if (filter.length === 2) return recursive(filter[1], negate, res);
-
-      for (let i = 1; i < filter.length; ++i) {
-        if (!isArray(filter[i]) || ops[filter[i][0]] == null)
-          throw new Error(`Invalid expression in ${op} clause`);
-      }
-
-      if ((!negate && op === "AND") || (negate && op === "OR")) {
-        res["$and"] = res["$and"] || [];
-        for (let i = 1; i < filter.length; ++i)
-          res["$and"].push(recursive(filter[i], negate));
-      } else if ((!negate && op === "OR") || (negate && op === "AND")) {
-        res["$or"] = res["$or"] || [];
-
-        for (let i = 1; i < filter.length; ++i)
-          res["$or"].push(recursive(filter[i], negate));
-      } else if (op === "NOT") {
-        recursive(filter[1], !negate, res);
-      }
-    } else if (ops[op] === 1) {
-      if (isArray(filter[2])) {
-        throw new Error(`Right operand of ${op} clause is not a primitive`);
-      } else if (
-        ["LIKE", "NOT LIKE"].includes(op) &&
-        isArray(filter[1]) &&
-        filter[1][0] === "FUNC" &&
-        ["UPPER", "LOWER"].includes(filter[1][1])
-      ) {
-        if (
-          (filter[1][1] === "UPPER" && filter[2] !== filter[2].toUpperCase()) ||
-          (filter[1][1] === "LOWER" && filter[2] !== filter[2].toLowerCase())
-        ) {
-          throw new Error(
-            `Cannot compare ${
-              filter[1][1]
-            }() against non-${filter[1][1].toLowerCase()}case pattern`
-          );
-        }
-      } else if (!isArray(filter[1]) || filter[1][0] !== "PARAM") {
-        throw new Error(`Left operand of ${op} clause is not a parameter`);
-      }
-
-      if (op === "=") {
-        const param = filter[1][1];
-        const p = (res[param] = res[param] || {});
-        if (negate) p["$ne"] = filter[2];
-        else p["$eq"] = filter[2];
-      } else if (op === "<>") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$ne"] = filter[2];
-        if (param !== "_tags") p["$exists"] = true;
-      } else if (op === ">") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gt"] = filter[2];
-      } else if (op === ">=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gte"] = filter[2];
-      } else if (op === "<") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lt"] = filter[2];
-      } else if (op === "<=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lte"] = filter[2];
-      } else if (op === "IS NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: negate };
-      } else if (op === "IS NOT NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: !negate };
-      } else {
-        if (op === "NOT LIKE") negate = !negate;
-        let param;
-        let flags;
-        if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
-          param = filter[1][2][1];
-          flags = "i";
-        } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
-          param = filter[1][2][1];
-          flags = "i";
-        } else {
-          param = filter[1][1];
-          flags = "";
-        }
-        const r = likePatternToRegExp(filter[2], filter[3], flags);
-        if (negate) res[param] = { $not: r };
-        else res[param] = r;
-      }
-    } else {
-      throw new Error(`Unrecognized operator ${op}`);
+    if (op === "AND") {
+      filter = filter.filter((f) => f !== true);
+      if (filter.length === 2) return recursive(filter[1], negate);
     }
 
-    return res;
+    if (op === "OR") {
+      filter = filter.filter((f) => f !== false);
+      if (filter.length === 2) return recursive(filter[1], negate);
+    }
+
+    if ((!negate && op === "AND") || (negate && op === "OR"))
+      return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if ((!negate && op === "OR") || (negate && op === "AND"))
+      return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if (op === "NOT") return recursive(filter[1], !negate);
+
+    if (isArray(filter[2]))
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "LIKE" || op === "NOT LIKE") {
+      if (op === "NOT LIKE") negate = !negate;
+      let param;
+      let flags;
+      if (!Array.isArray(filter[1]))
+        throw new Error(`Invalid LHS operand of ${op} clause`);
+      if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
+        if (filter[2] !== filter[2].toUpperCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
+        param = filter[1][2][1];
+        flags = "i";
+      } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
+        if (filter[2] !== filter[2].toLowerCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
+        param = filter[1][2][1];
+        flags = "i";
+      } else if (filter[1][0] === "PARAM") {
+        param = filter[1][1];
+        flags = "";
+      } else {
+        throw new Error(`Invalid LHS operand of ${op} clause`);
+      }
+      const r = likePatternToRegExp(filter[2], filter[3], flags);
+      if (negate) return { [param]: { $nin: [r, null] } };
+      else return { [param]: r };
+    }
+
+    if (op === "_tags") {
+      let t = filter[2];
+      if (negate) t = !t;
+      if (t) return { _tags: filter[1] };
+      else return { _tags: { $ne: filter[1] } };
+    }
+
+    if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
+      throw new Error(`Invalid LHS operand of ${op} clause`);
+
+    if (op === "IS NULL") {
+      const p = filter[1][1];
+      return { [p]: null };
+    }
+
+    if (op === "IS NOT NULL") {
+      const p = filter[1][1];
+      return { [p]: { $ne: null } };
+    }
+
+    if (isArray(filter[2]) || filter[2] == null)
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (!negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === "<>") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === ">") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lte: v } };
+      else return { [p]: { $gt: v } };
+    }
+
+    if (op === ">=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lt: v } };
+      else return { [p]: { $gte: v } };
+    }
+
+    if (op === "<") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gte: v } };
+      else return { [p]: { $lt: v } };
+    }
+
+    if (op === "<=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gt: v } };
+      else return { [p]: { $lte: v } };
+    }
+
+    throw new Error(`Unrecognized operator ${op}`);
   }
 
-  const _exp = evaluate(exp, null, Date.now());
-
-  if (!isArray(_exp)) {
-    if (_exp === true) return {};
+  if (!isArray(exp)) {
+    if (exp === true) return {};
     throw new Error("Primitives are not valid queries");
   }
 
-  return recursive(_exp, false);
+  return recursive(exp, false);
 }
 
 export function processDeviceProjection(
@@ -495,7 +484,9 @@ export function flattenTask(task: unknown): Task {
   return t;
 }
 
-export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
+export function convertOldPrecondition(
+  query: Record<string, unknown>
+): Expression {
   function recursive(_query): Expression {
     const expressions: Expression[] = [];
     for (const [k, v] of Object.entries(_query)) {
@@ -514,12 +505,22 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
       } else if (k === "_tags") {
         if (typeof v === "object") {
           if (isArray(v)) throw new Error(`Invalid type`);
-
-          if (v.hasOwnProperty("$ne"))
-            expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]);
-          else if (v.hasOwnProperty("$eq"))
-            expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v["$eq"]}`]]);
-          else throw new Error(`Invalid tag query`);
+          const conjs: Expression[] = [];
+          for (const [op, val] of Object.entries(v)) {
+            if (op === "$ne") {
+              if (typeof v["$ne"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NULL", ["PARAM", `Tags.${val}`]]);
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${val}`]]);
+            } else {
+              throw new Error(`Invalid tag query`);
+            }
+          }
+          if (conjs.length === 1) expressions.push(conjs[0]);
+          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
         } else {
           expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
         }
@@ -537,34 +538,31 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
 
         const exps: Expression[] = [];
         for (const [kk, vv] of Object.entries(v)) {
-          let op;
-          switch (kk) {
-            case "$eq":
-              op = "=";
-              break;
-            case "$ne":
-              op = "<>";
-              break;
-            case "$lt":
-              op = "<";
-              break;
-            case "$lte":
-              op = "<=";
-              break;
-            case "$gt":
-              op = ">";
-              break;
-            case "$gte":
-              op = ">=";
-              break;
-            default:
-              throw new Error(`Operator ${kk} not supported`);
+          if (kk === "$eq") {
+            exps.push(["=", ["PARAM", k], vv]);
+          } else if (kk === "$ne") {
+            exps.push([
+              "OR",
+              ["<>", ["PARAM", k], vv],
+              ["IS NULL", ["PARAM", k]],
+            ]);
+          } else if (kk === "$lt") {
+            exps.push(["<", ["PARAM", k], vv]);
+          } else if (kk === "$lte") {
+            exps.push(["<=", ["PARAM", k], vv]);
+          } else if (kk === "$gt") {
+            exps.push([">", ["PARAM", k], vv]);
+          } else if (kk === "$gte") {
+            exps.push([">=", ["PARAM", k], vv]);
+          } else {
+            throw new Error(`Operator ${kk} not supported`);
           }
-          exps.push([op, ["PARAM", k], vv]);
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
         }
         if (exps.length === 1) {
           expressions.push(exps[0]);
-        } else {
+        } else if (exps.length > 1) {
           const and: Expression = ["AND"];
           expressions.push(and.concat(exps));
         }
@@ -573,8 +571,8 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
       }
     }
     if (expressions.length === 1) return expressions[0];
-    const and: Expression = ["AND"];
-    return and.concat(expressions);
+    if (expressions.length === 0) return true;
+    return ["AND", ...expressions];
   }
 
   // empty filter
@@ -592,7 +590,9 @@ export function flattenPreset(
       // Try parse to check expression validity
       parse(p.precondition as string);
     } catch (error) {
-      p.precondition = mongoQueryToFilter(JSON.parse(p.precondition as string));
+      p.precondition = convertOldPrecondition(
+        JSON.parse(p.precondition as string)
+      );
       p.precondition = (p.precondition as string).length
         ? stringify(p.precondition as Expression)
         : "";

lib/session.ts

@@ -299,7 +299,7 @@ export async function transferComplete(
   sessionContext.operationsTouched[commandKey] = 1;
 
   if (rpcReq.faultStruct && rpcReq.faultStruct.faultCode !== "0") {
-    await revertDownloadParameters(sessionContext, operation.args.instance);
+    revertDownloadParameters(sessionContext, operation.args.instance);
 
     const fault: Fault = {
       code: `cwmp.${rpcReq.faultStruct.faultCode}`,
@@ -398,10 +398,10 @@ export async function transferComplete(
   };
 }
 
-async function revertDownloadParameters(
+function revertDownloadParameters(
   sessionContext: SessionContext,
   instance
-): Promise<void> {
+): void {
   const timestamp = sessionContext.timestamp + sessionContext.iteration + 1;
 
   let p;

lib/ui/api-functions.ts

@@ -110,7 +110,7 @@ export async function postTasks(
   try {
     await connectionRequest(deviceId, device);
   } catch (err) {
-    for (const t of statuses) db.deleteTask(new ObjectID(t._id));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
     return {
       connectionRequest: err.message,
       tasks: statuses,
@@ -138,9 +138,10 @@ export async function postTasks(
       r.status = "fault";
       r["fault"] = res[i * 2 + 1][0];
     }
-    db.deleteTask(new ObjectID(r._id));
   }
 
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+
   return { connectionRequest: "OK", tasks: statuses };
 }
 

lib/ui/api.ts

@@ -147,8 +147,9 @@ for (const [resource, flags] of Object.entries(resources)) {
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -195,8 +196,9 @@ for (const [resource, flags] of Object.entries(resources)) {
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -254,8 +256,9 @@ for (const [resource, flags] of Object.entries(resources)) {
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 

lib/ui/db.ts

@@ -128,78 +128,80 @@ export function query(
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then((client) => {
-      const collection = client
-        .db()
-        .collection(RESOURCE_COLLECTION[resource] || resource);
-      const cursor = collection.find(q);
-      if (options.projection) {
-        let projection = options.projection;
-        if (resource === "devices") {
-          projection = mongodbFunctions.processDeviceProjection(
-            options.projection
-          );
+    getClient()
+      .then((client) => {
+        const collection = client
+          .db()
+          .collection(RESOURCE_COLLECTION[resource] || resource);
+        const cursor = collection.find(q);
+        if (options.projection) {
+          let projection = options.projection;
+          if (resource === "devices") {
+            projection = mongodbFunctions.processDeviceProjection(
+              options.projection
+            );
+          }
+
+          if (resource === "presets") projection.configurations = 1;
+          cursor.project(projection);
         }
 
-        if (resource === "presets") projection.configurations = 1;
-        cursor.project(projection);
-      }
+        if (resource === "users") cursor.project({ password: 0, salt: 0 });
 
-      if (resource === "users") cursor.project({ password: 0, salt: 0 });
+        if (options.skip) cursor.skip(options.skip);
+        if (options.limit) cursor.limit(options.limit);
 
-      if (options.skip) cursor.skip(options.skip);
-      if (options.limit) cursor.limit(options.limit);
+        if (options.sort) {
+          let s = Object.entries(options.sort)
+            .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
+            .reduce(
+              (obj, [k, v]) =>
+                Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
+              {}
+            );
 
-      if (options.sort) {
-        let s = Object.entries(options.sort)
-          .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
-          .reduce(
-            (obj, [k, v]) =>
-              Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
-            {}
-          );
+          if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
+          cursor.sort(s);
+        }
 
-        if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
-        cursor.sort(s);
-      }
-
-      if (!callback) {
-        cursor.toArray((err, docs) => {
-          if (err) return reject(err);
-          if (resource === "devices")
-            docs = docs.map((d) => mongodbFunctions.flattenDevice(d));
-          else if (resource === "faults")
-            docs = docs.map((d) => mongodbFunctions.flattenFault(d));
-          else if (resource === "tasks")
-            docs = docs.map((d) => mongodbFunctions.flattenTask(d));
-          else if (resource === "presets")
-            docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
-          else if (resource === "files")
-            docs = docs.map((d) => mongodbFunctions.flattenFile(d));
-          return resolve(docs);
-        });
-      } else {
-        cursor.forEach(
-          (doc) => {
+        if (!callback) {
+          cursor.toArray((err, docs) => {
+            if (err) return reject(err);
             if (resource === "devices")
-              doc = mongodbFunctions.flattenDevice(doc);
+              docs = docs.map((d) => mongodbFunctions.flattenDevice(d));
             else if (resource === "faults")
-              doc = mongodbFunctions.flattenFault(doc);
+              docs = docs.map((d) => mongodbFunctions.flattenFault(d));
             else if (resource === "tasks")
-              doc = mongodbFunctions.flattenTask(doc);
+              docs = docs.map((d) => mongodbFunctions.flattenTask(d));
             else if (resource === "presets")
-              doc = mongodbFunctions.flattenPreset(doc);
+              docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
             else if (resource === "files")
-              doc = mongodbFunctions.flattenFile(doc);
-            callback(doc);
-          },
-          (err) => {
-            if (err) reject(err);
-            else resolve();
-          }
-        );
-      }
-    });
+              docs = docs.map((d) => mongodbFunctions.flattenFile(d));
+            return resolve(docs);
+          });
+        } else {
+          cursor.forEach(
+            (doc) => {
+              if (resource === "devices")
+                doc = mongodbFunctions.flattenDevice(doc);
+              else if (resource === "faults")
+                doc = mongodbFunctions.flattenFault(doc);
+              else if (resource === "tasks")
+                doc = mongodbFunctions.flattenTask(doc);
+              else if (resource === "presets")
+                doc = mongodbFunctions.flattenPreset(doc);
+              else if (resource === "files")
+                doc = mongodbFunctions.flattenFile(doc);
+              callback(doc);
+            },
+            (err) => {
+              if (err) reject(err);
+              else resolve();
+            }
+          );
+        }
+      })
+      .catch(reject);
   });
 }
 
@@ -221,15 +223,17 @@ export function count(resource: string, filter: Expression): Promise<number> {
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then((client) => {
-      const collection = client
-        .db()
-        .collection(RESOURCE_COLLECTION[resource] || resource);
-      collection.find(q).count((err, c) => {
-        if (err) reject(err);
-        else resolve(c);
-      });
-    });
+    getClient()
+      .then((client) => {
+        const collection = client
+          .db()
+          .collection(RESOURCE_COLLECTION[resource] || resource);
+        collection.find(q).count((err, c) => {
+          if (err) reject(err);
+          else resolve(c);
+        });
+      })
+      .catch(reject);
   });
 }
 

npm-shrinkwrap.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -498,9 +498,9 @@
       "dev": true
     },
     "@types/mongodb": {
-      "version": "3.5.26",
-      "resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-3.5.26.tgz",
-      "integrity": "sha512-p0X2VJgIBNHfNBdZdzzG8eQ/3bf6mQoXDT0UhVyVEdSzXEa1+2pFcwGvEZp72sjztyBwfRKlgrXMjCVavLcuGg==",
+      "version": "3.5.27",
+      "resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-3.5.27.tgz",
+      "integrity": "sha512-1jxKDgdfJEOO9zp+lv43p8jOqRs02xPrdUTzAZIVK9tVEySfCEmktL2jEu9A3wOBEOs18yKzpVIKUh8b8ALk3w==",
       "dev": true,
       "requires": {
         "@types/bson": "*",
@@ -1417,9 +1417,9 @@
       "dev": true
     },
     "bl": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.0.tgz",
-      "integrity": "sha512-wbgvOpqopSr7uq6fJrLH8EsvYMJf9gzfo2jCsL2eTy75qXPukA4pCgHamOQkZtY5vmfVtjB+P3LNlMHW5CEZXA==",
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz",
+      "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==",
       "requires": {
         "readable-stream": "^2.3.5",
         "safe-buffer": "^5.1.1"
@@ -3385,6 +3385,14 @@
         }
       }
     },
+    "espresso-iisojs": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/espresso-iisojs/-/espresso-iisojs-1.0.3.tgz",
+      "integrity": "sha512-VOu80fa4cEH5RVJ0Qqe+EcJKvDxVYWYyUsXybXXAsCTxO5CTvNSb0A5rhVSlp7++zCFg+r03FRUVgsXq6y6kVQ==",
+      "requires": {
+        "jsbi": "^3.1.3"
+      }
+    },
     "esprima": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
@@ -5396,9 +5404,9 @@
       }
     },
     "mongodb": {
-      "version": "3.6.0",
-      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.0.tgz",
-      "integrity": "sha512-/XWWub1mHZVoqEsUppE0GV7u9kanLvHxho6EvBxQbShXTKYF9trhZC2NzbulRGeG7xMJHD8IOWRcdKx5LPjAjQ==",
+      "version": "3.6.1",
+      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.1.tgz",
+      "integrity": "sha512-uH76Zzr5wPptnjEKJRQnwTsomtFOU/kQEU8a9hKHr2M7y9qVk7Q4Pkv0EQVp88742z9+RwvsdTw6dRjDZCNu1g==",
       "requires": {
         "bl": "^2.2.0",
         "bson": "^1.1.4",
@@ -9470,14 +9478,6 @@
         "decamelize": "^1.2.0"
       }
     },
-    "yebool": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/yebool/-/yebool-1.0.2.tgz",
-      "integrity": "sha512-3sgfZ1jSZNWyU5Xu0uIlA98tNl/Aj1sZVt89qZL4IXJVNG2+eWF4hp9NzQrOng6nO/ohvIPnzbIfp8qtBykBfw==",
-      "requires": {
-        "jsbi": "^3.1.3"
-      }
-    },
     "ylru": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/ylru/-/ylru-1.2.1.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -20,6 +20,7 @@
     "tools": "tools"
   },
   "dependencies": {
+    "espresso-iisojs": "^1.0.3",
     "iconv-lite": "^0.6.2",
     "ipaddr.js": "^2.0.0",
     "jsonwebtoken": "^8.5.1",
@@ -30,11 +31,10 @@
     "koa-router": "^9.4.0",
     "koa-static": "^5.0.0",
     "later": "^1.2.0",
-    "mongodb": "^3.6.0",
+    "mongodb": "^3.6.1",
     "parsimmon": "^1.16.0",
     "seedrandom": "^3.0.5",
-    "yaml": "^1.10.0",
-    "yebool": "^1.0.2"
+    "yaml": "^1.10.0"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^15.0.0",
@@ -47,7 +47,7 @@
     "@types/koa-router": "^7.4.1",
     "@types/koa-static": "^4.0.1",
     "@types/mithril": "^2.0.3",
-    "@types/mongodb": "^3.5.26",
+    "@types/mongodb": "^3.5.27",
     "@types/parsimmon": "^1.10.2",
     "@types/seedrandom": "^2.4.28",
     "@types/svgo": "^1.3.3",

test/mongodb-functions.ts

@@ -1,13 +1,13 @@
 import ava from "ava";
 import * as mongodbFunctions from "../lib/mongodb-functions";
-import { stringify } from "../lib/common/expression-parser";
+import { stringify, parse } from "../lib/common/expression-parser";
 
-ava("mongoQueryToFilter", (t) => {
+ava("convertOldPrecondition", (t) => {
   const tests = [
     [{}, "true"],
     [{ test: "test" }, 'test = "test"'],
     [{ test: { $eq: "test" } }, 'test = "test"'],
-    [{ test: { $ne: "test" } }, 'test <> "test"'],
+    [{ test: { $ne: "test" } }, 'test <> "test" OR test IS NULL'],
     [{ test: { $gte: "test" } }, 'test >= "test"'],
     [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
     [{ "Tags.test": false }, "Tags.test IS NULL"],
@@ -22,20 +22,20 @@ ava("mongoQueryToFilter", (t) => {
     [{ _tags: { $eq: "test" } }, "Tags.test IS NOT NULL"],
     [
       { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" AND test <> "test"',
+      'test = "test" AND (test <> "test" OR test IS NULL)',
     ],
     [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
     [
       { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" OR test <> "test"',
+      'test = "test" OR (test <> "test" OR test IS NULL)',
     ],
     [
       { test: { $gte: "test1", $ne: "test2" } },
-      'test >= "test1" AND test <> "test2"',
+      'test >= "test1" AND (test <> "test2" OR test IS NULL)',
     ],
     [
       { test: "test", test2: { $ne: "test2" } },
-      'test = "test" AND test2 <> "test2"',
+      'test = "test" AND (test2 <> "test2" OR test2 IS NULL)',
     ],
   ];
 
@@ -50,13 +50,83 @@ ava("mongoQueryToFilter", (t) => {
 
   t.plan(tests.length + 2 * shouldFailTests.length);
   for (const test of tests)
-    t.is(stringify(mongodbFunctions.mongoQueryToFilter(test[0])), test[1]);
+    t.is(stringify(mongodbFunctions.convertOldPrecondition(test[0])), test[1]);
 
   for (const test of shouldFailTests) {
     const func = (): void => {
-      mongodbFunctions.mongoQueryToFilter(test[0] as Record<string, unknown>);
+      mongodbFunctions.convertOldPrecondition(
+        test[0] as Record<string, unknown>
+      );
     };
     const error = t.throws(func, { instanceOf: Error });
     t.is(error.message, test[1]);
   }
 });
+
+ava("filterToMongoQuery", async (t) => {
+  const queries: [string, Record<string, unknown>][] = [
+    ["true", {}],
+    ["Tags.tag1 = true", { _tags: "tag1" }],
+    ["Tags.tag1 <> false", { _tags: "tag1" }],
+    ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }],
+    ["Tags.tag1 = 123", { "Tags.tag1": 123 }],
+    ["Param1 = 'value1'", { "Param1._value": "value1" }],
+    ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }],
+    ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
+    ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
+    ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }],
+    ["LOWER(Param1) LIKE 'value'", { "Param1._value": /^value$/i }],
+    [
+      "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          {
+            $and: [
+              { "Param2._value": { $nin: ["value1", null] } },
+              { "Param1._value": { $gte: "value2" } },
+            ],
+          },
+        ],
+      },
+    ],
+    [
+      "Param1 <> 'value2' OR Param1 IS NULL",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          { "Param1._value": null },
+        ],
+      },
+    ],
+  ];
+
+  for (const [expStr, expect] of queries) {
+    const exp = parse(expStr);
+    const query = mongodbFunctions.filterToMongoQuery(
+      mongodbFunctions.processDeviceFilter(exp)
+    );
+    t.deepEqual(query, expect);
+  }
+
+  const failQueries: [any, string][] = [
+    ["Param1 = Param2", "Invalid RHS operand of = clause"],
+    ["Param1 LIKE Param2", "Invalid RHS operand of LIKE clause"],
+    ["1 LIKE '1'", "Invalid LHS operand of LIKE clause"],
+    ["1 = 2", "Invalid LHS operand of = clause"],
+    ["1 IS NULL", "Invalid LHS operand of IS NULL clause"],
+    ["false", "Primitives are not valid queries"],
+    ["UPPER(Param1) LIKE 'value'", "Invalid RHS operand of LIKE clause"],
+  ];
+
+  for (const [expStr, err] of failQueries) {
+    const exp = parse(expStr);
+    t.throws(
+      () =>
+        mongodbFunctions.filterToMongoQuery(
+          mongodbFunctions.processDeviceFilter(exp)
+        ),
+      { message: err }
+    );
+  }
+});

ui/app.ts

@@ -71,7 +71,7 @@ let fulfillTimeout = null;
 function fulfill(): void {
   clearTimeout(fulfillTimeout);
   fulfillTimeout = setTimeout(() => {
-    store.fulfill(lastRenderTimestamp, pageVisitTimestamp).then((updated) => {
+    store.fulfill(lastRenderTimestamp, pageVisitTimestamp, (updated) => {
       if (updated) m.redraw();
     });
   }, 100);

ui/components/ping.ts

@@ -27,23 +27,28 @@ const component: ClosureComponent = (vn): Component => {
   let interval: ReturnType<typeof setInterval>;
   let host: string;
 
-  const refresh = async (): Promise<void> => {
-    let status = "";
-    if (host) {
-      try {
-        const res = await store.ping(host);
-        if (res["avg"] != null) status = `${Math.trunc(res["avg"])} ms`;
-        else status = "Unreachable";
-      } catch (err) {
-        setTimeout(() => {
-          throw err;
-        }, 0);
-      }
+  const refresh = (): void => {
+    if (!host) {
+      const dom = (vn as VnodeDOM).dom;
+      if (dom) dom.innerHTML = "";
+      return;
     }
 
-    const dom = (vn as VnodeDOM).dom;
-
-    if (dom) dom.innerHTML = `Pinging ${host}: ${status}`;
+    let status = "";
+    store
+      .ping(host)
+      .then((res) => {
+        if (res["avg"] != null) status = `${Math.trunc(res["avg"])} ms`;
+        else status = "Unreachable";
+      })
+      .catch(() => {
+        status = "Error!";
+        clearInterval(interval);
+      })
+      .finally(() => {
+        const dom = (vn as VnodeDOM).dom;
+        if (dom) dom.innerHTML = `Pinging ${host}: ${status}`;
+      });
   };
 
   return {

ui/components/summon-button.ts

@@ -79,6 +79,9 @@ const component: ClosureComponent = (): Component => {
               .then(() => {
                 e.target.disabled = false;
                 store.fulfill(0, Date.now());
+              })
+              .catch((err) => {
+                throw err;
               });
           },
         },

ui/drawer-component.ts

@@ -441,6 +441,9 @@ const component: ClosureComponent = (): Component => {
                   )
                   .then(() => {
                     store.fulfill(0, Date.now());
+                  })
+                  .catch((err) => {
+                    throw err;
                   });
               },
             },

ui/store.ts

@@ -152,6 +152,8 @@ function checkConnection(): void {
         }
       }
     },
+  }).catch((err) => {
+    notifications.push("error", err.message);
   });
 }
 
@@ -169,11 +171,25 @@ export async function xhrRequest(
   ): any => {
     if (typeof extract === "function") return extract(xhr, _options);
 
+    // https://mithril.js.org/request.html#error-handling
+    if (xhr.status !== 304 && Math.floor(xhr.status / 100) !== 2) {
+      const err = new Error();
+      err["message"] =
+        xhr.status === 0
+          ? "Server is unrachable"
+          : `Unexpected response status code ${xhr.status}`;
+      err["code"] = xhr.status;
+      err["response"] = xhr.responseText;
+      throw err;
+    }
+
     let response: any;
     if (typeof deserialize === "function") {
       response = deserialize(xhr.responseText);
     } else if (
-      xhr.getResponseHeader("content-type").startsWith("application/json")
+      (xhr.getResponseHeader("content-type") || "").startsWith(
+        "application/json"
+      )
     ) {
       try {
         response = xhr.responseText ? JSON.parse(xhr.responseText) : null;
@@ -184,15 +200,6 @@ export async function xhrRequest(
       response = xhr.responseText;
     }
 
-    // https://mithril.js.org/request.html#error-handling
-    if (xhr.status !== 304 && Math.floor(xhr.status / 100) !== 2) {
-      const err = new Error();
-      err["message"] = xhr.responseText;
-      err["code"] = xhr.status;
-      err["response"] = response;
-      throw err;
-    }
-
     return response;
   };
 
@@ -340,7 +347,7 @@ export function fetch(
   return queryResponse;
 }
 
-export function fulfill(
+function _fulfill(
   accessTimestamp: number,
   _fulfillTimestamp: number
 ): Promise<boolean> {
@@ -377,7 +384,16 @@ export function fulfill(
                 m.buildQueryString({
                   filter: memoizedStringify(filter),
                 }),
-              extract: (xhr) => +xhr.getResponseHeader("x-total-count"),
+              extract: (xhr) => {
+                if (!xhr.status) {
+                  throw new Error("Server is unrachable");
+                } else if (xhr.status !== 200) {
+                  throw new Error(
+                    `Unexpected response status code ${xhr.status}`
+                  );
+                }
+                return +xhr.getResponseHeader("x-total-count");
+              },
               background: true,
             })
               .then((c) => {
@@ -497,14 +513,14 @@ export function fulfill(
             continue;
           }
 
-          const combinedFilterDiff = diff;
-          resources[resourceType].combinedFilter = union;
-
           updated = true;
           let deleted = new Set<string>();
           if (!resources[resourceType].combinedFilter)
             deleted = new Set(resources[resourceType].objects.keys());
 
+          const combinedFilterDiff = diff;
+          resources[resourceType].combinedFilter = union;
+
           allPromises2.push(
             new Promise((resolve2, reject2) => {
               xhrRequest({
@@ -560,6 +576,17 @@ export function fulfill(
   });
 }
 
+export function fulfill(
+  accessTimestamp: number,
+  _fulfillTimestamp: number,
+  callback?: (updated: boolean) => void
+): void {
+  _fulfill(accessTimestamp, _fulfillTimestamp).then(callback, (err) => {
+    notifications.push("error", err.message);
+    if (callback) callback(false);
+  });
+}
+
 export function getTimestamp(): number {
   return fulfillTimestamp;
 }
@@ -647,7 +674,12 @@ export function resourceExists(resource: string, id: string): Promise<number> {
       m.buildQueryString({
         filter: memoizedStringify(filter),
       }),
-    extract: (xhr) => +xhr.getResponseHeader("x-total-count"),
+    extract: (xhr) => {
+      if (!xhr.status) throw new Error("Server is unrachable");
+      else if (xhr.status !== 200)
+        throw new Error(`Unexpected response status code ${xhr.status}`);
+      return +xhr.getResponseHeader("x-total-count");
+    },
     background: true,
   });
 }

ui/wizard-page.ts

@@ -87,16 +87,24 @@ export const component: ClosureComponent = (vnode): Component => {
               })
                 .then(() => {
                   setTimeout(() => {
-                    m.request({ url: "init" }).then((o) => {
-                      e.target.disabled = false;
-                      options = o;
-                      notifications.push("success", "Initialization complete", {
-                        "Open Sesame!": () => {
-                          m.route.set("/login");
-                          window.location.reload();
-                        },
+                    m.request({ url: "init" })
+                      .then((o) => {
+                        e.target.disabled = false;
+                        options = o;
+                        notifications.push(
+                          "success",
+                          "Initialization complete",
+                          {
+                            "Open Sesame!": () => {
+                              m.route.set("/login");
+                              window.location.reload();
+                            },
+                          }
+                        );
+                      })
+                      .catch((err) => {
+                        notifications.push("error", err.message);
                       });
-                    });
                   }, 3000);
                   if (opts["users"]) {
                     alert(