Convert presets preconditions to use expressions

lib/cwmp.ts

@@ -25,7 +25,7 @@ import * as config from "./config";
 import * as common from "./common";
 import * as soap from "./soap";
 import * as session from "./session";
-import * as query from "./query";
+import { evaluate, extractParams } from "./common/expression";
 import * as device from "./device";
 import * as cache from "./cache";
 import * as localCache from "./local-cache";
@@ -279,7 +279,7 @@ function transferComplete(sessionContext, rpc): void {
   );
 }
 
-// Append providions and remove duplicates
+// Append provisions and remove duplicates
 function appendProvisions(original, toAppend): boolean {
   let modified = false;
   const stringified = new WeakMap();
@@ -384,11 +384,8 @@ function applyPresets(sessionContext: SessionContext): void {
     }
 
     filteredPresets.push(preset);
-    for (const k of Object.keys(preset.precondition)) {
+    for (const k of extractParams(preset.precondition))
-      sessionContext.channels[preset.channel] = 0;
+      parameters[k] = Path.parse(k);
-      const p = k.split(/([^a-zA-Z0-9\-_.].*)/, 1)[0];
-      parameters[p] = Path.parse(p);
-    }
   }
 
   const declarations = Object.values(parameters).map(v => ({
@@ -429,8 +426,9 @@ function applyPresets(sessionContext: SessionContext): void {
         session.addProvisions(sessionContext, whiteList, whiteListProvisions);
 
       const appendProvisionsToFaults = {};
+      const now = Date.now();
       for (const p of filteredPresets) {
-        if (query.testFilter(parameterValues, p.precondition)) {
+        if (evaluate(p.precondition, parameterValues, now)) {
           if (blackList[p.channel] === 2) {
             appendProvisionsToFaults[p.channel] = (
               appendProvisionsToFaults[p.channel] || []

lib/local-cache.ts

@@ -22,7 +22,7 @@ import * as crypto from "crypto";
 import * as config from "./config";
 import * as db from "./db";
 import * as cache from "./cache";
-import * as query from "./query";
+import { mongoQueryToFilter } from "./mongodb-functions";
 import * as logger from "./logger";
 import * as scheduling from "./scheduling";
 import * as expression from "./common/expression";
@@ -164,9 +164,15 @@ function refresh(callback): void {
                 }
 
                 const events = preset.events || {};
-                const precondition = query.convertMongoQueryToFilters(
+                let precondition;
+                try {
+                  precondition = parse(preset.precondition);
+                } catch (error) {
+                  precondition = mongoQueryToFilter(
                     JSON.parse(preset.precondition)
                   );
+                }
+
                 const _provisions = preset.provisions || [];
 
                 // Generate provisions from the old configuration format

lib/mongodb-functions.ts

@@ -1,7 +1,7 @@
 import { ObjectID } from "mongodb";
-import { map, parse, stringify } from "../common/expression-parser";
+import { map, parse, stringify } from "./common/expression-parser";
-import { likePatternToRegExp, evaluate } from "../common/expression";
+import { likePatternToRegExp, evaluate } from "./common/expression";
-import { Expression, Fault, Task } from "../types";
+import { Expression, Fault, Task } from "./types";
 
 const isArray = Array.isArray;
 
@@ -436,33 +436,44 @@ export function flattenTask(task): Task {
   return t;
 }
 
-function mongoQueryToFilter(query): Expression {
+export function mongoQueryToFilter(query): Expression {
-  const expressions = [];
+  function recursive(_query): Expression {
-  function recursive(_query): void {
+    const expressions: Expression[] = [];
     for (const [k, v] of Object.entries(_query)) {
       if (k[0] === "$") {
-        if (k === "$and") for (const vv of Object.values(v)) recursive(vv);
+        if (k === "$and") {
-        else throw new Error(`Operator ${k} not supported`);
+          const and: Expression = ["AND"];
+          for (const vv of Object.values(v)) and.push(recursive(vv));
+          expressions.push(and);
+        } else if (k === "$or") {
+          const or: Expression = ["OR"];
+          for (const vv of Object.values(v)) or.push(recursive(vv));
+          expressions.push(or);
+        } else {
+          throw new Error(`Operator ${k} not supported`);
+        }
       } else if (k === "_tags") {
         if (typeof v === "object") {
           if (isArray(v)) throw new Error(`Invalid type`);
 
-          for (const [kk, vv] of Object.entries(v)) {
+          if (v.hasOwnProperty("$ne"))
-            if (kk === "$ne")
+            expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]);
-              expressions.push(["IS NULL", ["PARAM", `Tags.${vv}`]]);
+          else throw new Error(`Invalid tag query`);
-            else throw new Error(`Operator ${kk} not supported`);
-          }
         } else {
           expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
         }
       } else if (k.startsWith("Tags.")) {
-        expressions.push([
+        let exists: boolean;
-          v["$exists"] ? "IS NOT NULL" : "IS NULL",
+        if (typeof v === "boolean") exists = v;
-          ["PARAM", k]
+        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
-        ]);
+        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
+        else throw new Error(`Invalid tag query`);
+
+        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
       } else if (typeof v === "object") {
         if (isArray(v)) throw new Error(`Invalid type`);
 
+        const exps: Expression[] = [];
         for (const [kk, vv] of Object.entries(v)) {
           let op;
           switch (kk) {
@@ -487,28 +498,37 @@ function mongoQueryToFilter(query): Expression {
             default:
               throw new Error(`Operator ${kk} not supported`);
           }
-          expressions.push([op, ["PARAM", k], vv]);
+          exps.push([op, ["PARAM", k], vv]);
         }
+        if (exps.length === 1) return exps[0];
+        const and: Expression = ["AND"];
+        expressions.push(and.concat(exps));
       } else {
         expressions.push(["=", ["PARAM", k], v]);
       }
     }
-  }
-  recursive(query);
-
-  if (expressions.length > 1) return ["AND"].concat(expressions);
-
     if (expressions.length === 1) return expressions[0];
+    const and: Expression = ["AND"];
+    return and.concat(expressions);
+  }
 
-  return expressions;
+  // empty filter
+  if (!Object.keys(query).length) return true;
+
+  return recursive(query);
 }
 
 export function flattenPreset(preset): {} {
   const p = Object.assign({}, preset);
   if (p.precondition) {
+    try {
+      // Try parse to check expression validity
+      parse(p.precondition);
+    } catch (error) {
       p.precondition = mongoQueryToFilter(JSON.parse(p.precondition));
       p.precondition = p.precondition.length ? stringify(p.precondition) : "";
     }
+  }
 
   if (p.events) {
     const e = [];
@@ -546,10 +566,10 @@ export function flattenFile(file): {} {
 
 export function preProcessPreset(data): {} {
   const preset = Object.assign({}, data);
-  preset.precondition = preset.precondition
+
-    ? filterToMongoQuery(parse(preset.precondition))
+  if (!preset.precondition) preset.precondition = "";
-    : {};
+  // Try parse to check expression validity
-  preset.precondition = JSON.stringify(preset.precondition);
+  parse(preset.precondition);
 
   preset.weight = parseInt(preset.weight) || 0;
 

lib/query.ts

@@ -216,135 +216,6 @@ export function test(params, query): boolean {
   return true;
 }
 
-function matchType(src, dst): string | number | boolean {
-  const type = typeof src;
-  if (type === "string") {
-    return String(dst);
-  } else if (type === "number") {
-    if (+dst === parseFloat(dst)) return +dst;
-    else if (!isNaN(Date.parse(dst))) return Date.parse(dst);
-  } else if (type === "boolean") {
-    const v = String(dst)
-      .trim()
-      .toLowerCase();
-    if (v === "true" || v === "1") return true;
-    else if (v === "false" || v === "0") return false;
-  }
-  return dst;
-}
-
-export function testFilter(obj, filter): boolean {
-  for (const [k, v] of Object.entries(filter)) {
-    const [param, op] = k.split(/([^a-zA-Z0-9\-_.].*)/, 2);
-    const val = matchType(obj[param], v);
-    switch (op) {
-      case "=":
-      case undefined:
-        if (obj[param] !== val) return false;
-        break;
-      case ">":
-        if (!(obj[param] > val)) return false;
-        break;
-      case ">=":
-        if (!(obj[param] >= val)) return false;
-        break;
-      case "<":
-        if (!(obj[param] < val)) return false;
-        break;
-      case "<=":
-        if (!(obj[param] <= val)) return false;
-        break;
-      case "!":
-        if (obj[param] === val) return false;
-        break;
-      default:
-        throw new Error(`Unrecognized operator ${op}`);
-    }
-  }
-  return true;
-}
-
-export function convertMongoQueryToFilters(query, filters = {}): {} {
-  for (let [k, v] of Object.entries(query)) {
-    if (k[0] === "$") {
-      if (k === "$and")
-        for (const vv of v as any[]) convertMongoQueryToFilters(vv, filters);
-      else throw new Error(`Operator ${k} not supported`);
-    } else if (k === "_tags") {
-      if (common.typeOf(v) === common.OBJECT_TYPE) {
-        for (let [kk, vv] of Object.entries(v)) {
-          vv = vv.replace(/[^a-zA-Z0-9-]+/g, "_");
-          if (kk === "$ne") filters[`Tags.${vv}!`] = true;
-          else throw new Error(`Operator ${kk} not supported`);
-        }
-      } else {
-        v = (v as string).replace(/[^a-zA-Z0-9-]+/g, "_");
-        filters[`Tags.${v}`] = true;
-      }
-    } else {
-      switch (k) {
-        case "_id":
-          k = "DeviceID.ID";
-          break;
-        case "_deviceId._Manufacturer":
-          k = "DeviceID.Manufacturer";
-          break;
-        case "_deviceId._OUI":
-          k = "DeviceID.OUI";
-          break;
-        case "_deviceId._ProductClass":
-          k = "DeviceID.ProductClass";
-          break;
-        case "_deviceId._SerialNumber":
-          k = "DeviceID.SerialNumber";
-          break;
-        case "_lastInform":
-          k = "Events.Inform";
-          break;
-        case "_lastBootstrap":
-          k = "Events.0_BOOTSTRAP";
-          break;
-        case "_lastBoot":
-          k = "Events.1_BOOT";
-          break;
-        case "_registered":
-          k = "Events.Registered";
-      }
-
-      if (common.typeOf(v) === common.OBJECT_TYPE) {
-        for (const [kk, vv] of Object.entries(v)) {
-          switch (kk) {
-            case "$eq":
-              filters[k] = vv;
-              break;
-            case "$ne":
-              filters[`${k}!`] = vv;
-              break;
-            case "$lt":
-              filters[`${k}<`] = vv;
-              break;
-            case "$lte":
-              filters[`${k}<=`] = vv;
-              break;
-            case "$gt":
-              filters[`${k}>`] = vv;
-              break;
-            case "$gte":
-              filters[`${k}>=`] = vv;
-              break;
-            default:
-              throw new Error(`Oprator ${kk} not supported`);
-          }
-        }
-      } else {
-        filters[k] = v;
-      }
-    }
-  }
-
-  return filters;
-}
-
 export function sanitizeQueryTypes(query, types): {} {
   for (const [k, v] of Object.entries(query)) {
     if (k[0] === "$") {

lib/ui/db.ts

@@ -1,8 +1,7 @@
 import { MongoClient, GridFSBucket } from "mongodb";
 import * as config from "../config";
-import * as mongodbFunctions from "./mongodb-functions";
+import * as mongodbFunctions from "../mongodb-functions";
 import * as expression from "../common/expression";
-import { preProcessPreset } from "./mongodb-functions";
 import { QueryOptions } from "../types";
 
 const CACHE_TTL = 300000;
@@ -257,7 +256,7 @@ function deleteResource(resource, id): Promise<void> {
 }
 
 export function putPreset(id, object): Promise<void> {
-  object = preProcessPreset(object);
+  object = mongodbFunctions.preProcessPreset(object);
   return putResource("presets", id, object);
 }
 

test/mongodb-functions.ts

@@ -0,0 +1,55 @@
+import ava from "ava";
+import * as mongodbFunctions from "../lib/mongodb-functions";
+import { stringify } from "../lib/common/expression-parser";
+
+ava("mongoQueryToFilter", t => {
+  const tests = [
+    [{}, "true"],
+    [{ test: "test" }, 'test = "test"'],
+    [{ test: { $eq: "test" } }, 'test = "test"'],
+    [{ test: { $ne: "test" } }, 'test <> "test"'],
+    [{ test: { $gte: "test" } }, 'test >= "test"'],
+    [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
+    [{ "Tags.test": false }, "Tags.test IS NULL"],
+    [{ "Tags.test": { $exists: true } }, "Tags.test IS NOT NULL"],
+    [{ "Tags.test": { $exists: false } }, "Tags.test IS NULL"],
+    [{ "Tags.test": { $ne: true } }, "Tags.test IS NULL"],
+    [{ "Tags.test": { $ne: false } }, "Tags.test IS NOT NULL"],
+    [{ _tags: "test" }, "Tags.test IS NOT NULL"],
+    [{ _tags: { $ne: "test" } }, "Tags.test IS NULL"],
+    [
+      { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
+      'test = "test" AND test <> "test"'
+    ],
+    [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
+    [
+      { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
+      'test = "test" OR test <> "test"'
+    ],
+    [
+      { test: { $gte: "test1", $ne: "test2" } },
+      'test >= "test1" AND test <> "test2"'
+    ]
+  ];
+
+  const shouldFailTests = [
+    [{ test: { $gee: "test" } }, "Operator $gee not supported"],
+    [{ test: [] }, "Invalid type"],
+    [{ "Tags.test": { $eq: true } }, "Invalid tag query"],
+    [{ _tags: [] }, "Invalid type"],
+    [{ _tags: { $eq: "test" } }, "Invalid tag query"],
+    [{ $nor: [] }, "Operator $nor not supported"]
+  ];
+
+  t.plan(tests.length + 2 * shouldFailTests.length);
+  for (const test of tests)
+    t.is(stringify(mongodbFunctions.mongoQueryToFilter(test[0])), test[1]);
+
+  for (const test of shouldFailTests) {
+    const func = (): void => {
+      mongodbFunctions.mongoQueryToFilter(test[0]);
+    };
+    const error = t.throws(func, Error);
+    t.is(error.message, test[1]);
+  }
+});

ui/filter-component.ts

@@ -3,7 +3,7 @@ import { parse, stringify, map } from "../lib/common/expression-parser";
 import memoize from "../lib/common/memoize";
 import Autocomplete from "./autocomplete-compnent";
 import * as smartQuery from "./smart-query";
-import { filterToMongoQuery } from "../lib/ui/mongodb-functions";
+import { filterToMongoQuery } from "../lib/mongodb-functions";
 import { Expression } from "../lib/types";
 
 const getAutocomplete = memoize(resource => {