From edadfe54c8c0fa1864f783565962ca515c2dfd23 Mon Sep 17 00:00:00 2001 From: Shweelan Samson Date: Wed, 6 Mar 2019 22:30:41 +0300 Subject: [PATCH] Convert presets preconditions to use expressions --- lib/cwmp.ts | 14 ++-- lib/local-cache.ts | 14 +++- lib/{ui => }/mongodb-functions.ts | 78 +++++++++++------- lib/query.ts | 129 ------------------------------ lib/ui/db.ts | 5 +- test/mongodb-functions.ts | 55 +++++++++++++ ui/filter-component.ts | 2 +- 7 files changed, 123 insertions(+), 174 deletions(-) rename lib/{ui => }/mongodb-functions.ts (88%) create mode 100644 test/mongodb-functions.ts diff --git a/lib/cwmp.ts b/lib/cwmp.ts index e3d4ba2..e8cd9a5 100644 --- a/lib/cwmp.ts +++ b/lib/cwmp.ts @@ -25,7 +25,7 @@ import * as config from "./config"; import * as common from "./common"; import * as soap from "./soap"; import * as session from "./session"; -import * as query from "./query"; +import { evaluate, extractParams } from "./common/expression"; import * as device from "./device"; import * as cache from "./cache"; import * as localCache from "./local-cache"; @@ -279,7 +279,7 @@ function transferComplete(sessionContext, rpc): void { ); } -// Append providions and remove duplicates +// Append provisions and remove duplicates function appendProvisions(original, toAppend): boolean { let modified = false; const stringified = new WeakMap(); @@ -384,11 +384,8 @@ function applyPresets(sessionContext: SessionContext): void { } filteredPresets.push(preset); - for (const k of Object.keys(preset.precondition)) { - sessionContext.channels[preset.channel] = 0; - const p = k.split(/([^a-zA-Z0-9\-_.].*)/, 1)[0]; - parameters[p] = Path.parse(p); - } + for (const k of extractParams(preset.precondition)) + parameters[k] = Path.parse(k); } const declarations = Object.values(parameters).map(v => ({ @@ -429,8 +426,9 @@ function applyPresets(sessionContext: SessionContext): void { session.addProvisions(sessionContext, whiteList, whiteListProvisions); const appendProvisionsToFaults = {}; + const now = Date.now(); for (const p of filteredPresets) { - if (query.testFilter(parameterValues, p.precondition)) { + if (evaluate(p.precondition, parameterValues, now)) { if (blackList[p.channel] === 2) { appendProvisionsToFaults[p.channel] = ( appendProvisionsToFaults[p.channel] || [] diff --git a/lib/local-cache.ts b/lib/local-cache.ts index cf04d49..7d66f79 100644 --- a/lib/local-cache.ts +++ b/lib/local-cache.ts @@ -22,7 +22,7 @@ import * as crypto from "crypto"; import * as config from "./config"; import * as db from "./db"; import * as cache from "./cache"; -import * as query from "./query"; +import { mongoQueryToFilter } from "./mongodb-functions"; import * as logger from "./logger"; import * as scheduling from "./scheduling"; import * as expression from "./common/expression"; @@ -164,9 +164,15 @@ function refresh(callback): void { } const events = preset.events || {}; - const precondition = query.convertMongoQueryToFilters( - JSON.parse(preset.precondition) - ); + let precondition; + try { + precondition = parse(preset.precondition); + } catch (error) { + precondition = mongoQueryToFilter( + JSON.parse(preset.precondition) + ); + } + const _provisions = preset.provisions || []; // Generate provisions from the old configuration format diff --git a/lib/ui/mongodb-functions.ts b/lib/mongodb-functions.ts similarity index 88% rename from lib/ui/mongodb-functions.ts rename to lib/mongodb-functions.ts index 058fd29..45dfe18 100644 --- a/lib/ui/mongodb-functions.ts +++ b/lib/mongodb-functions.ts @@ -1,7 +1,7 @@ import { ObjectID } from "mongodb"; -import { map, parse, stringify } from "../common/expression-parser"; -import { likePatternToRegExp, evaluate } from "../common/expression"; -import { Expression, Fault, Task } from "../types"; +import { map, parse, stringify } from "./common/expression-parser"; +import { likePatternToRegExp, evaluate } from "./common/expression"; +import { Expression, Fault, Task } from "./types"; const isArray = Array.isArray; @@ -436,33 +436,44 @@ export function flattenTask(task): Task { return t; } -function mongoQueryToFilter(query): Expression { - const expressions = []; - function recursive(_query): void { +export function mongoQueryToFilter(query): Expression { + function recursive(_query): Expression { + const expressions: Expression[] = []; for (const [k, v] of Object.entries(_query)) { if (k[0] === "$") { - if (k === "$and") for (const vv of Object.values(v)) recursive(vv); - else throw new Error(`Operator ${k} not supported`); + if (k === "$and") { + const and: Expression = ["AND"]; + for (const vv of Object.values(v)) and.push(recursive(vv)); + expressions.push(and); + } else if (k === "$or") { + const or: Expression = ["OR"]; + for (const vv of Object.values(v)) or.push(recursive(vv)); + expressions.push(or); + } else { + throw new Error(`Operator ${k} not supported`); + } } else if (k === "_tags") { if (typeof v === "object") { if (isArray(v)) throw new Error(`Invalid type`); - for (const [kk, vv] of Object.entries(v)) { - if (kk === "$ne") - expressions.push(["IS NULL", ["PARAM", `Tags.${vv}`]]); - else throw new Error(`Operator ${kk} not supported`); - } + if (v.hasOwnProperty("$ne")) + expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]); + else throw new Error(`Invalid tag query`); } else { expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]); } } else if (k.startsWith("Tags.")) { - expressions.push([ - v["$exists"] ? "IS NOT NULL" : "IS NULL", - ["PARAM", k] - ]); + let exists: boolean; + if (typeof v === "boolean") exists = v; + else if (v.hasOwnProperty("$ne")) exists = !v["$ne"]; + else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"]; + else throw new Error(`Invalid tag query`); + + expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]); } else if (typeof v === "object") { if (isArray(v)) throw new Error(`Invalid type`); + const exps: Expression[] = []; for (const [kk, vv] of Object.entries(v)) { let op; switch (kk) { @@ -487,27 +498,36 @@ function mongoQueryToFilter(query): Expression { default: throw new Error(`Operator ${kk} not supported`); } - expressions.push([op, ["PARAM", k], vv]); + exps.push([op, ["PARAM", k], vv]); } + if (exps.length === 1) return exps[0]; + const and: Expression = ["AND"]; + expressions.push(and.concat(exps)); } else { expressions.push(["=", ["PARAM", k], v]); } } + if (expressions.length === 1) return expressions[0]; + const and: Expression = ["AND"]; + return and.concat(expressions); } - recursive(query); - if (expressions.length > 1) return ["AND"].concat(expressions); + // empty filter + if (!Object.keys(query).length) return true; - if (expressions.length === 1) return expressions[0]; - - return expressions; + return recursive(query); } export function flattenPreset(preset): {} { const p = Object.assign({}, preset); if (p.precondition) { - p.precondition = mongoQueryToFilter(JSON.parse(p.precondition)); - p.precondition = p.precondition.length ? stringify(p.precondition) : ""; + try { + // Try parse to check expression validity + parse(p.precondition); + } catch (error) { + p.precondition = mongoQueryToFilter(JSON.parse(p.precondition)); + p.precondition = p.precondition.length ? stringify(p.precondition) : ""; + } } if (p.events) { @@ -546,10 +566,10 @@ export function flattenFile(file): {} { export function preProcessPreset(data): {} { const preset = Object.assign({}, data); - preset.precondition = preset.precondition - ? filterToMongoQuery(parse(preset.precondition)) - : {}; - preset.precondition = JSON.stringify(preset.precondition); + + if (!preset.precondition) preset.precondition = ""; + // Try parse to check expression validity + parse(preset.precondition); preset.weight = parseInt(preset.weight) || 0; diff --git a/lib/query.ts b/lib/query.ts index 20e95f2..da3220b 100644 --- a/lib/query.ts +++ b/lib/query.ts @@ -216,135 +216,6 @@ export function test(params, query): boolean { return true; } -function matchType(src, dst): string | number | boolean { - const type = typeof src; - if (type === "string") { - return String(dst); - } else if (type === "number") { - if (+dst === parseFloat(dst)) return +dst; - else if (!isNaN(Date.parse(dst))) return Date.parse(dst); - } else if (type === "boolean") { - const v = String(dst) - .trim() - .toLowerCase(); - if (v === "true" || v === "1") return true; - else if (v === "false" || v === "0") return false; - } - return dst; -} - -export function testFilter(obj, filter): boolean { - for (const [k, v] of Object.entries(filter)) { - const [param, op] = k.split(/([^a-zA-Z0-9\-_.].*)/, 2); - const val = matchType(obj[param], v); - switch (op) { - case "=": - case undefined: - if (obj[param] !== val) return false; - break; - case ">": - if (!(obj[param] > val)) return false; - break; - case ">=": - if (!(obj[param] >= val)) return false; - break; - case "<": - if (!(obj[param] < val)) return false; - break; - case "<=": - if (!(obj[param] <= val)) return false; - break; - case "!": - if (obj[param] === val) return false; - break; - default: - throw new Error(`Unrecognized operator ${op}`); - } - } - return true; -} - -export function convertMongoQueryToFilters(query, filters = {}): {} { - for (let [k, v] of Object.entries(query)) { - if (k[0] === "$") { - if (k === "$and") - for (const vv of v as any[]) convertMongoQueryToFilters(vv, filters); - else throw new Error(`Operator ${k} not supported`); - } else if (k === "_tags") { - if (common.typeOf(v) === common.OBJECT_TYPE) { - for (let [kk, vv] of Object.entries(v)) { - vv = vv.replace(/[^a-zA-Z0-9-]+/g, "_"); - if (kk === "$ne") filters[`Tags.${vv}!`] = true; - else throw new Error(`Operator ${kk} not supported`); - } - } else { - v = (v as string).replace(/[^a-zA-Z0-9-]+/g, "_"); - filters[`Tags.${v}`] = true; - } - } else { - switch (k) { - case "_id": - k = "DeviceID.ID"; - break; - case "_deviceId._Manufacturer": - k = "DeviceID.Manufacturer"; - break; - case "_deviceId._OUI": - k = "DeviceID.OUI"; - break; - case "_deviceId._ProductClass": - k = "DeviceID.ProductClass"; - break; - case "_deviceId._SerialNumber": - k = "DeviceID.SerialNumber"; - break; - case "_lastInform": - k = "Events.Inform"; - break; - case "_lastBootstrap": - k = "Events.0_BOOTSTRAP"; - break; - case "_lastBoot": - k = "Events.1_BOOT"; - break; - case "_registered": - k = "Events.Registered"; - } - - if (common.typeOf(v) === common.OBJECT_TYPE) { - for (const [kk, vv] of Object.entries(v)) { - switch (kk) { - case "$eq": - filters[k] = vv; - break; - case "$ne": - filters[`${k}!`] = vv; - break; - case "$lt": - filters[`${k}<`] = vv; - break; - case "$lte": - filters[`${k}<=`] = vv; - break; - case "$gt": - filters[`${k}>`] = vv; - break; - case "$gte": - filters[`${k}>=`] = vv; - break; - default: - throw new Error(`Oprator ${kk} not supported`); - } - } - } else { - filters[k] = v; - } - } - } - - return filters; -} - export function sanitizeQueryTypes(query, types): {} { for (const [k, v] of Object.entries(query)) { if (k[0] === "$") { diff --git a/lib/ui/db.ts b/lib/ui/db.ts index d9f245f..b208409 100644 --- a/lib/ui/db.ts +++ b/lib/ui/db.ts @@ -1,8 +1,7 @@ import { MongoClient, GridFSBucket } from "mongodb"; import * as config from "../config"; -import * as mongodbFunctions from "./mongodb-functions"; +import * as mongodbFunctions from "../mongodb-functions"; import * as expression from "../common/expression"; -import { preProcessPreset } from "./mongodb-functions"; import { QueryOptions } from "../types"; const CACHE_TTL = 300000; @@ -257,7 +256,7 @@ function deleteResource(resource, id): Promise { } export function putPreset(id, object): Promise { - object = preProcessPreset(object); + object = mongodbFunctions.preProcessPreset(object); return putResource("presets", id, object); } diff --git a/test/mongodb-functions.ts b/test/mongodb-functions.ts new file mode 100644 index 0000000..bc14ff1 --- /dev/null +++ b/test/mongodb-functions.ts @@ -0,0 +1,55 @@ +import ava from "ava"; +import * as mongodbFunctions from "../lib/mongodb-functions"; +import { stringify } from "../lib/common/expression-parser"; + +ava("mongoQueryToFilter", t => { + const tests = [ + [{}, "true"], + [{ test: "test" }, 'test = "test"'], + [{ test: { $eq: "test" } }, 'test = "test"'], + [{ test: { $ne: "test" } }, 'test <> "test"'], + [{ test: { $gte: "test" } }, 'test >= "test"'], + [{ "Tags.test": true }, "Tags.test IS NOT NULL"], + [{ "Tags.test": false }, "Tags.test IS NULL"], + [{ "Tags.test": { $exists: true } }, "Tags.test IS NOT NULL"], + [{ "Tags.test": { $exists: false } }, "Tags.test IS NULL"], + [{ "Tags.test": { $ne: true } }, "Tags.test IS NULL"], + [{ "Tags.test": { $ne: false } }, "Tags.test IS NOT NULL"], + [{ _tags: "test" }, "Tags.test IS NOT NULL"], + [{ _tags: { $ne: "test" } }, "Tags.test IS NULL"], + [ + { $and: [{ test: "test" }, { test: { $ne: "test" } }] }, + 'test = "test" AND test <> "test"' + ], + [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'], + [ + { $or: [{ test: "test" }, { test: { $ne: "test" } }] }, + 'test = "test" OR test <> "test"' + ], + [ + { test: { $gte: "test1", $ne: "test2" } }, + 'test >= "test1" AND test <> "test2"' + ] + ]; + + const shouldFailTests = [ + [{ test: { $gee: "test" } }, "Operator $gee not supported"], + [{ test: [] }, "Invalid type"], + [{ "Tags.test": { $eq: true } }, "Invalid tag query"], + [{ _tags: [] }, "Invalid type"], + [{ _tags: { $eq: "test" } }, "Invalid tag query"], + [{ $nor: [] }, "Operator $nor not supported"] + ]; + + t.plan(tests.length + 2 * shouldFailTests.length); + for (const test of tests) + t.is(stringify(mongodbFunctions.mongoQueryToFilter(test[0])), test[1]); + + for (const test of shouldFailTests) { + const func = (): void => { + mongodbFunctions.mongoQueryToFilter(test[0]); + }; + const error = t.throws(func, Error); + t.is(error.message, test[1]); + } +}); diff --git a/ui/filter-component.ts b/ui/filter-component.ts index 1ec989f..34b66b7 100644 --- a/ui/filter-component.ts +++ b/ui/filter-component.ts @@ -3,7 +3,7 @@ import { parse, stringify, map } from "../lib/common/expression-parser"; import memoize from "../lib/common/memoize"; import Autocomplete from "./autocomplete-compnent"; import * as smartQuery from "./smart-query"; -import { filterToMongoQuery } from "../lib/ui/mongodb-functions"; +import { filterToMongoQuery } from "../lib/mongodb-functions"; import { Expression } from "../lib/types"; const getAutocomplete = memoize(resource => {