From ea2e3707e392c6f602beaf6357bd0babecaacf25 Mon Sep 17 00:00:00 2001 From: Zaid Abdulla Date: Fri, 19 Jan 2018 16:08:30 -0800 Subject: [PATCH] Support value expressions in filter --- common/.eslintrc.json | 9 ++ common/filter-parser.js | 324 ++++++++++++++++++++++++++++++++++++++++ package.json | 2 +- server/api-functions.js | 28 +++- server/config.js | 2 + server/device.js | 18 +++ server/filter-parser.js | 207 ------------------------- 7 files changed, 378 insertions(+), 212 deletions(-) create mode 100644 common/.eslintrc.json create mode 100644 common/filter-parser.js delete mode 100644 server/filter-parser.js diff --git a/common/.eslintrc.json b/common/.eslintrc.json new file mode 100644 index 0000000..b46a7a5 --- /dev/null +++ b/common/.eslintrc.json @@ -0,0 +1,9 @@ +{ + "env": { + "node": true, + "browser": true + }, + "parserOptions": { + "ecmaVersion": 8 + } +} diff --git a/common/filter-parser.js b/common/filter-parser.js new file mode 100644 index 0000000..227436e --- /dev/null +++ b/common/filter-parser.js @@ -0,0 +1,324 @@ +"use strict"; + +const parsimmon = require("parsimmon"); + +// Turn escaped characters into real ones (e.g. "\\n" becomes "\n"). +function interpretEscapes(str) { + let escapes = { + b: "\b", + f: "\f", + n: "\n", + r: "\r", + t: "\t" + }; + return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => { + let type = escape.charAt(0); + let hex = escape.slice(1); + if (type === "u") return String.fromCharCode(parseInt(hex, 16)); + + if (escapes.hasOwnProperty(type)) return escapes[type]; + + return type; + }); +} + +function binaryLeft(operatorsParser, nextParser) { + return parsimmon.seqMap( + nextParser, + parsimmon.seq(operatorsParser, nextParser).many(), + (first, rest) => + rest.reduce((acc, ch) => { + let [op, another] = ch; + if (op === acc[0]) return acc.concat([another]); + if (op === another[0]) return [op, acc].concat(another.slice(1)); + return [op, acc, another]; + }, first) + ); +} + +const lang = parsimmon.createLanguage({ + ComparisonOperator: function() { + return parsimmon + .alt( + parsimmon.string(">="), + parsimmon.string("<>"), + parsimmon.string("<="), + parsimmon.string("="), + parsimmon.string(">"), + parsimmon.string("<") + ) + .skip(parsimmon.optWhitespace); + }, + IsNullOperator: function() { + return parsimmon + .alt( + parsimmon + .regexp(/is\s+null/i) + .result("IS NULL") + .desc("IS NULL"), + parsimmon + .regexp(/is\s+not\s+null/i) + .result("IS NOT NULL") + .desc("IS NOT NULL") + ) + .skip(parsimmon.optWhitespace); + }, + NotOperator: function() { + return parsimmon + .regexp(/not/i) + .result("NOT") + .skip(parsimmon.optWhitespace) + .desc("NOT"); + }, + AndOperator: function() { + return parsimmon + .regexp(/and/i) + .result("AND") + .skip(parsimmon.optWhitespace) + .desc("AND"); + }, + OrOperator: function() { + return parsimmon + .regexp(/or/i) + .result("OR") + .skip(parsimmon.optWhitespace) + .desc("OR"); + }, + Parameter: function(r) { + return parsimmon + .alt( + parsimmon.regexp(/[a-zA-Z0-9_.*]+/), + r.ValueExpression.wrap( + parsimmon.string("{").skip(parsimmon.optWhitespace), + parsimmon.string("}") + ) + ) + .atLeast(1) + .map(x => (x.length > 1 ? ["||"].concat(x) : x[0])) + .skip(parsimmon.optWhitespace) + .desc("parameter"); + }, + StringValueSql: function() { + return parsimmon + .regexp(/'([^']*)'/, 1) + .atLeast(1) + .skip(parsimmon.optWhitespace) + .map(s => s.join("'")) + .desc("string"); + }, + StringValueJs: function() { + return parsimmon + .regexp(/"((?:\\.|.)*?)"/, 1) + .skip(parsimmon.optWhitespace) + .map(interpretEscapes) + .desc("string"); + }, + NumberValue: function() { + return parsimmon + .regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/) + .skip(parsimmon.optWhitespace) + .map(Number) + .desc("number"); + }, + FuncValue: function(r) { + return parsimmon.seqMap( + parsimmon + .regexp(/([a-zA-Z0-0_]+)/, 1) + .skip(parsimmon.optWhitespace) + .desc("function"), + r.Value.sepBy(parsimmon.string(",").skip(parsimmon.optWhitespace)).wrap( + parsimmon.string("(").skip(parsimmon.optWhitespace), + parsimmon.string(")").skip(parsimmon.optWhitespace) + ), + (f, args) => ["FUNC", f.toUpperCase()].concat(args) + ); + }, + Value: function(r) { + return parsimmon.alt( + r.NumberValue, + r.StringValueSql, + r.StringValueJs, + r.FuncValue + ); + }, + ValueExpression: function(r) { + return binaryLeft( + parsimmon.string("||"), + binaryLeft( + parsimmon + .alt(parsimmon.string("+"), parsimmon.string("-")) + .skip(parsimmon.optWhitespace), + binaryLeft( + parsimmon + .alt(parsimmon.string("*"), parsimmon.string("/")) + .skip(parsimmon.optWhitespace), + parsimmon.alt( + r.Value, + r.ValueExpression.wrap( + parsimmon.string("(").skip(parsimmon.optWhitespace), + parsimmon.string(")").skip(parsimmon.optWhitespace) + ) + ) + ) + ) + ); + }, + Comparison: function(r) { + return parsimmon.alt( + parsimmon.seqMap(r.Parameter, r.IsNullOperator, (p, o) => [o, p]), + parsimmon.seqMap( + r.Parameter, + r.ComparisonOperator, + r.ValueExpression, + (p, o, v) => [o, p, v] + ) + ); + }, + Expression: function(r) { + function unary(operatorsParser, nextParser) { + return parsimmon.seq(operatorsParser, nextParser).or(nextParser); + } + + return binaryLeft( + r.OrOperator, + binaryLeft( + r.AndOperator, + unary( + r.NotOperator, + r.Comparison.or( + r.Expression.wrap( + parsimmon.string("("), + parsimmon.string(")").skip(parsimmon.optWhitespace) + ) + ) + ) + ) + ).trim(parsimmon.optWhitespace); + } +}); + +function parse(filter) { + return lang.Expression.tryParse(filter); +} + +function parseExpression(exp) { + return lang.ValueExpression.tryParse(exp); +} + +function parseParameter(p) { + return lang.Parameter.tryParse(p); +} + +function stringify(filter) { + function value(v) { + if (Array.isArray(v)) + if (v[0].toUpperCase() === "FUNC") { + return `${v[1]}(${v + .slice(2) + .map(e => value(e)) + .join(", ")})`; + } else { + return v + .slice(1) + .map(e => value(e)) + .join(` ${v[0]} `); + } + + return JSON.stringify(v); + } + + function expressions(v) { + return v.map(e => { + let str = stringify(e); + if (e[0] === "AND" || e[0] === "OR") return `(${str})`; + else return str; + }); + } + + function parameter(v) { + if (Array.isArray(v) && v[0] === "||") + return v + .slice(1) + .map(x => { + if (Array.isArray(x)) return `{${value(x)}}`; + else return x; + }) + .join(""); + + return v; + } + + const op = filter[0].toUpperCase(); + + if (["AND", "OR"].includes(op)) + return expressions(filter.slice(1)).join(` ${op} `); + else if (op === "NOT") return `NOT ${expressions(filter.slice(1))[0]}`; + else if ([">=", "<>", "<=", "=", ">", "<"].includes(op)) + return `${parameter(filter[1])} ${op} ${value(filter[2])}`; + else if (["IS NULL", "IS NOT NULL"].includes(op)) + return `${parameter(filter[1])} ${op}`; + else throw new Error("Unrecognized operator"); +} + +function map(filter, callback) { + let clone; + for (let i = 1; i < filter.length; ++i) + if (Array.isArray(filter[i])) { + let sub = map(filter[i], callback); + if (sub != null && sub !== filter[i]) { + clone = clone || filter.slice(); + clone[i] = sub; + } + } + + return callback(clone || filter) || clone || filter; +} + +function and(filter1, filter2) { + if (!filter1) return filter2; + else if (!filter2) return filter1; + + let res = ["AND"]; + + if (filter1[0] === "AND") res = res.concat(filter1.slice(1)); + else res.push(filter1); + + if (filter2[0] === "AND") res = res.concat(filter2.slice(1)); + else res.push(filter2); + + return res; +} + +function evaluateExpressions(filter, now = Date.now()) { + return map(filter, exp => { + if (exp[0] === "FUNC" && exp[1] === "NOW") { + return now; + } else if (exp[0] === "*") { + let v = exp[1]; + for (let i = 2; i < exp.length; ++i) v *= exp[i]; + return v; + } else if (exp[0] === "/") { + let v = exp[1]; + for (let i = 2; i < exp.length; ++i) v /= exp[i]; + return v; + } else if (exp[0] === "+") { + let v = exp[1]; + for (let i = 2; i < exp.length; ++i) v += exp[i]; + return v; + } else if (exp[0] === "-") { + let v = exp[1]; + for (let i = 2; i < exp.length; ++i) v -= exp[i]; + return v; + } else if (exp[0] === "||") { + return exp.slice(1).join(""); + } + }); +} + +exports.parse = parse; +exports.parseExpression = parseExpression; +exports.parseParameter = parseParameter; +exports.stringify = stringify; +exports.map = map; +exports.and = and; +exports.evaluateExpressions = evaluateExpressions; diff --git a/package.json b/package.json index e31be69..c175433 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", - "lint": "eslint --fix server/*.js client/*.js;prettier --write server/*.js client/*.js client/css/*.css", + "lint": "eslint --fix common/*.js server/*.js client/*.js;prettier --write common/*.js server/*.js client/*.js client/css/*.css", "build": "npm run lint;webpack;postcss -c postcss.config.js client/css/app.css -o public/app.css", "start": "node server/server" }, diff --git a/server/api-functions.js b/server/api-functions.js index 1a583ae..7549ef0 100644 --- a/server/api-functions.js +++ b/server/api-functions.js @@ -7,17 +7,29 @@ const querystring = require("querystring"); const config = require("./config"); const device = require("./device"); -const filterParser = require("./filter-parser"); +const filterParser = require("../common/filter-parser"); const commaAscii = ",".charCodeAt(0); const newLineAscii = "\n".charCodeAt(0); +function unpackTimestamps(filter) { + return filterParser.map(filter, exp => { + if (["=", "<>", ">=", "<=", "<", ">"].includes(exp[0])) + if (typeof exp[2] === "number") { + let alt = exp.slice(); + alt[2] = new Date(alt[2]).toJSON(); + return ["OR", exp, alt]; + } + }); +} + function filterToQuery(filter, negate = false, res = {}) { const op = filter[0]; if ((!negate && op === "AND") || (negate && op === "OR")) { + res["$and"] = res["$and"] || []; for (let i = 1; i < filter.length; ++i) - filterToQuery(filter[i], negate, res); + res["$and"].push(filterToQuery(filter[i], negate)); } else if ((!negate && op === "OR") || (negate && op === "AND")) { res["$or"] = res["$or"] || []; @@ -68,7 +80,11 @@ function count(resource, filter, limit) { return new Promise((resolve, reject) => { let qs = {}; if (filter) { - let q = filterToQuery(filterParser.parse(filter)); + let f = filterParser.parse(filter); + f = filterParser.evaluateExpressions(f); + f = unpackTimestamps(f); + + let q = filterToQuery(f); if (resource === "devices") q = device.transposeQuery(q); qs.query = JSON.stringify(q); } @@ -102,7 +118,11 @@ function query(resource, filter, limit, callback) { if (!callback) ret = []; let qs = {}; if (filter) { - let q = filterToQuery(filterParser.parse(filter)); + let f = filterParser.parse(filter); + f = filterParser.evaluateExpressions(f); + f = unpackTimestamps(f); + + let q = filterToQuery(f); if (resource === "devices") q = device.transposeQuery(q); qs.query = JSON.stringify(q); diff --git a/server/config.js b/server/config.js index 3a7809a..69dd68d 100644 --- a/server/config.js +++ b/server/config.js @@ -1,4 +1,6 @@ "use strict"; +const filterParser = require("../common/filter-parser.js"); + exports.PORT = 3000; exports.GENIEACS_NBI = "http://127.0.0.1:7557/"; diff --git a/server/device.js b/server/device.js index fd57a79..e426076 100644 --- a/server/device.js +++ b/server/device.js @@ -150,6 +150,8 @@ function transposeQuery(query) { newQuery["_deviceId._SerialNumber"] = val; } else if (key === "DeviceID.OUI") { newQuery["_deviceId._OUI"] = val; + } else if (key === "DeviceID.ProductClass") { + newQuery["_deviceId._ProductClass"] = val; } else if (key === "DeviceID.Manufacturer") { newQuery["_deviceId._Manufacturer"] = val; } else if (key.startsWith("Tags.")) { @@ -162,6 +164,22 @@ function transposeQuery(query) { newQuery["_tags"]["$nin"] = newQuery["_tags"]["$nin"] || []; newQuery["_tags"]["$nin"].push(key.slice(5)); } + } else if (key === "tag") { + if (typeof val === "string") { + newQuery["_tags"] = newQuery["_tags"] || {}; + newQuery["_tags"]["$all"] = newQuery["_tags"]["$all"] || []; + newQuery["_tags"]["$all"].push(val); + } else if (val["$eq"]) { + newQuery["_tags"] = newQuery["_tags"] || {}; + newQuery["_tags"]["$all"] = newQuery["_tags"]["$all"] || []; + newQuery["_tags"]["$all"].push(val["$eq"]); + } else if (val["$ne"]) { + newQuery["_tags"] = newQuery["_tags"] || {}; + newQuery["_tags"]["$nin"] = newQuery["_tags"]["$nin"] || []; + newQuery["_tags"]["$nin"].push(val["$ne"]); + } + } else if (key === "Events.Inform") { + newQuery["_lastInform"] = val; } else { newQuery[key] = val; } diff --git a/server/filter-parser.js b/server/filter-parser.js deleted file mode 100644 index 13a77a5..0000000 --- a/server/filter-parser.js +++ /dev/null @@ -1,207 +0,0 @@ -"use strict"; - -const parsimmon = require("parsimmon"); - -// Turn escaped characters into real ones (e.g. "\\n" becomes "\n"). -function interpretEscapes(str) { - let escapes = { - b: "\b", - f: "\f", - n: "\n", - r: "\r", - t: "\t" - }; - return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => { - let type = escape.charAt(0); - let hex = escape.slice(1); - if (type === "u") return String.fromCharCode(parseInt(hex, 16)); - - if (escapes.hasOwnProperty(type)) return escapes[type]; - - return type; - }); -} - -const lang = parsimmon.createLanguage({ - ComparisonOperator: function(r) { - return parsimmon - .alt( - parsimmon.string(">="), - parsimmon.string("<>"), - parsimmon.string("<="), - parsimmon.string("="), - parsimmon.string(">"), - parsimmon.string("<") - ) - .skip(r._); - }, - IsNullOperator: function(r) { - return parsimmon - .alt( - parsimmon - .regexp(/is\s+null/i) - .result("IS NULL") - .desc("IS NULL"), - parsimmon - .regexp(/is\s+not\s+null/i) - .result("IS NOT NULL") - .desc("IS NOT NULL") - ) - .skip(r._); - }, - NotOperator: function(r) { - return parsimmon - .regexp(/not/i) - .result("NOT") - .skip(r._) - .desc("NOT"); - }, - AndOperator: function(r) { - return parsimmon - .regexp(/and/i) - .result("AND") - .skip(r._) - .desc("AND"); - }, - OrOperator: function(r) { - return parsimmon - .regexp(/or/i) - .result("OR") - .skip(r._) - .desc("OR"); - }, - Parameter: function(r) { - return parsimmon - .regexp(/[a-zA-Z0-9_.*]+/) - .skip(r._) - .desc("parameter"); - }, - StringValueSql: function(r) { - return parsimmon - .regexp(/'([^']*)'/, 1) - .atLeast(1) - .skip(r._) - .map(s => s.join("'")) - .desc("string"); - }, - StringValueJs: function(r) { - return parsimmon - .regexp(/"((?:\\.|.)*?)"/, 1) - .skip(r._) - .map(interpretEscapes) - .desc("string"); - }, - NumberValue: function(r) { - return parsimmon - .regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/) - .skip(r._) - .map(Number) - .desc("number"); - }, - FuncValue: function(r) { - return parsimmon.seqMap( - parsimmon - .regexp(/([a-zA-Z0-0_]+)/, 1) - .skip(r._) - .desc("function"), - r.Value.sepBy(parsimmon.string(",").skip(r._)).wrap( - parsimmon.string("(").skip(r._), - parsimmon.string(")").skip(r._) - ), - (f, args) => ["FUNC", f.toUpperCase()].concat(args) - ); - }, - Value: function(r) { - return parsimmon.alt( - r.NumberValue, - r.StringValueSql, - r.StringValueJs, - r.FuncValue - ); - }, - Comparison: function(r) { - return parsimmon.alt( - parsimmon.seqMap(r.Parameter, r.IsNullOperator, (p, o) => [o, p]), - parsimmon.seqMap( - r.Parameter, - r.ComparisonOperator, - r.Value, - (p, o, v) => [o, p, v] - ) - ); - }, - Expression: function(r) { - function binaryLeft(operatorsParser, nextParser) { - return parsimmon.seqMap( - nextParser, - parsimmon.seq(operatorsParser, nextParser).many(), - (first, rest) => - rest.reduce((acc, ch) => { - let [op, another] = ch; - if (op === acc[0]) return acc.concat([another]); - if (op === another[0]) return [op, acc].concat(another.slice(1)); - return [op, acc, another]; - }, first) - ); - } - - function unary(operatorsParser, nextParser) { - return parsimmon.seq(operatorsParser, nextParser).or(nextParser); - } - - return binaryLeft( - r.OrOperator, - binaryLeft( - r.AndOperator, - unary( - r.NotOperator, - r.Comparison.or( - r.Expression.wrap( - parsimmon.string("("), - parsimmon.string(")").skip(r._) - ) - ) - ) - ) - ).trim(r._); - }, - _: function() { - return parsimmon.optWhitespace; - } -}); - -function parse(filter) { - return lang.Expression.tryParse(filter); -} - -function stringify(filter) { - function value(v) { - if (Array.isArray(v) && v[0].toUpperCase() === "FUNC") - return `${v[1]}(${v - .slice(2) - .map(e => value(e)) - .join(", ")})`; - return JSON.stringify(v); - } - - function expressions(v) { - return v.map(e => { - let str = stringify(e); - if (e[0] === "AND" || e[0] === "OR") return `(${str})`; - else return str; - }); - } - - const op = filter[0].toUpperCase(); - - if (["AND", "OR"].includes(op)) - return expressions(filter.slice(1)).join(` ${op} `); - else if (op === "NOT") return `NOT ${expressions(filter.slice(1))[0]}`; - else if ([">=", "<>", "<=", "=", ">", "<"].includes(op)) - return `${filter[1]} ${op} ${value(filter[2])}`; - else if (["IS NULL", "IS NOT NULL"].includes(op)) return `${filter[1]} ${op}`; - else throw new Error("Unrecognized operator"); -} - -exports.parse = parse; -exports.stringify = stringify;