From c2d13e836426cb79cc2357f8bf967a00ea0f2e00 Mon Sep 17 00:00:00 2001 From: Zaid Abdulla Date: Tue, 17 Jul 2018 00:37:09 -0700 Subject: [PATCH] Replace Filter class with expression structure This is in order to be able to support UPPER() and LOWER() functions in queries which are needed for case-insensitive search. --- client/app.js | 3 - client/components/container.js | 8 +- client/components/device-faults.js | 14 +- client/components/overview-dot.js | 8 +- client/components/parameter-list.js | 11 +- client/components/parameter-table.js | 16 +- client/components/summon-button.js | 11 +- client/device-page.js | 3 +- client/devices-page.js | 27 ++- client/drawer-component.js | 3 +- client/faults-page.js | 17 +- client/filter-component.js | 20 +- client/overview-page.js | 4 +- client/store.js | 120 +++++---- common/authorizer.js | 10 +- common/expression-cnf.js | 172 +++++++++++++ ...{filter-parser.js => expression-parser.js} | 227 +++++++++--------- common/expression.js | 188 +++++++++++++++ common/filter-cnf.js | 138 ----------- common/filter.js | 145 ----------- common/func-cache.js | 2 + server/api-functions.js | 83 ++++--- server/api.js | 42 ++-- server/db.js | 6 +- server/server.js | 16 +- 25 files changed, 716 insertions(+), 578 deletions(-) create mode 100644 common/expression-cnf.js rename common/{filter-parser.js => expression-parser.js} (58%) create mode 100644 common/expression.js delete mode 100644 common/filter-cnf.js delete mode 100644 common/filter.js diff --git a/client/app.js b/client/app.js index ae72780..fef3830 100644 --- a/client/app.js +++ b/client/app.js @@ -11,7 +11,6 @@ import * as devicePage from "./device-page"; import * as errorPage from "./error-page"; import * as faultsPage from "./faults-page"; import Authorizer from "../common/authorizer"; -import * as funcCache from "../common/func-cache"; window.authorizer = new Authorizer(window.permissionSets); @@ -69,8 +68,6 @@ function pagify(pageName, page) { return component; } -setInterval(funcCache.purge, 120000); - m.route(document.body, "/overview", { "/login": pagify("login", loginPage), "/overview": pagify("overview", overviewPage), diff --git a/client/components/container.js b/client/components/container.js index e907090..19603c3 100644 --- a/client/components/container.js +++ b/client/components/container.js @@ -3,16 +3,14 @@ import m from "mithril"; import * as components from "../components"; import * as store from "../store"; -import Filter from "../../common/filter"; +import * as expression from "../../common/expression"; import * as funcCache from "../../common/func-cache"; const component = { view: vnode => { if (vnode.attrs.filter) { - const filter = store.unpackFilter( - funcCache.get(Filter.parse, vnode.attrs.filter) - ); - if (!filter.test(vnode.attrs.device)) return; + const filter = funcCache.get(expression.parse, vnode.attrs.filter); + if (!store.evaluateExpression(filter, vnode.attrs.device)) return; } let children = Object.values(vnode.attrs.components).map(c => { diff --git a/client/components/device-faults.js b/client/components/device-faults.js index a2ce634..74145bb 100644 --- a/client/components/device-faults.js +++ b/client/components/device-faults.js @@ -2,21 +2,17 @@ import m from "mithril"; import * as store from "../store"; -import Filter from "../../common/filter"; import * as notifications from "../notifications"; const component = { view: vnode => { const device = vnode.attrs.device; const deviceId = device["DeviceID.ID"].value[0]; - const faults = store.fetch( - "faults", - new Filter([ - "AND", - [">", "_id", `${deviceId}:`], - ["<", "_id", `${deviceId}:zzzz`] - ]) - ); + const faults = store.fetch("faults", [ + "AND", + [">", ["PARAM", "_id"], `${deviceId}:`], + ["<", ["PARAM", "_id"], `${deviceId}:zzzz`] + ]); const headers = ["Channel", "Code", "Message", "Retries", "Timestamp"].map( l => m("th", l) diff --git a/client/components/overview-dot.js b/client/components/overview-dot.js index e306be8..918e927 100644 --- a/client/components/overview-dot.js +++ b/client/components/overview-dot.js @@ -4,7 +4,7 @@ import m from "mithril"; import config from "../config"; import * as store from "../store"; import * as funcCache from "../../common/func-cache"; -import Filter from "../../common/filter"; +import * as expression from "../../common/expression"; const CHARTS = config.ui.overview.charts; @@ -13,10 +13,8 @@ const component = { const device = vnode.attrs.device; const chart = CHARTS[vnode.attrs.chart]; for (let slice of Object.values(chart.slices)) { - const filter = store.unpackFilter( - funcCache.get(Filter.parse, slice.filter) - ); - if (filter.test(device)) { + const filter = funcCache.get(expression.parse, slice.filter); + if (store.evaluateExpression(filter, device)) { const dot = m( "svg", { diff --git a/client/components/parameter-list.js b/client/components/parameter-list.js index 9636851..77a9e31 100644 --- a/client/components/parameter-list.js +++ b/client/components/parameter-list.js @@ -3,14 +3,21 @@ import m from "mithril"; import * as components from "../components"; import * as store from "../store"; -import * as filterParser from "../../common/filter-parser"; +import * as funcCache from "../../common/func-cache"; +import * as expression from "../../common/expression"; + +const parseParameter = funcCache.getter(p => { + p = expression.parse(p); + if (Array.isArray(p) && p[0] === "PARAM") p = p[1]; + return p; +}); const component = { oninit: vnode => { vnode.state.parameters = Object.values(vnode.attrs.parameters).map( parameter => Object.assign({}, parameter, { - parameter: filterParser.parseParameter(parameter.parameter) + parameter: parseParameter(parameter.parameter) }) ); }, diff --git a/client/components/parameter-table.js b/client/components/parameter-table.js index be5d286..84bb939 100644 --- a/client/components/parameter-table.js +++ b/client/components/parameter-table.js @@ -4,15 +4,23 @@ import m from "mithril"; import * as components from "../components"; import * as taskQueue from "../task-queue"; import * as store from "../store"; -import * as filterParser from "../../common/filter-parser"; +import * as expression from "../../common/expression"; +import * as funcCache from "../../common/func-cache"; + +const parseParameter = funcCache.getter(p => { + p = expression.parse(p); + if (Array.isArray(p) && p[0] === "PARAM") p = p[1]; + return p; +}); const component = { oninit: vnode => { - vnode.state.object = filterParser.parseParameter(vnode.attrs.parameter); + vnode.state.object = parseParameter(vnode.attrs.parameter); vnode.state.parameters = Object.values(vnode.attrs.childParameters).map( parameter => { - let p = filterParser.parseParameter(parameter.parameter); - return Object.assign({}, parameter, { parameter: p }); + return Object.assign({}, parameter, { + parameter: parseParameter(parameter.parameter) + }); } ); }, diff --git a/client/components/summon-button.js b/client/components/summon-button.js index 850b8cb..f379049 100644 --- a/client/components/summon-button.js +++ b/client/components/summon-button.js @@ -4,12 +4,19 @@ import m from "mithril"; import * as taskQueue from "../task-queue"; import * as store from "../store"; import * as notifications from "../notifications"; -import * as filterParser from "../../common/filter-parser"; +import * as funcCache from "../../common/func-cache"; +import * as expression from "../../common/expression"; + +const parseParameter = funcCache.getter(p => { + p = expression.parse(p); + if (Array.isArray(p) && p[0] === "PARAM") p = p[1]; + return p; +}); const component = { oninit: vnode => { vnode.state.parameters = Object.values(vnode.attrs.parameters).map( - parameter => filterParser.parseParameter(parameter) + parameter => parseParameter(parameter) ); }, view: vnode => { diff --git a/client/device-page.js b/client/device-page.js index a42a045..907c6cd 100644 --- a/client/device-page.js +++ b/client/device-page.js @@ -3,7 +3,6 @@ import m from "mithril"; import config from "./config"; -import Filter from "../common/filter"; import * as store from "./store"; import * as components from "./components"; @@ -15,7 +14,7 @@ function init(args) { return Promise.resolve({ deviceId: args.id, - deviceFilter: new Filter(["=", "DeviceID.ID", args.id]) + deviceFilter: ["=", ["PARAM", "DeviceID.ID"], args.id] }); } diff --git a/client/devices-page.js b/client/devices-page.js index 777e947..a508715 100644 --- a/client/devices-page.js +++ b/client/devices-page.js @@ -4,24 +4,24 @@ import m from "mithril"; import config from "./config"; import filterComponent from "./filter-component"; -import * as filterParser from "../common/filter-parser"; -import Filter from "../common/filter"; import * as store from "./store"; import * as components from "./components"; import * as taskQueue from "./task-queue"; import * as notifications from "./notifications"; +import * as expression from "../common/expression"; function init(args) { return new Promise((resolve, reject) => { if (!window.authorizer.hasAccess("devices", 2)) return reject(new Error("You are not authorized to view this page")); - let filter = new Filter(args.filter); - let indexParameters = Object.values(config.ui.index).map(p => - Object.assign({}, p, { - parameter: filterParser.parseParameter(p.parameter) - }) - ); + let filter = null; + if (args.filter != null) filter = expression.parse(`${args.filter}`); + let indexParameters = Object.values(config.ui.index).map(p => { + let param = expression.parse(p.parameter); + if (Array.isArray(param) && param[0] === "PARAM") param = param[1]; + return Object.assign({}, p, { parameter: param }); + }); resolve({ filter: filter, indexParameters: indexParameters }); }); } @@ -281,16 +281,19 @@ const component = { function onFilterChanged(filter) { let params = {}; - if (filter) params.filter = filterParser.stringify(filter); + if (filter != null) params.filter = expression.stringify(filter); m.route.set("/devices", params); } let devs = store.fetch( "devices", - vnode.attrs.filter, + vnode.attrs.filter == null ? true : vnode.attrs.filter, vnode.state.showCount || 10 ); - let count = store.count("devices", vnode.attrs.filter); + let count = store.count( + "devices", + vnode.attrs.filter == null ? true : vnode.attrs.filter + ); let selected = new Set(); if (vnode.state.selected) @@ -303,7 +306,7 @@ const component = { m("h1", "Listing devices"), m(filterComponent, { predefined: Object.values(config.ui.filters), - filter: vnode.attrs.filter ? vnode.attrs.filter.ast : null, + filter: vnode.attrs.filter, onChange: onFilterChanged }), renderTable( diff --git a/client/drawer-component.js b/client/drawer-component.js index 1631d74..6bb44bc 100644 --- a/client/drawer-component.js +++ b/client/drawer-component.js @@ -5,7 +5,6 @@ import m from "mithril"; import * as taskQueue from "./task-queue"; import * as store from "./store"; import * as notifications from "./notifications"; -import Filter from "../common/filter"; function mparam(param) { return m("span.parameter", { title: param }, `${param}`); @@ -41,7 +40,7 @@ function renderStagingSpv(task, queueFunc, cancelFunc) { function renderStagingDownload(task) { task.invalid = !task.fileName || !task.fileType; - const files = store.fetch("files", new Filter()); + const files = store.fetch("files", true); const idParts = task.device.split("-"); const oui = decodeURIComponent(idParts[0]); const productClass = diff --git a/client/faults-page.js b/client/faults-page.js index 8faf397..6653d8e 100644 --- a/client/faults-page.js +++ b/client/faults-page.js @@ -3,10 +3,9 @@ import m from "mithril"; import filterComponent from "./filter-component"; -import * as filterParser from "../common/filter-parser"; -import Filter from "../common/filter"; import * as store from "./store"; import * as notifications from "./notifications"; +import * as expression from "../common/expression"; function init(args) { if (!window.authorizer.hasAccess("faults", 2)) @@ -14,7 +13,8 @@ function init(args) { new Error("You are not authorized to view this page") ); - let filter = new Filter(args.filter); + let filter = null; + if (args.filter != null) filter = expression.parse(`${args.filter}`); return Promise.resolve({ filter: filter }); } @@ -153,16 +153,19 @@ const component = { function onFilterChanged(filter) { let params = {}; - if (filter) params.filter = filterParser.stringify(filter); + if (filter != null) params.filter = expression.stringify(filter); m.route.set("/faults", params); } let faults = store.fetch( "faults", - vnode.attrs.filter, + vnode.attrs.filter == null ? true : vnode.attrs.filter, vnode.state.showCount || 10 ); - let count = store.count("faults", vnode.attrs.filter); + let count = store.count( + "faults", + vnode.attrs.filter == null ? true : vnode.attrs.filter + ); let selected = new Set(); if (vnode.state.selected) @@ -180,7 +183,7 @@ const component = { { parameter: "retries" }, { parameter: "timestamp" } ], - filter: vnode.attrs.filter ? vnode.attrs.filter.ast : null, + filter: vnode.attrs.filter, onChange: onFilterChanged }), renderTable(faults, count.value, selected, showMore) diff --git a/client/filter-component.js b/client/filter-component.js index e362da4..34c4df7 100644 --- a/client/filter-component.js +++ b/client/filter-component.js @@ -2,7 +2,7 @@ import m from "mithril"; -import * as filterParser from "../common/filter-parser"; +import * as expression from "../common/expression"; const component = { view: vnode => { @@ -15,16 +15,14 @@ const component = { if (vnode.attrs.filter !== vnode.state.filter) { vnode.state.filterList = []; vnode.state.filter = vnode.attrs.filter; - if (vnode.attrs.filter) + if (vnode.attrs.filter != null) if (vnode.attrs.filter[0] === "AND") for (let i = 1; i < vnode.attrs.filter.length; ++i) vnode.state.filterList.push( - filterParser.stringify(vnode.attrs.filter[i]) + expression.stringify(vnode.attrs.filter[i]) ); else - vnode.state.filterList.push( - filterParser.stringify(vnode.attrs.filter) - ); + vnode.state.filterList.push(expression.stringify(vnode.attrs.filter)); vnode.state.filterList.push(""); } @@ -33,18 +31,18 @@ const component = { vnode.state.filterList = vnode.state.filterList.filter( f => f && f.trim() ); - let filterAst = vnode.state.filterList.map((f, idx) => { + let filter = vnode.state.filterList.map((f, idx) => { try { - return filterParser.parse(f); + return expression.parse(f); } catch (err) { vnode.state.filterInvalid |= 1 << idx; } }); - if (filterAst.length) filterAst = ["AND"].concat(filterAst); - else filterAst = null; + if (filter.length) filter = ["AND"].concat(filter); + else filter = null; vnode.state.filterList.push(""); - if (!vnode.state.filterInvalid) vnode.attrs.onChange(filterAst); + if (!vnode.state.filterInvalid) vnode.attrs.onChange(filter); } return m( diff --git a/client/overview-page.js b/client/overview-page.js index 362902a..302cc50 100644 --- a/client/overview-page.js +++ b/client/overview-page.js @@ -4,8 +4,8 @@ import m from "mithril"; import config from "./config"; import * as store from "./store"; import pieChartComponent from "./pie-chart-component"; -import Filter from "../common/filter"; import * as funcCache from "../common/func-cache"; +import * as expression from "../common/expression"; const GROUPS = config.ui.overview.groups; const CHARTS = {}; @@ -19,7 +19,7 @@ function queryCharts(charts) { charts[chartName] = chart = Object.assign({}, chart); chart.slices = Object.assign({}, chart.slices); for (let [sliceName, slice] of Object.entries(chart.slices)) { - const filter = funcCache.get(Filter.parse, slice.filter); + const filter = funcCache.get(expression.parse, slice.filter); chart.slices[sliceName] = slice = Object.assign({}, slice); slice.count = store.count("devices", filter); } diff --git a/client/store.js b/client/store.js index ac8435f..02ba2c3 100644 --- a/client/store.js +++ b/client/store.js @@ -1,15 +1,15 @@ "use strict"; import m from "mithril"; -import Filter from "../common/filter"; import config from "./config"; -import * as filterParser from "../common/filter-parser"; +import * as expressionParser from "../common/expression-parser"; +import * as expression from "../common/expression"; import * as funcCache from "../common/func-cache"; let fulfillTimestamp = 0; -let unpackedFiltersCache = new WeakMap(); -let expressionsCache = new WeakMap(); +let unpackExpressionCache = new WeakMap(); +let evaluateExpressionCache = new WeakMap(); const queries = { filter: new WeakMap(), @@ -47,46 +47,54 @@ class QueryResponse { } } -function unpackFilter(filter) { - let f = unpackedFiltersCache.get(filter); - if (!f) { - f = f.evaluateExpressions(fulfillTimestamp); - unpackedFiltersCache.set(filter, f); +function unpackExpression(exp) { + if (!Array.isArray(exp)) return exp; + let e = unpackExpressionCache.get(exp); + if (e === undefined) { + e = expression.evaluate(e, null, fulfillTimestamp); + unpackExpressionCache.set(exp, e); } - return f; + return e; } function count(resourceType, filter) { - let queryResponse = resources[resourceType].count.get(filter.toString()); + const filterStr = funcCache.get(expression.stringify, filter); + let queryResponse = resources[resourceType].count.get(filterStr); if (queryResponse) return queryResponse; queryResponse = new QueryResponse(); - resources[resourceType].count.set(filter.toString(), queryResponse); + resources[resourceType].count.set(filterStr, queryResponse); queries.filter.set(queryResponse, filter); return queryResponse; } function limitFilter(resourceType, filter, last) { if (resourceType === "devices") - return filter.and(new Filter(["<=", "DeviceID.ID", last])); - else return filter.and(new Filter(["<=", "_id", last])); + return expression.and(filter, ["<=", "DeviceID.ID", last]); + else return expression.and(filter, ["<=", ["PARAM", "_id"], last]); } function findMatches(resourceType, filter, limit) { // Handle "tag =" and "tag <>" special cases - if (resourceType === "devices" && filter.ast) { - const ast = filterParser.map(filter.ast, e => { - if (e[1] === "tag") - if (e[0] === "=") return ["=", `Tags.${e[2]}`, true]; - else if (e[0] === "<>") return ["NOT", ["=", `Tags.${e[2]}`, true]]; + if (resourceType === "devices") + filter = expressionParser.map(filter, e => { + if ( + Array.isArray(e) && + Array.isArray(e[1]) && + e[1][0] === "PARAM" && + e[1][0] === "tag" + ) + if (e[0] === "=") return ["=", ["PARAM", `Tags.${e[2]}`], true]; + else if (e[0] === "<>") + return ["NOT", ["=", ["PARAM", `Tags.${e[2]}`], true]]; + return e; }); - filter = new Filter(ast); - } let value = []; for (let [id, obj] of resources[resourceType].objects.entries()) - if (filter.test(obj)) value.push(id); + if (expression.evaluate(filter, obj, fulfillTimestamp)) + value.push(id); value = value.sort(); if (limit) value = value.slice(0, limit); @@ -97,13 +105,13 @@ function findMatches(resourceType, filter, limit) { function inferQuery(resourceType, queryResponse) { const limit = queries.limit.get(queryResponse); let filter = queries.filter.get(queryResponse); - filter = unpackFilter(filter); + filter = unpackExpression(filter); let last = queries.last.get(queryResponse); if (last || !limit) { if (last) filter = limitFilter(resourceType, filter, last); if ( resources[resourceType].combinedFilter && - filter.subset(resources[resourceType].combinedFilter) + expression.subset(filter, resources[resourceType].combinedFilter) ) queries.fulfilled.set(queryResponse, fulfillTimestamp); } @@ -112,7 +120,8 @@ function inferQuery(resourceType, queryResponse) { } function fetch(resourceType, filter, limit = 0) { - const key = `${limit}:${filter.toString()}`; + const filterStr = funcCache.get(expression.stringify, filter); + const key = `${limit}:${filterStr}`; let queryResponse = resources[resourceType].fetch.get(key); if (queryResponse) return queryResponse; @@ -131,8 +140,8 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { let updated = false; if (_fulfillTimestamp > fulfillTimestamp) { - unpackedFiltersCache = new WeakMap(); - expressionsCache = new WeakMap(); + unpackExpressionCache = new WeakMap(); + evaluateExpressionCache = new WeakMap(); for (let resource of Object.values(resources)) resource.combinedFilter = null; @@ -156,14 +165,14 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { new Promise((resolve, reject) => { updated = true; let filter = queries.filter.get(queryResponse); - filter = unpackFilter(filter); + filter = unpackExpression(filter); m .request({ method: "HEAD", url: `/api/${resourceType}/?` + m.buildQueryString({ - filter: filter.toString() + filter: funcCache.get(expression.stringify, filter) }), extract: xhr => +xhr.getResponseHeader("x-total-count"), background: true @@ -201,14 +210,14 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { new Promise((resolve, reject) => { updated = true; let filter = queries.filter.get(queryResponse); - filter = unpackFilter(filter); + filter = unpackExpression(filter); m .request({ method: "GET", url: `/api/${resourceType}/?` + m.buildQueryString({ - filter: filter.toString(), + filter: funcCache.get(expression.stringify, filter), limit: 1, skip: limit - 1, projection: "_id" @@ -240,13 +249,13 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { toFetch = toFetch.filter(queryResponse => { let filter = queries.filter.get(queryResponse); - filter = unpackFilter(filter); + filter = unpackExpression(filter); let last = queries.last.get(queryResponse); if (last) filter = limitFilter(resourceType, filter, last); if ( resources[resourceType].combinedFilter && - filter.subset(resources[resourceType].combinedFilter) + expression.subset(filter, resources[resourceType].combinedFilter) ) { queries.value.set( queryResponse, @@ -257,11 +266,11 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { return false; } - combinedFilter = filter.or(combinedFilter); + combinedFilter = expression.or(combinedFilter, filter); return true; }); - if (!combinedFilter) return resolve(updated); + if (combinedFilter == null) continue; updated = true; let deleted = new Set(); @@ -269,10 +278,12 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { deleted = new Set(resources[resourceType].objects.keys()); let combinedFilterDiff = combinedFilter; if (resources[resourceType].combinedFilter) - combinedFilterDiff = combinedFilterDiff.and( - resources[resourceType].combinedFilter.not() + combinedFilterDiff = expression.and( + combinedFilterDiff, + expression.not(resources[resourceType].combinedFilter) ); - resources[resourceType].combinedFilter = combinedFilter.or( + resources[resourceType].combinedFilter = expression.or( + combinedFilter, resources[resourceType].combinedFilter ); @@ -284,7 +295,10 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { url: `/api/${resourceType}/?` + m.buildQueryString({ - filter: combinedFilterDiff.toString() + filter: funcCache.get( + expression.stringify, + combinedFilterDiff + ) }) }) .then(res => { @@ -299,13 +313,19 @@ function fulfill(accessTimestamp, _fulfillTimestamp) { for (let d of deleted) { const obj = resources[resourceType].objects.get(d); - if (combinedFilterDiff.test(obj)) + if ( + expression.evaluate( + combinedFilterDiff, + obj, + fulfillTimestamp + ) + ) resources[resourceType].objects.delete(d); } for (let queryResponse of toFetch) { let filter = queries.filter.get(queryResponse); - filter = unpackFilter(filter); + filter = unpackExpression(filter); let last = queries.last.get(queryResponse); if (last) filter = limitFilter(resourceType, filter, last); @@ -379,20 +399,18 @@ function deleteResource(resourceType, id) { }); } -function evaluateExpression(exp, device) { +function evaluateExpression(exp, obj) { if (!Array.isArray(exp)) return exp; - let exps = expressionsCache.get(exp); - if (!exps) expressionsCache.set(exp, (exps = new WeakMap())); + let exps = evaluateExpressionCache.get(exp); + if (!exps) evaluateExpressionCache.set(exp, (exps = new WeakMap())); - if (!exps.has(device)) { - let v = filterParser.evaluateExpressions(exp, e => { - if (e[0] === "FUNC" && e[1] === "NOW") return fulfillTimestamp; - }); - exps.set(device, v); + if (!exps.has(obj)) { + let v = expression.evaluate(exp, obj, fulfillTimestamp); + exps.set(obj, v); } - return exps.get(device); + return exps.get(obj); } function logIn(username, password) { @@ -421,7 +439,7 @@ export { count, fetch, fulfill, - unpackFilter, + unpackExpression, getTimestamp, postTasks, updateTags, diff --git a/common/authorizer.js b/common/authorizer.js index b7bae2c..03e0122 100644 --- a/common/authorizer.js +++ b/common/authorizer.js @@ -1,5 +1,7 @@ "use strict"; +const expression = require("./expression"); + const validators = { pass: require("./validators/pass"), test: require("./validators/test") @@ -18,15 +20,13 @@ class Authorizer { return this.hasAccessCache[cacheKey]; let has = false; - for (const permissionSet of this.permissionSets) { - for (const perm of permissionSet) { - if (perm[resourceType]) { + for (let permissionSet of this.permissionSets) { + for (let perm of permissionSet) + if (perm[resourceType]) if (perm[resourceType].access >= access) { has = true; break; } - } - } } this.hasAccessCache[cacheKey] = has; diff --git a/common/expression-cnf.js b/common/expression-cnf.js new file mode 100644 index 0000000..d2ea2ce --- /dev/null +++ b/common/expression-cnf.js @@ -0,0 +1,172 @@ +"use strict"; + +function collectRanges(exp, ranges = new Map()) { + if (["AND", "OR", "NOT"].includes(exp[0])) + for (let i = 1; i < exp.length; ++i) collectRanges(exp[i], ranges); + else if ([">", ">=", "<", "<=", "=", "<>"].includes(exp[0])) + if (!Array.isArray(exp[2])) { + const key = `${JSON.stringify(exp[1])}:${typeof exp[2]}`; + let s = ranges.get(key); + if (!s) ranges.set(key, (s = new Set())); + s.add(exp[2]); + } else if (!Array.isArray(exp[1])) { + const key = `${JSON.stringify(exp[2])}:${typeof exp[1]}`; + let s = ranges.get(key); + if (!s) ranges.set(key, (s = new Set())); + s.add(exp[2]); + } + + return ranges; +} + +function or(cnf1, cnf2) { + if (cnf1.length === 0) return cnf2; + else if (cnf2.length === 0) return cnf1; + + const res = []; + for (let i = 0; i < cnf1.length; ++i) + for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j])); + + return res; +} + +function booleanCnf(exp) { + const ranges = collectRanges(exp); + for (let [k, v] of ranges) + if (k.endsWith(":number")) + ranges.set(k, Array.from(v).sort((a, b) => a - b)); + else ranges.set(k, Array.from(v).sort()); + + const expressions = new Map(); + const mutuallyExclusive = new Map(); + + function getVariable(lhs, op, rhs = "") { + const key = `${lhs}:${op}:${rhs}`; + let f = expressions.get(key); + if (!f) { + expressions.set(key, (f = expressions.size + 1)); + let m = mutuallyExclusive.get(lhs); + if (!m) mutuallyExclusive.set(lhs, (m = new Set())); + m.add(f); + if (rhs || key !== "null") m.add(getVariable(lhs, "null")); + } + return f; + } + + function getComparisonVariables(lhs, from, fromInclusive, to, toInclusive) { + const type = from != null ? typeof from : typeof to; + const range = ranges.get(`${lhs}:${type}`); + const vars = []; + + if (from == null) vars.push(`${lhs}:--${range[0]}`); + if (to == null) vars.push(`${lhs}:${range[range.length - 1]}--`); + if (fromInclusive) vars.push(`${lhs}:${from}`); + if (toInclusive) vars.push(`${lhs}:${to}`); + + for (let i = 1; i < range.length; ++i) + if ((from == null || range[i] > from) && (to == null || range[i] <= to)) { + vars.push(`${lhs}:${range[i - 1]}`); + vars.push(`${lhs}:${range[i]}`); + vars.push(`${lhs}:${range[i - 1]}--${range[i]}`); + } + + return vars.map(key => { + let f = expressions.get(key); + if (!f) { + expressions.set(key, (f = expressions.size + 1)); + let m = mutuallyExclusive.get(lhs); + if (!m) mutuallyExclusive.set(lhs, (m = new Set())); + m.add(f); + } + return f; + }); + } + + const comparisonOperators = { + "=": v => [v, true, v, true], + ">": v => [v, false, null, null], + ">=": v => [v, true, null, null], + "<": v => [null, null, v, false], + "<=": v => [null, null, v, true] + }; + + function recursive(clause, negate) { + let op = clause[0]; + + if ((op === "AND" && !negate) || (op === "OR" && negate)) { + let res = []; + for (let i = 1; i < clause.length; ++i) + res = res.concat(recursive(clause[i], negate)); + return res; + } else if ((op === "OR" && !negate) || (op === "AND" && negate)) { + let res = []; + for (let i = 1; i < clause.length; ++i) + res = or(res, recursive(clause[i], negate)); + return res; + } else if (op === "NOT") { + return recursive(clause[1], !negate); + } else if (op === "<>") { + const vars = getComparisonVariables( + JSON.stringify(clause[1]), + ...comparisonOperators["="](clause[2]) + ); + if (!negate) return vars.map(x => [0 - x]); + else return [vars]; + } else if (op === "IS NULL") { + const v = getVariable(JSON.stringify(clause[1]), "null"); + if (negate) return [[0 - v]]; + else return [[v]]; + } else if (op === "IS NOT NULL") { + const v = getVariable(JSON.stringify(clause[1]), "null"); + if (!negate) return [[0 - v]]; + else return [[v]]; + } else if (op in comparisonOperators) { + if (!Array.isArray(clause[2])) { + const vars = getComparisonVariables( + JSON.stringify(clause[1]), + ...comparisonOperators[op](clause[2]) + ); + if (negate) return vars.map(x => [0 - x]); + else return [vars]; + } else if (!Array.isArray(clause[1])) { + op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op; + const vars = getComparisonVariables( + JSON.stringify(clause[2]), + ...comparisonOperators[op](clause[1]) + ); + if (negate) return vars.map(x => [0 - x]); + else return [vars]; + } else { + let lhs = JSON.stringify(clause[1]); + let rhs = JSON.stringify(clause[2]); + // For consistency + if (rhs > lhs) { + let t = rhs; + rhs = lhs; + lhs = t; + op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op; + } + + const v = getVariable(lhs, op, rhs); + if (negate) return [[0 - v]]; + else return [[v]]; + } + } else { + const v = getVariable(JSON.stringify(clause), ""); + if (negate) return [[0 - v]]; + else return [[v]]; + } + } + + const cnf = recursive(exp, false); + + for (let m of mutuallyExclusive.values()) { + let ar = Array.from(m); + for (let i = 0; i < ar.length; ++i) + for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]); + } + + return { vars: expressions.size, clauses: cnf }; +} + +exports.booleanCnf = booleanCnf; diff --git a/common/filter-parser.js b/common/expression-parser.js similarity index 58% rename from common/filter-parser.js rename to common/expression-parser.js index 3063de3..db235cb 100644 --- a/common/filter-parser.js +++ b/common/expression-parser.js @@ -22,6 +22,21 @@ function interpretEscapes(str) { }); } +function map(exp, callback) { + if (!Array.isArray(exp)) return callback(exp); + + let clone; + for (let i = 1; i < exp.length; ++i) { + let sub = map(exp[i], callback); + if (sub !== exp[i]) { + clone = clone || exp.slice(); + clone[i] = sub; + } + } + + return callback(clone || exp); +} + function binaryLeft(operatorsParser, nextParser) { return parsimmon.seqMap( nextParser, @@ -29,8 +44,9 @@ function binaryLeft(operatorsParser, nextParser) { (first, rest) => rest.reduce((acc, ch) => { let [op, another] = ch; - if (op === acc[0]) return acc.concat([another]); - if (op === another[0]) return [op, acc].concat(another.slice(1)); + if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]); + if (Array.isArray(another) && op === another[0]) + return [op, acc].concat(another.slice(1)); return [op, acc, another]; }, first) ); @@ -61,12 +77,14 @@ const lang = parsimmon.createLanguage({ .result("IS NOT NULL") .desc("IS NOT NULL") ) + .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/)) .skip(parsimmon.optWhitespace); }, NotOperator: function() { return parsimmon .regexp(/not/i) .result("NOT") + .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/)) .skip(parsimmon.optWhitespace) .desc("NOT"); }, @@ -74,6 +92,7 @@ const lang = parsimmon.createLanguage({ return parsimmon .regexp(/and/i) .result("AND") + .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/)) .skip(parsimmon.optWhitespace) .desc("AND"); }, @@ -81,6 +100,7 @@ const lang = parsimmon.createLanguage({ return parsimmon .regexp(/or/i) .result("OR") + .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/)) .skip(parsimmon.optWhitespace) .desc("OR"); }, @@ -88,13 +108,13 @@ const lang = parsimmon.createLanguage({ return parsimmon .alt( parsimmon.regexp(/[a-zA-Z0-9_.*]+/), - r.ValueExpression.wrap( + r.Expression.wrap( parsimmon.string("{").skip(parsimmon.optWhitespace), parsimmon.string("}") ) ) .atLeast(1) - .map(x => (x.length > 1 ? ["||"].concat(x) : x[0])) + .map(x => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]]) .skip(parsimmon.optWhitespace) .desc("parameter"); }, @@ -120,13 +140,38 @@ const lang = parsimmon.createLanguage({ .map(Number) .desc("number"); }, + BooleanValue: function() { + return parsimmon + .alt( + parsimmon + .regexp(/true/i) + .result(true) + .desc("TRUE"), + parsimmon + .regexp(/false/i) + .result(false) + .desc("FALSE") + ) + .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/)) + .skip(parsimmon.optWhitespace); + }, + NullValue: function() { + return parsimmon + .regexp(/null/i) + .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/)) + .skip(parsimmon.optWhitespace) + .result(null) + .desc("NULL"); + }, FuncValue: function(r) { return parsimmon.seqMap( parsimmon - .regexp(/([a-zA-Z0-0_]+)/, 1) + .regexp(/([a-zA-Z0-9_]+)/, 1) .skip(parsimmon.optWhitespace) .desc("function"), - r.Value.sepBy(parsimmon.string(",").skip(parsimmon.optWhitespace)).wrap( + r.ValueExpression.sepBy( + parsimmon.string(",").skip(parsimmon.optWhitespace) + ).wrap( parsimmon.string("(").skip(parsimmon.optWhitespace), parsimmon.string(")").skip(parsimmon.optWhitespace) ), @@ -135,6 +180,8 @@ const lang = parsimmon.createLanguage({ }, Value: function(r) { return parsimmon.alt( + r.NullValue, + r.BooleanValue, r.NumberValue, r.StringValueSql, r.StringValueJs, @@ -143,7 +190,7 @@ const lang = parsimmon.createLanguage({ }, ValueExpression: function(r) { return binaryLeft( - parsimmon.string("||"), + parsimmon.string("||").skip(parsimmon.optWhitespace), binaryLeft( parsimmon .alt(parsimmon.string("+"), parsimmon.string("-")) @@ -154,7 +201,8 @@ const lang = parsimmon.createLanguage({ .skip(parsimmon.optWhitespace), parsimmon.alt( r.Value, - r.ValueExpression.wrap( + r.Parameter, + r.Expression.wrap( parsimmon.string("(").skip(parsimmon.optWhitespace), parsimmon.string(")").skip(parsimmon.optWhitespace) ) @@ -167,7 +215,7 @@ const lang = parsimmon.createLanguage({ return parsimmon.alt( parsimmon.seqMap(r.Parameter, r.IsNullOperator, (p, o) => [o, p]), parsimmon.seqMap( - r.Parameter, + r.ValueExpression, r.ComparisonOperator, r.ValueExpression, (p, o, v) => [o, p, v] @@ -183,129 +231,78 @@ const lang = parsimmon.createLanguage({ r.OrOperator, binaryLeft( r.AndOperator, - unary( - r.NotOperator, - r.Comparison.or( - r.Expression.wrap( - parsimmon.string("("), - parsimmon.string(")").skip(parsimmon.optWhitespace) - ) - ) - ) + unary(r.NotOperator, r.Comparison.or(r.ValueExpression)) ) ).trim(parsimmon.optWhitespace); } }); -function parse(filter) { - return lang.Expression.tryParse(filter); +function parse(str) { + if (!str) return null; + return lang.Expression.tryParse(str); } -function parseExpression(exp) { - return lang.ValueExpression.tryParse(exp); -} +function stringify(exp) { + if (!Array.isArray(exp)) return JSON.stringify(exp); -function evaluateExpressions(ast, funcCallback) { - return map(ast, exp => { - if (exp[0] === "FUNC") { - return funcCallback(exp); - } else if (exp[0] === "*") { - let v = exp[1]; - for (let i = 2; i < exp.length; ++i) v *= exp[i]; - return v; - } else if (exp[0] === "/") { - let v = exp[1]; - for (let i = 2; i < exp.length; ++i) v /= exp[i]; - return v; - } else if (exp[0] === "+") { - let v = exp[1]; - for (let i = 2; i < exp.length; ++i) v += exp[i]; - return v; - } else if (exp[0] === "-") { - let v = exp[1]; - for (let i = 2; i < exp.length; ++i) v -= exp[i]; - return v; - } else if (exp[0] === "||") { - return exp.slice(1).join(""); - } - }); -} + const opLevels = { + OR: 10, + AND: 11, + NOT: 12, + "=": 20, + "<>": 21, + ">": 22, + ">=": 23, + "<": 24, + "<=": 25, + "||": 30, + "+": 31, + "-": 31, + "*": 32, + "/": 32 + }; -function parseParameter(p) { - return lang.Parameter.tryParse(p); -} + const op = exp[0].toUpperCase(); -function stringify(filter) { - function value(v) { - if (Array.isArray(v)) - if (v[0].toUpperCase() === "FUNC") { - return `${v[1]}(${v - .slice(2) - .map(e => value(e)) - .join(", ")})`; - } else { - return v - .slice(1) - .map(e => value(e)) - .join(` ${v[0]} `); - } - - return JSON.stringify(v); - } - - function expressions(v) { - return v.map(e => { - let str = stringify(e); - if (e[0] === "AND" || e[0] === "OR") return `(${str})`; - else return str; - }); - } - - function parameter(v) { - if (Array.isArray(v) && v[0] === "||") - return v + if (op === "FUNC") { + return `${exp[1]}(${exp + .slice(2) + .map(e => stringify(e)) + .join(", ")})`; + } else if (op === "PARAM") { + if (typeof exp[1] === "string") return exp[1]; + else if (Array.isArray(exp[1]) && exp[1][0] === "||") + return exp[1] .slice(1) - .map(x => { - if (Array.isArray(x)) return `{${value(x)}}`; - else return x; + .map(p => { + if (typeof p === "string") return p; + else return `{${stringify(p)}}`; }) .join(""); + else return `{${stringify(exp[1])}}`; + } else if (op === "IS NULL" || op === "IS NOT NULL") { + return `${stringify(exp[1])} ${op}`; + } else if (op in opLevels) { + const parts = exp.slice(1).map((e, i) => { + if ( + Array.isArray(e) && + (opLevels[e[0]] < opLevels[exp[0]] || + (opLevels[e[0]] < opLevels[exp[0]] && i > 0) || + (opLevels[exp[0]] >= 20 && + opLevels[e[0]] >= 20 && + opLevels[e[0]] < 30)) + ) + return `(${stringify(e)})`; + else return stringify(e); + }); - return v; + if (op === "NOT") return `${op} ${parts[0]}`; + else return parts.join(` ${op} `); + } else { + throw new Error(`Unrecognized operator ${exp[0]}`); } - - const op = filter[0].toUpperCase(); - - if (["AND", "OR"].includes(op)) - return expressions(filter.slice(1)).join(` ${op} `); - else if (op === "NOT") return `NOT ${expressions(filter.slice(1))[0]}`; - else if ([">=", "<>", "<=", "=", ">", "<"].includes(op)) - return `${parameter(filter[1])} ${op} ${value(filter[2])}`; - else if (["IS NULL", "IS NOT NULL"].includes(op)) - return `${parameter(filter[1])} ${op}`; - else throw new Error("Unrecognized operator"); -} - -function map(filter, callback) { - let clone; - for (let i = 1; i < filter.length; ++i) - if (Array.isArray(filter[i])) { - let sub = map(filter[i], callback); - if (sub != null && sub !== filter[i]) { - clone = clone || filter.slice(); - clone[i] = sub; - } - } - - let r = callback(clone || filter); - if (r != null) return r; - - return clone || filter; } exports.parse = parse; -exports.parseExpression = parseExpression; -exports.evaluateExpressions = evaluateExpressions; -exports.parseParameter = parseParameter; exports.stringify = stringify; exports.map = map; diff --git a/common/expression.js b/common/expression.js new file mode 100644 index 0000000..14e6c53 --- /dev/null +++ b/common/expression.js @@ -0,0 +1,188 @@ +"use strict"; + +const expressionParser = require("./expression-parser"); +const expressionCnf = require("./expression-cnf"); +const satSolver = require("./sat-solver"); + +const isArray = Array.isArray; + +function* permute(arr) { + if (arr.length <= 1) { + for (let i = 0; i < arr[0]; ++i) yield [i]; + return; + } + + let slc = arr.slice(1); + for (let i = 0; i < arr[0]; ++i) + for (let innerArr of permute(slc)) yield [i].concat(innerArr); +} + +function reduce(exp, callback) { + let loop = true; + while (loop) { + loop = false; + for (let i = 2; i < exp.length; ++i) { + let res = callback(exp[i - 1], exp[i]); + if (res !== undefined) { + loop = true; + exp = exp.slice(); + exp.splice(i - 1, 2, res); + } + } + } + if (exp.length === 2) return exp[1]; + return exp; +} + +function evaluate(exp, obj, now) { + return expressionParser.map(exp, e => { + if (!isArray(e)) return e; + + if (e[0] === "FUNC") { + if (e[1] === "NOW") { + if (now) return now; + } else if (e[1] === "UPPER") { + if (e[2] == null) return null; + if (!isArray(e[2])) return `${e[2]}`.toUpperCase(); + } else if (e[1] === "LOWER") { + if (e[12] == null) return null; + if (!isArray(e[2])) return `${e[2]}`.toLowerCase(); + } + } else if (e[0] === "PARAM") { + if (e[1] == null) return null; + if (obj && !isArray(e[1])) + if (typeof obj[e[1]] === "object") + return obj[e[1]].value ? obj[e[1]].value[0] : null; + else return obj[e[1]]; + } else if (e[0] === "AND") { + return reduce(e, (a, b) => { + if (!isArray(a)) return a ? b : a; + if (!isArray(b)) return b ? a : b; + }); + } else if (e[0] === "OR") { + return reduce(e, (a, b) => { + if (!isArray(a)) return a ? a : b; + if (!isArray(b)) return b ? b : a; + }); + } else if (e[0] === "NOT") { + if (!isArray(e[1])) return !e[1]; + else if (e[1][0] === "NOT") return e[1][1]; + } else if (e[0] === "IS NULL") { + if (isArray(e[1])) return e; + return e[1] == null; + } else if (e[0] === "IS NOT NULL") { + if (isArray(e[1])) return e; + return e[1] != null; + } else if (e[0] === "=") { + if (isArray(e[1]) || isArray(e[2])) return e; + if (e[1] == null || e[2] == null) return null; + return e[1] === e[2]; + } else if (e[0] === "<>") { + if (isArray(e[1]) || isArray(e[2])) return e; + if (e[1] == null || e[2] == null) return null; + return e[1] !== e[2]; + } else if (e[0] === ">") { + if (isArray(e[1]) || isArray(e[2])) return e; + if (e[1] == null || e[2] == null) return null; + return e[1] > e[2]; + } else if (e[0] === ">=") { + if (isArray(e[1]) || isArray(e[2])) return e; + if (e[1] == null || e[2] == null) return null; + return e[1] >= e[2]; + } else if (e[0] === "<") { + if (isArray(e[1]) || isArray(e[2])) return e; + if (e[1] == null || e[2] == null) return null; + return e[1] < e[2]; + } else if (e[0] === "<=") { + if (isArray(e[1]) || isArray(e[2])) return e; + if (e[1] == null || e[2] == null) return null; + return e[1] <= e[2]; + } else if (e[0] === "*") { + return reduce(e, (a, b) => { + if (!isArray(a) && !isArray(b)) { + if (a == null || b == null) return null; + return a * b; + } + }); + } else if (e[0] === "/") { + return reduce(e, (a, b) => { + if (!isArray(a) && !isArray(b)) { + if (a == null || b == null) return null; + return a / b; + } + }); + } else if (e[0] === "+") { + return reduce(e, (a, b) => { + if (!isArray(a) && !isArray(b)) { + if (a == null || b == null) return null; + return a + b; + } + }); + } else if (e[0] === "-") { + return reduce(e, (a, b) => { + if (!isArray(a) && !isArray(b)) { + if (a == null || b == null) return null; + return a - b; + } + }); + } else if (e[0] === "||") { + return reduce(e, (a, b) => { + if (!isArray(a) && !isArray(b)) { + if (a == null || b == null) return null; + return `${a}${b}`; + } + }); + } + return e; + }); +} + +function and(exp1, exp2) { + if (!isArray(exp1)) return exp1 ? exp2 : exp1; + if (!isArray(exp2)) return exp2 ? exp1 : exp2; + + let res = ["AND"]; + + if (exp1[0] === "AND") res = res.concat(exp1.slice(1)); + else res.push(exp1); + + if (exp2[0] === "AND") res = res.concat(exp2.slice(1)); + else res.push(exp2); + + return res; +} + +function or(exp1, exp2) { + if (!isArray(exp1)) return exp1 ? exp1 : exp2; + if (!isArray(exp2)) return exp2 ? exp2 : exp1; + + let res = ["OR"]; + + if (exp1[0] === "OR") res = res.concat(exp1.slice(1)); + else res.push(exp1); + + if (exp2[0] === "OR") res = res.concat(exp2.slice(1)); + else res.push(exp2); + + return res; +} + +function not(exp) { + if (isArray(exp) && exp[0] === "NOT") return exp[1]; + return ["NOT", exp]; +} + +function subset(exp1, exp2) { + const e = evaluate(["NOT", ["OR", ["NOT", exp1], exp2]]); + if (!isArray(e)) return !e; + const { vars, clauses } = expressionCnf.booleanCnf(e); + return !satSolver.naiveDpll(clauses, vars); +} + +exports.evaluate = evaluate; +exports.and = and; +exports.or = or; +exports.not = not; +exports.subset = subset; +exports.parse = expressionParser.parse; +exports.stringify = expressionParser.stringify; diff --git a/common/filter-cnf.js b/common/filter-cnf.js deleted file mode 100644 index 20da585..0000000 --- a/common/filter-cnf.js +++ /dev/null @@ -1,138 +0,0 @@ -"use strict"; - -function collectRanges(filter, ranges = new Map()) { - if (["AND", "OR", "NOT"].includes(filter[0])) { - for (let i = 1; i < filter.length; ++i) collectRanges(filter[i], ranges); - } else if ([">", ">=", "<", "<=", "=", "<>"].includes(filter[0])) { - const key = `${filter[1]}:${typeof filter[2]}`; - let s = ranges.get(key); - if (!s) ranges.set(key, (s = new Set())); - s.add(filter[2]); - } - return ranges; -} - -function or(cnf1, cnf2) { - if (cnf1.length === 0) return cnf2; - else if (cnf2.length === 0) return cnf1; - - const res = []; - for (let i = 0; i < cnf1.length; ++i) - for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j])); - - return res; -} - -function booleanCnf(filter) { - const ranges = collectRanges(filter); - for (let [k, v] of ranges) - if (k.endsWith(":number")) - ranges.set(k, Array.from(v).sort((a, b) => a - b)); - else ranges.set(k, Array.from(v).sort()); - - const expressions = new Map(); - const mutuallyExclusive = new Map(); - - function getIsNullVariable(param) { - const key = `${param}:null`; - let f = expressions.get(key); - if (!f) { - expressions.set(key, (f = expressions.size + 1)); - let m = mutuallyExclusive.get(param); - if (!m) mutuallyExclusive.set(param, (m = new Set())); - m.add(f); - } - return f; - } - - function getComparisonVariables(param, from, fromInclusive, to, toInclusive) { - const type = from != null ? typeof from : typeof to; - const range = ranges.get(`${param}:${type}`); - const vars = []; - - if (from == null) vars.push(`${param}:--${range[0]}`); - if (to == null) vars.push(`${param}:${range[range.length - 1]}--`); - if (fromInclusive) vars.push(`${param}:${from}`); - if (toInclusive) vars.push(`${param}:${to}`); - - for (let i = 1; i < range.length; ++i) - if ((from == null || range[i] > from) && (to == null || range[i] <= to)) { - vars.push(`${param}:${range[i - 1]}`); - vars.push(`${param}:${range[i]}`); - vars.push(`${param}:${range[i - 1]}--${range[i]}`); - } - - return vars.map(key => { - let f = expressions.get(key); - if (!f) { - expressions.set(key, (f = expressions.size + 1)); - let m = mutuallyExclusive.get(param); - if (!m) mutuallyExclusive.set(param, (m = new Set())); - m.add(f); - } - return f; - }); - } - - const comparisonOperators = { - "=": v => [v, true, v, true], - ">": v => [v, false, null, null], - ">=": v => [v, true, null, null], - "<": v => [null, null, v, false], - "<=": v => [null, null, v, true] - }; - - function recursive(clause, negate) { - const op = clause[0]; - - if ((op === "AND" && !negate) || (op === "OR" && negate)) { - let res = []; - for (let i = 1; i < clause.length; ++i) - res = res.concat(recursive(clause[i], negate)); - return res; - } else if ((op === "OR" && !negate) || (op === "AND" && negate)) { - let res = []; - for (let i = 1; i < clause.length; ++i) - res = or(res, recursive(clause[i], negate)); - return res; - } else if (op === "NOT") { - return recursive(clause[1], !negate); - } else if (op in comparisonOperators) { - const vars = getComparisonVariables( - clause[1], - ...comparisonOperators[op](clause[2]) - ); - if (negate) return vars.map(x => [0 - x]); - else return [vars]; - } else if (op === "<>") { - const vars = getComparisonVariables( - clause[1], - ...comparisonOperators["="](clause[2]) - ); - if (!negate) return vars.map(x => [0 - x]); - else return [vars]; - } else if (op === "IS NULL") { - const v = getIsNullVariable(clause[1]); - if (negate) return [[0 - v]]; - else return [[v]]; - } else if (op === "IS NOT NULL") { - const v = getIsNullVariable(clause[1]); - if (!negate) return [[0 - v]]; - else return [[v]]; - } else { - throw new Error(`Unknown operator ${op}`); - } - } - - const cnf = recursive(filter, false); - - for (let m of mutuallyExclusive.values()) { - let ar = Array.from(m); - for (let i = 0; i < ar.length; ++i) - for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]); - } - - return { vars: expressions.size, clauses: cnf }; -} - -exports.booleanCnf = booleanCnf; diff --git a/common/filter.js b/common/filter.js deleted file mode 100644 index 24788d1..0000000 --- a/common/filter.js +++ /dev/null @@ -1,145 +0,0 @@ -"use strict"; - -const filterParser = require("./filter-parser"); -const filterCnf = require("./filter-cnf"); -const satSolver = require("./sat-solver"); -const funcCache = require("./func-cache"); - -function* permute(arr) { - if (arr.length <= 1) { - for (let i = 0; i < arr[0]; ++i) yield [i]; - return; - } - - let slc = arr.slice(1); - for (let i = 0; i < arr[0]; ++i) - for (let innerArr of permute(slc)) yield [i].concat(innerArr); -} - -class Filter { - constructor(str) { - if (!str) this.ast = null; - else if (Array.isArray(str)) this.ast = str; - else if (str instanceof Filter) this.ast = str.ast; - else if (Array.isArray(str.ast) || str.ast === null) this.ast = str.ast; - else this.ast = filterParser.parse(str); - } - - static parse(str) { - return new Filter(str); - } - - toString() { - if (!this.string) - if (this.ast) this.string = filterParser.stringify(this.ast); - else this.string = ""; - - return this.string; - } - - or(fltr) { - if (!fltr || !this.ast) return this; - else if (!fltr.ast) return fltr; - - let f = ["OR"]; - - if (this.ast[0] === "OR") f = f.concat(this.ast.slice(1)); - else f.push(this.ast); - - if (fltr.ast[0] === "OR") f = f.concat(fltr.ast.slice(1)); - else f.push(fltr.ast); - - return new Filter(f); - } - - and(fltr) { - if (!fltr || !fltr.ast) return this; - else if (!this.ast) return fltr; - - let f = ["AND"]; - - if (this.ast[0] === "AND") f = f.concat(this.ast.slice(1)); - else f.push(this.ast); - - if (fltr.ast[0] === "AND") f = f.concat(fltr.ast.slice(1)); - else f.push(fltr.ast); - - return new Filter(f); - } - - not() { - if (this.ast[0] === "NOT") return new Filter(this.ast[1]); - else return new Filter(["NOT", this.ast]); - } - - evaluateExpressions(now = Date.now()) { - if (!this.ast) return this; - - let ast = filterParser.evaluateExpressions(this.ast, exp => { - if (exp[0] === "FUNC" && exp[1] === "NOW") return now; - }); - - if (ast === this.ast) return this; - else return new Filter(ast); - } - - subset(fltr) { - if (!fltr.ast) return true; - else if (!this.ast) return false; - - const f = this.not() - .or(fltr) - .not(); - const { vars, clauses } = filterCnf.booleanCnf(f.ast); - return !satSolver.naiveDpll(clauses, vars); - } - - test(obj) { - if (!this.ast) return true; - return filterParser.map(this.ast, exp => { - const op = exp[0]; - - if (op === "AND") { - for (let i = 1; i < exp.length; ++i) if (!exp[i]) return false; - return true; - } else if (op === "OR") { - for (let i = 1; i < exp.length; ++i) if (exp[i]) return true; - return false; - } else if (op === "NOT") { - return !exp[1]; - } else if (op === "=") { - const val = - typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]]; - return val === exp[2]; - } else if (op === "<>") { - const val = - typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]]; - return val !== exp[2]; - } else if (op === ">=") { - const val = - typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]]; - return typeof val === typeof exp[2] && val >= exp[2]; - } else if (op === ">") { - const val = - typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]]; - return typeof val === typeof exp[2] && val > exp[2]; - } else if (op === "<=") { - const val = - typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]]; - return typeof val === typeof exp[2] && val <= exp[2]; - } else if (op === "<") { - const val = - typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]]; - return typeof val === typeof exp[2] && val < exp[2]; - } else if (op === "IS NULL") { - return obj[exp[1]] == null; - } else if (op === "IS NOT NULL") { - return obj[exp[1]] != null; - } else { - throw new Error(`Unsupported operator ${op}`); - } - }); - } -} - -module.exports = Filter; diff --git a/common/func-cache.js b/common/func-cache.js index 837f01d..4c9f081 100644 --- a/common/func-cache.js +++ b/common/func-cache.js @@ -22,6 +22,8 @@ function purge() { cache1 = new WeakMap(); } +setInterval(purge, 120000); + exports.get = get; exports.getter = getter; exports.purge = purge; diff --git a/server/api-functions.js b/server/api-functions.js index c10c476..e41cd92 100644 --- a/server/api-functions.js +++ b/server/api-functions.js @@ -7,20 +7,22 @@ const querystring = require("querystring"); const config = require("./config"); const device = require("./device"); -const filterParser = require("../common/filter-parser"); -const Filter = require("../common/filter"); +const expressionParser = require("../common/expression-parser"); +const expression = require("../common/expression"); const commaAscii = ",".charCodeAt(0); const newLineAscii = "\n".charCodeAt(0); function unpackTimestamps(filter) { - return filterParser.map(filter, exp => { + if (!Array.isArray(filter)) return filter; + return expressionParser.map(filter, exp => { if (["=", "<>", ">=", "<=", "<", ">"].includes(exp[0])) - if (typeof exp[2] === "number") { + if (Array.isArray(exp) && typeof exp[2] === "number") { let alt = exp.slice(); alt[2] = new Date(alt[2]).toJSON(); return ["OR", exp, alt]; } + return exp; }); } @@ -39,37 +41,42 @@ function filterToMongoQuery(filter, negate = false, res = {}) { } else if (op === "NOT") { filterToMongoQuery(filter[1], !negate, res); } else if (op === "=") { - let p = (res[filter[1]] = res[filter[1]] || {}); + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); if (negate) p["$ne"] = filter[2]; else p["$eq"] = filter[2]; } else if (op === "<>") { - let p = (res[filter[1]] = res[filter[1]] || {}); - if (negate) { - p["$eq"] = filter[2]; - } else { - p["$ne"] = filter[2]; - p["$exists"] = true; - } + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); + if (negate) p = p["$not"] = p["$not"] || {}; + p["$ne"] = filter[2]; + p["$exists"] = true; } else if (op === ">") { - let p = (res[filter[1]] = res[filter[1]] || {}); + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); if (negate) p = p["$not"] = p["$not"] || {}; p["$gt"] = filter[2]; } else if (op === ">=") { - let p = (res[filter[1]] = res[filter[1]] || {}); + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); if (negate) p = p["$not"] = p["$not"] || {}; p["$gte"] = filter[2]; } else if (op === "<") { - let p = (res[filter[1]] = res[filter[1]] || {}); + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); if (negate) p = p["$not"] = p["$not"] || {}; p["$lt"] = filter[2]; } else if (op === "<=") { - let p = (res[filter[1]] = res[filter[1]] || {}); + const param = filter[1][1]; + let p = (res[param] = res[param] || {}); if (negate) p = p["$not"] = p["$not"] || {}; p["$lte"] = filter[2]; } else if (op === "IS NULL") { - res[filter[1]] = { $exists: negate }; + const param = filter[1][1]; + res[param] = { $exists: negate }; } else if (op === "IS NOT NULL") { - res[filter[1]] = { $exists: !negate }; + const param = filter[1][1]; + res[param] = { $exists: !negate }; } else { throw new Error(`Unrecognized operator ${op}`); } @@ -80,14 +87,15 @@ function filterToMongoQuery(filter, negate = false, res = {}) { function count(resource, filter, limit) { return new Promise((resolve, reject) => { let qs = {}; - if (filter && filter.ast) { - filter = filter.evaluateExpressions(); - let ast = unpackTimestamps(filter.ast); + filter = expression.evaluate(filter, null, Date.now()); + filter = unpackTimestamps(filter); - let q = filterToMongoQuery(ast); - if (resource === "devices") q = device.transposeQuery(q); - qs.query = JSON.stringify(q); - } + let q = {}; + if (Array.isArray(filter)) q = filterToMongoQuery(filter); + else if (filter != null && !filter) return resolve(0); + + if (resource === "devices") q = device.transposeQuery(q); + qs.query = JSON.stringify(q); if (limit) qs.limit = limit; @@ -117,15 +125,15 @@ function query(resource, filter, limit, skip, projection, callback) { let ret; if (!callback) ret = []; let qs = {}; - if (filter && filter.ast) { - filter = filter.evaluateExpressions(); - let ast = unpackTimestamps(filter.ast); + filter = expression.evaluate(filter, null, Date.now()); + filter = unpackTimestamps(filter); - let q = filterToMongoQuery(ast); - if (resource === "devices") q = device.transposeQuery(q); + let q = {}; + if (Array.isArray(filter)) q = filterToMongoQuery(filter); + else if (filter != null && !filter) return resolve(ret); - qs.query = JSON.stringify(q); - } + if (resource === "devices") q = device.transposeQuery(q); + qs.query = JSON.stringify(q); if (limit) qs.limit = limit; if (skip) qs.skip = skip; @@ -308,12 +316,13 @@ function postTasks(deviceId, tasks) { let promises2 = []; for (let s of statuses) { - promises2.push(query("tasks", new Filter(["=", "_id", s._id]))); + promises2.push(query("tasks", ["=", ["PARAM", "_id"], s._id])); promises2.push( - query( - "faults", - new Filter(["=", "_id", `${deviceId}:task_${s._id}`]) - ) + query("faults", [ + "=", + ["PARAM", "_id"], + `${deviceId}:task_${s._id}` + ]) ); } diff --git a/server/api.js b/server/api.js index 587ca9c..37ec195 100644 --- a/server/api.js +++ b/server/api.js @@ -4,7 +4,7 @@ const Router = require("koa-router"); const db = require("./db"); const apiFunctions = require("./api-functions"); -const Filter = require("../common/filter"); +const expression = require("../common/expression"); const router = new Router(); @@ -33,17 +33,20 @@ const resources = { for (let [resource, flags] of Object.entries(resources)) { router.head(`/${resource}`, async (ctx, next) => { - let filter, limit; - if (ctx.request.query.filter) filter = new Filter(ctx.request.query.filter); + let filter = true; + let limit; + if (ctx.request.query.filter) + filter = expression.parse(ctx.request.query.filter); if (ctx.request.query.limit) limit = +ctx.request.query.limit; if (!ctx.state.authorizer.hasAccess(resource, 1)) return next(); // Exclude temporary tasks and faults - if (resource === "tasks" || resource === "faults") { - let f = new Filter(["NOT", ["<", "expiry", Date.now() + 60000]]); - filter = f.and(filter); - } + if (resource === "tasks" || resource === "faults") + filter = expression.and(filter, [ + "NOT", + ["<", ["PARAM", "expiry"], Date.now() + 60000] + ]); let count; if (flags & RESOURCE_DB) count = await db.count(resource, filter, limit); @@ -54,8 +57,10 @@ for (let [resource, flags] of Object.entries(resources)) { }); router.get(`/${resource}`, async (ctx, next) => { - let filter, limit, skip, projection; - if (ctx.request.query.filter) filter = new Filter(ctx.request.query.filter); + let filter = true; + let limit, skip, projection; + if (ctx.request.query.filter) + filter = expression.parse(ctx.request.query.filter); if (ctx.request.query.limit) limit = +ctx.request.query.limit; if (ctx.request.query.skip) skip = +ctx.request.query.skip; if (ctx.request.query.projection) projection = ctx.request.query.projection; @@ -63,10 +68,11 @@ for (let [resource, flags] of Object.entries(resources)) { if (!ctx.state.authorizer.hasAccess(resource, 2)) return next(); // Exclude temporary tasks and faults - if (resource === "tasks" || resource === "faults") { - let f = new Filter(["NOT", ["<", "expiry", Date.now() + 60000]]); - filter = f.and(filter); - } + if (resource === "tasks" || resource === "faults") + filter = expression.and(filter, [ + "NOT", + ["<", ["PARAM", "expiry"], Date.now() + 60000] + ]); let res; if (flags & RESOURCE_DB) @@ -78,7 +84,7 @@ for (let [resource, flags] of Object.entries(resources)) { }); router.head(`/${resource}/:id`, async (ctx, next) => { - let filter = new Filter(`${RESOURCE_IDS[resource]} = "${ctx.params.id}"`); + let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id]; if (!ctx.state.authorizer.hasAccess(resource, 2)) return next(); let res; @@ -90,7 +96,7 @@ for (let [resource, flags] of Object.entries(resources)) { }); router.get(`/${resource}/:id`, async (ctx, next) => { - let filter = new Filter(`${RESOURCE_IDS[resource]} = "${ctx.params.id}"`); + let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id]; if (!ctx.state.authorizer.hasAccess(resource, 2)) return next(); let res; @@ -104,7 +110,7 @@ for (let [resource, flags] of Object.entries(resources)) { if (flags & RESOURCE_DELETE && !(flags & RESOURCE_DB)) router.delete(`/${resource}/:id`, async (ctx, next) => { const authorizer = ctx.state.authorizer; - let filter = new Filter(`${RESOURCE_IDS[resource]} = "${ctx.params.id}"`); + let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id]; if (!authorizer.hasAccess(resource, 2)) return next(); let res = await apiFunctions.query(resource, filter); if (!res.length) return next(); @@ -119,7 +125,7 @@ for (let [resource, flags] of Object.entries(resources)) { router.post("/devices/:id/tasks", async (ctx, next) => { const authorizer = ctx.state.authorizer; - let filter = new Filter(`DeviceID.ID = "${ctx.params.id}"`); + let filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id]; if (!authorizer.hasAccess("devices", 2)) return next(); let devices = await apiFunctions.query("devices", filter); if (!devices.length) return next(); @@ -135,7 +141,7 @@ router.post("/devices/:id/tasks", async (ctx, next) => { router.post("/devices/:id/tags", async (ctx, next) => { const authorizer = ctx.state.authorizer; - let filter = new Filter(`DeviceID.ID = "${ctx.params.id}"`); + let filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id]; if (!authorizer.hasAccess("devices", 2)) return next(); let res = await apiFunctions.query("devices", filter); if (!res.length) return next(); diff --git a/server/db.js b/server/db.js index 088c7d6..5147001 100644 --- a/server/db.js +++ b/server/db.js @@ -20,7 +20,8 @@ function getClient() { function query(resource, filter, limit, skip, callback) { let q; - if (filter && filter.ast) q = apiFunctions.filterToMongoQuery(filter.ast); + if (Array.isArray(filter)) q = apiFunctions.filterToMongoQuery(filter); + else if (filter != null && !filter) return Promise.resolve([]); return new Promise((resolve, reject) => { getClient().then(client => { @@ -50,7 +51,8 @@ function query(resource, filter, limit, skip, callback) { function count(resource, filter) { let q; - if (filter && filter.ast) q = apiFunctions.filterToMongoQuery(filter.ast); + if (Array.isArray(filter)) q = apiFunctions.filterToMongoQuery(filter); + else if (filter != null && !filter) return Promise.resolve([]); return new Promise((resolve, reject) => { getClient().then(client => { diff --git a/server/server.js b/server/server.js index bf5bdfd..62b64ad 100644 --- a/server/server.js +++ b/server/server.js @@ -11,6 +11,7 @@ const jwt = require("jsonwebtoken"); const config = require("./config"); const api = require("./api"); const Authorizer = require("../common/authorizer"); +const expression = require("../common/expression"); const koa = new Koa(); const router = new Router(); @@ -20,7 +21,20 @@ const JWT_COOKIE = "genieacs-ui-jwt"; function getPermissionSets(roles) { const allPermissions = config.permissions; - const permissionSets = roles.map(r => Object.values(allPermissions[r] || {})); + const permissionSets = roles.map(role => { + return Object.values(allPermissions[role] || {}).map(p => { + p = Object.assign({}, p); + for (let [k, v] of Object.entries(p)) { + p[k] = v = Object.assign({}, v); + if (["devices", "faults", "files"].includes(k)) + if (v.filter == null || v.filter === "") v.filter = true; + else { + v.filter = expression.parse(v.filter); + } + } + return p; + }); + }); return permissionSets; }