rodrigo/genieacs
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace Filter class with expression structure

Browse Source 
This is in order to be able to support UPPER() and LOWER() functions in
queries which are needed for case-insensitive search.
This commit is contained in:
Zaid Abdulla committed 2018-12-26 00:22:07 -08:00
1 parent 7194fcab44
commit c2d13e8364
25 files changed
+716 -578
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

client/app.js
-3
View File
@@ -11,7 +11,6 @@ import * as devicePage from "./device-page";
import * as errorPage from "./error-page";
import * as faultsPage from "./faults-page";
import Authorizer from "../common/authorizer";
import * as funcCache from "../common/func-cache";
window.authorizer = new Authorizer(window.permissionSets);
@@ -69,8 +68,6 @@ function pagify(pageName, page) {
return component;
}
setInterval(funcCache.purge, 120000);
m.route(document.body, "/overview", {
"/login": pagify("login", loginPage),
"/overview": pagify("overview", overviewPage),
client/components/container.js
+3 -5
View File
@@ -3,16 +3,14 @@
import m from "mithril";
import * as components from "../components";
import * as store from "../store";
import Filter from "../../common/filter";
import * as expression from "../../common/expression";
import * as funcCache from "../../common/func-cache";
const component = {
view: vnode => {
if (vnode.attrs.filter) {
const filter = store.unpackFilter(
funcCache.get(Filter.parse, vnode.attrs.filter)
);
if (!filter.test(vnode.attrs.device)) return;
const filter = funcCache.get(expression.parse, vnode.attrs.filter);
if (!store.evaluateExpression(filter, vnode.attrs.device)) return;
}
let children = Object.values(vnode.attrs.components).map(c => {
client/components/device-faults.js
+5 -9
View File
@@ -2,21 +2,17 @@
import m from "mithril";
import * as store from "../store";
import Filter from "../../common/filter";
import * as notifications from "../notifications";
const component = {
view: vnode => {
const device = vnode.attrs.device;
const deviceId = device["DeviceID.ID"].value[0];
const faults = store.fetch(
"faults",
new Filter([
"AND",
[">", "_id", `${deviceId}:`],
["<", "_id", `${deviceId}:zzzz`]
])
);
const faults = store.fetch("faults", [
"AND",
[">", ["PARAM", "_id"], `${deviceId}:`],
["<", ["PARAM", "_id"], `${deviceId}:zzzz`]
]);
const headers = ["Channel", "Code", "Message", "Retries", "Timestamp"].map(
l => m("th", l)
client/components/overview-dot.js
+3 -5
View File
@@ -4,7 +4,7 @@ import m from "mithril";
import config from "../config";
import * as store from "../store";
import * as funcCache from "../../common/func-cache";
import Filter from "../../common/filter";
import * as expression from "../../common/expression";
const CHARTS = config.ui.overview.charts;
@@ -13,10 +13,8 @@ const component = {
const device = vnode.attrs.device;
const chart = CHARTS[vnode.attrs.chart];
for (let slice of Object.values(chart.slices)) {
const filter = store.unpackFilter(
funcCache.get(Filter.parse, slice.filter)
);
if (filter.test(device)) {
const filter = funcCache.get(expression.parse, slice.filter);
if (store.evaluateExpression(filter, device)) {
const dot = m(
"svg",
{
client/components/parameter-list.js
+9 -2
View File
@@ -3,14 +3,21 @@
import m from "mithril";
import * as components from "../components";
import * as store from "../store";
import * as filterParser from "../../common/filter-parser";
import * as funcCache from "../../common/func-cache";
import * as expression from "../../common/expression";
const parseParameter = funcCache.getter(p => {
p = expression.parse(p);
if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
return p;
});
const component = {
oninit: vnode => {
vnode.state.parameters = Object.values(vnode.attrs.parameters).map(
parameter =>
Object.assign({}, parameter, {
parameter: filterParser.parseParameter(parameter.parameter)
parameter: parseParameter(parameter.parameter)
})
);
},
client/components/parameter-table.js
+12 -4
View File
@@ -4,15 +4,23 @@ import m from "mithril";
import * as components from "../components";
import * as taskQueue from "../task-queue";
import * as store from "../store";
import * as filterParser from "../../common/filter-parser";
import * as expression from "../../common/expression";
import * as funcCache from "../../common/func-cache";
const parseParameter = funcCache.getter(p => {
p = expression.parse(p);
if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
return p;
});
const component = {
oninit: vnode => {
vnode.state.object = filterParser.parseParameter(vnode.attrs.parameter);
vnode.state.object = parseParameter(vnode.attrs.parameter);
vnode.state.parameters = Object.values(vnode.attrs.childParameters).map(
parameter => {
let p = filterParser.parseParameter(parameter.parameter);
return Object.assign({}, parameter, { parameter: p });
return Object.assign({}, parameter, {
parameter: parseParameter(parameter.parameter)
});
}
);
},
client/components/summon-button.js
+9 -2
View File
@@ -4,12 +4,19 @@ import m from "mithril";
import * as taskQueue from "../task-queue";
import * as store from "../store";
import * as notifications from "../notifications";
import * as filterParser from "../../common/filter-parser";
import * as funcCache from "../../common/func-cache";
import * as expression from "../../common/expression";
const parseParameter = funcCache.getter(p => {
p = expression.parse(p);
if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
return p;
});
const component = {
oninit: vnode => {
vnode.state.parameters = Object.values(vnode.attrs.parameters).map(
parameter => filterParser.parseParameter(parameter)
parameter => parseParameter(parameter)
);
},
view: vnode => {
client/device-page.js
+1 -2
View File
@@ -3,7 +3,6 @@
import m from "mithril";
import config from "./config";
import Filter from "../common/filter";
import * as store from "./store";
import * as components from "./components";
@@ -15,7 +14,7 @@ function init(args) {
return Promise.resolve({
deviceId: args.id,
deviceFilter: new Filter(["=", "DeviceID.ID", args.id])
deviceFilter: ["=", ["PARAM", "DeviceID.ID"], args.id]
});
}
client/devices-page.js
+15 -12
View File
@@ -4,24 +4,24 @@ import m from "mithril";
import config from "./config";
import filterComponent from "./filter-component";
import * as filterParser from "../common/filter-parser";
import Filter from "../common/filter";
import * as store from "./store";
import * as components from "./components";
import * as taskQueue from "./task-queue";
import * as notifications from "./notifications";
import * as expression from "../common/expression";
function init(args) {
return new Promise((resolve, reject) => {
if (!window.authorizer.hasAccess("devices", 2))
return reject(new Error("You are not authorized to view this page"));
let filter = new Filter(args.filter);
let indexParameters = Object.values(config.ui.index).map(p =>
Object.assign({}, p, {
parameter: filterParser.parseParameter(p.parameter)
})
);
let filter = null;
if (args.filter != null) filter = expression.parse(`${args.filter}`);
let indexParameters = Object.values(config.ui.index).map(p => {
let param = expression.parse(p.parameter);
if (Array.isArray(param) && param[0] === "PARAM") param = param[1];
return Object.assign({}, p, { parameter: param });
});
resolve({ filter: filter, indexParameters: indexParameters });
});
}
@@ -281,16 +281,19 @@ const component = {
function onFilterChanged(filter) {
let params = {};
if (filter) params.filter = filterParser.stringify(filter);
if (filter != null) params.filter = expression.stringify(filter);
m.route.set("/devices", params);
}
let devs = store.fetch(
"devices",
vnode.attrs.filter,
vnode.attrs.filter == null ? true : vnode.attrs.filter,
vnode.state.showCount || 10
);
let count = store.count("devices", vnode.attrs.filter);
let count = store.count(
"devices",
vnode.attrs.filter == null ? true : vnode.attrs.filter
);
let selected = new Set();
if (vnode.state.selected)
@@ -303,7 +306,7 @@ const component = {
m("h1", "Listing devices"),
m(filterComponent, {
predefined: Object.values(config.ui.filters),
filter: vnode.attrs.filter ? vnode.attrs.filter.ast : null,
filter: vnode.attrs.filter,
onChange: onFilterChanged
}),
renderTable(
client/drawer-component.js
+1 -2
View File
@@ -5,7 +5,6 @@ import m from "mithril";
import * as taskQueue from "./task-queue";
import * as store from "./store";
import * as notifications from "./notifications";
import Filter from "../common/filter";
function mparam(param) {
return m("span.parameter", { title: param }, `${param}`);
@@ -41,7 +40,7 @@ function renderStagingSpv(task, queueFunc, cancelFunc) {
function renderStagingDownload(task) {
task.invalid = !task.fileName || !task.fileType;
const files = store.fetch("files", new Filter());
const files = store.fetch("files", true);
const idParts = task.device.split("-");
const oui = decodeURIComponent(idParts[0]);
const productClass =
client/faults-page.js
+10 -7
View File
@@ -3,10 +3,9 @@
import m from "mithril";
import filterComponent from "./filter-component";
import * as filterParser from "../common/filter-parser";
import Filter from "../common/filter";
import * as store from "./store";
import * as notifications from "./notifications";
import * as expression from "../common/expression";
function init(args) {
if (!window.authorizer.hasAccess("faults", 2))
@@ -14,7 +13,8 @@ function init(args) {
new Error("You are not authorized to view this page")
);
let filter = new Filter(args.filter);
let filter = null;
if (args.filter != null) filter = expression.parse(`${args.filter}`);
return Promise.resolve({ filter: filter });
}
@@ -153,16 +153,19 @@ const component = {
function onFilterChanged(filter) {
let params = {};
if (filter) params.filter = filterParser.stringify(filter);
if (filter != null) params.filter = expression.stringify(filter);
m.route.set("/faults", params);
}
let faults = store.fetch(
"faults",
vnode.attrs.filter,
vnode.attrs.filter == null ? true : vnode.attrs.filter,
vnode.state.showCount || 10
);
let count = store.count("faults", vnode.attrs.filter);
let count = store.count(
"faults",
vnode.attrs.filter == null ? true : vnode.attrs.filter
);
let selected = new Set();
if (vnode.state.selected)
@@ -180,7 +183,7 @@ const component = {
{ parameter: "retries" },
{ parameter: "timestamp" }
],
filter: vnode.attrs.filter ? vnode.attrs.filter.ast : null,
filter: vnode.attrs.filter,
onChange: onFilterChanged
}),
renderTable(faults, count.value, selected, showMore)
client/filter-component.js
+9 -11
View File
@@ -2,7 +2,7 @@
import m from "mithril";
import * as filterParser from "../common/filter-parser";
import * as expression from "../common/expression";
const component = {
view: vnode => {
@@ -15,16 +15,14 @@ const component = {
if (vnode.attrs.filter !== vnode.state.filter) {
vnode.state.filterList = [];
vnode.state.filter = vnode.attrs.filter;
if (vnode.attrs.filter)
if (vnode.attrs.filter != null)
if (vnode.attrs.filter[0] === "AND")
for (let i = 1; i < vnode.attrs.filter.length; ++i)
vnode.state.filterList.push(
filterParser.stringify(vnode.attrs.filter[i])
expression.stringify(vnode.attrs.filter[i])
);
else
vnode.state.filterList.push(
filterParser.stringify(vnode.attrs.filter)
);
vnode.state.filterList.push(expression.stringify(vnode.attrs.filter));
vnode.state.filterList.push("");
}
@@ -33,18 +31,18 @@ const component = {
vnode.state.filterList = vnode.state.filterList.filter(
f => f && f.trim()
);
let filterAst = vnode.state.filterList.map((f, idx) => {
let filter = vnode.state.filterList.map((f, idx) => {
try {
return filterParser.parse(f);
return expression.parse(f);
} catch (err) {
vnode.state.filterInvalid |= 1 << idx;
}
});
if (filterAst.length) filterAst = ["AND"].concat(filterAst);
else filterAst = null;
if (filter.length) filter = ["AND"].concat(filter);
else filter = null;
vnode.state.filterList.push("");
if (!vnode.state.filterInvalid) vnode.attrs.onChange(filterAst);
if (!vnode.state.filterInvalid) vnode.attrs.onChange(filter);
}
return m(
client/overview-page.js
+2 -2
View File
@@ -4,8 +4,8 @@ import m from "mithril";
import config from "./config";
import * as store from "./store";
import pieChartComponent from "./pie-chart-component";
import Filter from "../common/filter";
import * as funcCache from "../common/func-cache";
import * as expression from "../common/expression";
const GROUPS = config.ui.overview.groups;
const CHARTS = {};
@@ -19,7 +19,7 @@ function queryCharts(charts) {
charts[chartName] = chart = Object.assign({}, chart);
chart.slices = Object.assign({}, chart.slices);
for (let [sliceName, slice] of Object.entries(chart.slices)) {
const filter = funcCache.get(Filter.parse, slice.filter);
const filter = funcCache.get(expression.parse, slice.filter);
chart.slices[sliceName] = slice = Object.assign({}, slice);
slice.count = store.count("devices", filter);
}
client/store.js
+69 -51
View File
@@ -1,15 +1,15 @@
"use strict";
import m from "mithril";
import Filter from "../common/filter";
import config from "./config";
import * as filterParser from "../common/filter-parser";
import * as expressionParser from "../common/expression-parser";
import * as expression from "../common/expression";
import * as funcCache from "../common/func-cache";
let fulfillTimestamp = 0;
let unpackedFiltersCache = new WeakMap();
let expressionsCache = new WeakMap();
let unpackExpressionCache = new WeakMap();
let evaluateExpressionCache = new WeakMap();
const queries = {
filter: new WeakMap(),
@@ -47,46 +47,54 @@ class QueryResponse {
}
}
function unpackFilter(filter) {
let f = unpackedFiltersCache.get(filter);
if (!f) {
f = f.evaluateExpressions(fulfillTimestamp);
unpackedFiltersCache.set(filter, f);
function unpackExpression(exp) {
if (!Array.isArray(exp)) return exp;
let e = unpackExpressionCache.get(exp);
if (e === undefined) {
e = expression.evaluate(e, null, fulfillTimestamp);
unpackExpressionCache.set(exp, e);
}
return f;
return e;
}
function count(resourceType, filter) {
let queryResponse = resources[resourceType].count.get(filter.toString());
const filterStr = funcCache.get(expression.stringify, filter);
let queryResponse = resources[resourceType].count.get(filterStr);
if (queryResponse) return queryResponse;
queryResponse = new QueryResponse();
resources[resourceType].count.set(filter.toString(), queryResponse);
resources[resourceType].count.set(filterStr, queryResponse);
queries.filter.set(queryResponse, filter);
return queryResponse;
}
function limitFilter(resourceType, filter, last) {
if (resourceType === "devices")
return filter.and(new Filter(["<=", "DeviceID.ID", last]));
else return filter.and(new Filter(["<=", "_id", last]));
return expression.and(filter, ["<=", "DeviceID.ID", last]);
else return expression.and(filter, ["<=", ["PARAM", "_id"], last]);
}
function findMatches(resourceType, filter, limit) {
// Handle "tag =" and "tag <>" special cases
if (resourceType === "devices" && filter.ast) {
const ast = filterParser.map(filter.ast, e => {
if (e[1] === "tag")
if (e[0] === "=") return ["=", `Tags.${e[2]}`, true];
else if (e[0] === "<>") return ["NOT", ["=", `Tags.${e[2]}`, true]];
if (resourceType === "devices")
filter = expressionParser.map(filter, e => {
if (
Array.isArray(e) &&
Array.isArray(e[1]) &&
e[1][0] === "PARAM" &&
e[1][0] === "tag"
)
if (e[0] === "=") return ["=", ["PARAM", `Tags.${e[2]}`], true];
else if (e[0] === "<>")
return ["NOT", ["=", ["PARAM", `Tags.${e[2]}`], true]];
return e;
});
filter = new Filter(ast);
}
let value = [];
for (let [id, obj] of resources[resourceType].objects.entries())
if (filter.test(obj)) value.push(id);
if (expression.evaluate(filter, obj, fulfillTimestamp))
value.push(id);
value = value.sort();
if (limit) value = value.slice(0, limit);
@@ -97,13 +105,13 @@ function findMatches(resourceType, filter, limit) {
function inferQuery(resourceType, queryResponse) {
const limit = queries.limit.get(queryResponse);
let filter = queries.filter.get(queryResponse);
filter = unpackFilter(filter);
filter = unpackExpression(filter);
let last = queries.last.get(queryResponse);
if (last || !limit) {
if (last) filter = limitFilter(resourceType, filter, last);
if (
resources[resourceType].combinedFilter &&
filter.subset(resources[resourceType].combinedFilter)
expression.subset(filter, resources[resourceType].combinedFilter)
)
queries.fulfilled.set(queryResponse, fulfillTimestamp);
}
@@ -112,7 +120,8 @@ function inferQuery(resourceType, queryResponse) {
}
function fetch(resourceType, filter, limit = 0) {
const key = `${limit}:${filter.toString()}`;
const filterStr = funcCache.get(expression.stringify, filter);
const key = `${limit}:${filterStr}`;
let queryResponse = resources[resourceType].fetch.get(key);
if (queryResponse) return queryResponse;
@@ -131,8 +140,8 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
let updated = false;
if (_fulfillTimestamp > fulfillTimestamp) {
unpackedFiltersCache = new WeakMap();
expressionsCache = new WeakMap();
unpackExpressionCache = new WeakMap();
evaluateExpressionCache = new WeakMap();
for (let resource of Object.values(resources))
resource.combinedFilter = null;
@@ -156,14 +165,14 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
new Promise((resolve, reject) => {
updated = true;
let filter = queries.filter.get(queryResponse);
filter = unpackFilter(filter);
filter = unpackExpression(filter);
m
.request({
method: "HEAD",
url:
`/api/${resourceType}/?` +
m.buildQueryString({
filter: filter.toString()
filter: funcCache.get(expression.stringify, filter)
}),
extract: xhr => +xhr.getResponseHeader("x-total-count"),
background: true
@@ -201,14 +210,14 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
new Promise((resolve, reject) => {
updated = true;
let filter = queries.filter.get(queryResponse);
filter = unpackFilter(filter);
filter = unpackExpression(filter);
m
.request({
method: "GET",
url:
`/api/${resourceType}/?` +
m.buildQueryString({
filter: filter.toString(),
filter: funcCache.get(expression.stringify, filter),
limit: 1,
skip: limit - 1,
projection: "_id"
@@ -240,13 +249,13 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
toFetch = toFetch.filter(queryResponse => {
let filter = queries.filter.get(queryResponse);
filter = unpackFilter(filter);
filter = unpackExpression(filter);
let last = queries.last.get(queryResponse);
if (last) filter = limitFilter(resourceType, filter, last);
if (
resources[resourceType].combinedFilter &&
filter.subset(resources[resourceType].combinedFilter)
expression.subset(filter, resources[resourceType].combinedFilter)
) {
queries.value.set(
queryResponse,
@@ -257,11 +266,11 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
return false;
}
combinedFilter = filter.or(combinedFilter);
combinedFilter = expression.or(combinedFilter, filter);
return true;
});
if (!combinedFilter) return resolve(updated);
if (combinedFilter == null) continue;
updated = true;
let deleted = new Set();
@@ -269,10 +278,12 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
deleted = new Set(resources[resourceType].objects.keys());
let combinedFilterDiff = combinedFilter;
if (resources[resourceType].combinedFilter)
combinedFilterDiff = combinedFilterDiff.and(
resources[resourceType].combinedFilter.not()
combinedFilterDiff = expression.and(
combinedFilterDiff,
expression.not(resources[resourceType].combinedFilter)
);
resources[resourceType].combinedFilter = combinedFilter.or(
resources[resourceType].combinedFilter = expression.or(
combinedFilter,
resources[resourceType].combinedFilter
);
@@ -284,7 +295,10 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
url:
`/api/${resourceType}/?` +
m.buildQueryString({
filter: combinedFilterDiff.toString()
filter: funcCache.get(
expression.stringify,
combinedFilterDiff
)
})
})
.then(res => {
@@ -299,13 +313,19 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
for (let d of deleted) {
const obj = resources[resourceType].objects.get(d);
if (combinedFilterDiff.test(obj))
if (
expression.evaluate(
combinedFilterDiff,
obj,
fulfillTimestamp
)
)
resources[resourceType].objects.delete(d);
}
for (let queryResponse of toFetch) {
let filter = queries.filter.get(queryResponse);
filter = unpackFilter(filter);
filter = unpackExpression(filter);
let last = queries.last.get(queryResponse);
if (last) filter = limitFilter(resourceType, filter, last);
@@ -379,20 +399,18 @@ function deleteResource(resourceType, id) {
});
}
function evaluateExpression(exp, device) {
function evaluateExpression(exp, obj) {
if (!Array.isArray(exp)) return exp;
let exps = expressionsCache.get(exp);
if (!exps) expressionsCache.set(exp, (exps = new WeakMap()));
let exps = evaluateExpressionCache.get(exp);
if (!exps) evaluateExpressionCache.set(exp, (exps = new WeakMap()));
if (!exps.has(device)) {
let v = filterParser.evaluateExpressions(exp, e => {
if (e[0] === "FUNC" && e[1] === "NOW") return fulfillTimestamp;
});
exps.set(device, v);
if (!exps.has(obj)) {
let v = expression.evaluate(exp, obj, fulfillTimestamp);
exps.set(obj, v);
}
return exps.get(device);
return exps.get(obj);
}
function logIn(username, password) {
@@ -421,7 +439,7 @@ export {
count,
fetch,
fulfill,
unpackFilter,
unpackExpression,
getTimestamp,
postTasks,
updateTags,
common/authorizer.js
+5 -5
View File
@@ -1,5 +1,7 @@
"use strict";
const expression = require("./expression");
const validators = {
pass: require("./validators/pass"),
test: require("./validators/test")
@@ -18,15 +20,13 @@ class Authorizer {
return this.hasAccessCache[cacheKey];
let has = false;
for (const permissionSet of this.permissionSets) {
for (const perm of permissionSet) {
if (perm[resourceType]) {
for (let permissionSet of this.permissionSets) {
for (let perm of permissionSet)
if (perm[resourceType])
if (perm[resourceType].access >= access) {
has = true;
break;
}
}
}
}
this.hasAccessCache[cacheKey] = has;
common/expression-cnf.js
+172
View File
@@ -0,0 +1,172 @@
"use strict";
function collectRanges(exp, ranges = new Map()) {
if (["AND", "OR", "NOT"].includes(exp[0]))
for (let i = 1; i < exp.length; ++i) collectRanges(exp[i], ranges);
else if ([">", ">=", "<", "<=", "=", "<>"].includes(exp[0]))
if (!Array.isArray(exp[2])) {
const key = `${JSON.stringify(exp[1])}:${typeof exp[2]}`;
let s = ranges.get(key);
if (!s) ranges.set(key, (s = new Set()));
s.add(exp[2]);
} else if (!Array.isArray(exp[1])) {
const key = `${JSON.stringify(exp[2])}:${typeof exp[1]}`;
let s = ranges.get(key);
if (!s) ranges.set(key, (s = new Set()));
s.add(exp[2]);
}
return ranges;
}
function or(cnf1, cnf2) {
if (cnf1.length === 0) return cnf2;
else if (cnf2.length === 0) return cnf1;
const res = [];
for (let i = 0; i < cnf1.length; ++i)
for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j]));
return res;
}
function booleanCnf(exp) {
const ranges = collectRanges(exp);
for (let [k, v] of ranges)
if (k.endsWith(":number"))
ranges.set(k, Array.from(v).sort((a, b) => a - b));
else ranges.set(k, Array.from(v).sort());
const expressions = new Map();
const mutuallyExclusive = new Map();
function getVariable(lhs, op, rhs = "") {
const key = `${lhs}:${op}:${rhs}`;
let f = expressions.get(key);
if (!f) {
expressions.set(key, (f = expressions.size + 1));
let m = mutuallyExclusive.get(lhs);
if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
m.add(f);
if (rhs || key !== "null") m.add(getVariable(lhs, "null"));
}
return f;
}
function getComparisonVariables(lhs, from, fromInclusive, to, toInclusive) {
const type = from != null ? typeof from : typeof to;
const range = ranges.get(`${lhs}:${type}`);
const vars = [];
if (from == null) vars.push(`${lhs}:--${range[0]}`);
if (to == null) vars.push(`${lhs}:${range[range.length - 1]}--`);
if (fromInclusive) vars.push(`${lhs}:${from}`);
if (toInclusive) vars.push(`${lhs}:${to}`);
for (let i = 1; i < range.length; ++i)
if ((from == null || range[i] > from) && (to == null || range[i] <= to)) {
vars.push(`${lhs}:${range[i - 1]}`);
vars.push(`${lhs}:${range[i]}`);
vars.push(`${lhs}:${range[i - 1]}--${range[i]}`);
}
return vars.map(key => {
let f = expressions.get(key);
if (!f) {
expressions.set(key, (f = expressions.size + 1));
let m = mutuallyExclusive.get(lhs);
if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
m.add(f);
}
return f;
});
}
const comparisonOperators = {
"=": v => [v, true, v, true],
">": v => [v, false, null, null],
">=": v => [v, true, null, null],
"<": v => [null, null, v, false],
"<=": v => [null, null, v, true]
};
function recursive(clause, negate) {
let op = clause[0];
if ((op === "AND" && !negate) || (op === "OR" && negate)) {
let res = [];
for (let i = 1; i < clause.length; ++i)
res = res.concat(recursive(clause[i], negate));
return res;
} else if ((op === "OR" && !negate) || (op === "AND" && negate)) {
let res = [];
for (let i = 1; i < clause.length; ++i)
res = or(res, recursive(clause[i], negate));
return res;
} else if (op === "NOT") {
return recursive(clause[1], !negate);
} else if (op === "<>") {
const vars = getComparisonVariables(
JSON.stringify(clause[1]),
...comparisonOperators["="](clause[2])
);
if (!negate) return vars.map(x => [0 - x]);
else return [vars];
} else if (op === "IS NULL") {
const v = getVariable(JSON.stringify(clause[1]), "null");
if (negate) return [[0 - v]];
else return [[v]];
} else if (op === "IS NOT NULL") {
const v = getVariable(JSON.stringify(clause[1]), "null");
if (!negate) return [[0 - v]];
else return [[v]];
} else if (op in comparisonOperators) {
if (!Array.isArray(clause[2])) {
const vars = getComparisonVariables(
JSON.stringify(clause[1]),
...comparisonOperators[op](clause[2])
);
if (negate) return vars.map(x => [0 - x]);
else return [vars];
} else if (!Array.isArray(clause[1])) {
op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
const vars = getComparisonVariables(
JSON.stringify(clause[2]),
...comparisonOperators[op](clause[1])
);
if (negate) return vars.map(x => [0 - x]);
else return [vars];
} else {
let lhs = JSON.stringify(clause[1]);
let rhs = JSON.stringify(clause[2]);
// For consistency
if (rhs > lhs) {
let t = rhs;
rhs = lhs;
lhs = t;
op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
}
const v = getVariable(lhs, op, rhs);
if (negate) return [[0 - v]];
else return [[v]];
}
} else {
const v = getVariable(JSON.stringify(clause), "");
if (negate) return [[0 - v]];
else return [[v]];
}
}
const cnf = recursive(exp, false);
for (let m of mutuallyExclusive.values()) {
let ar = Array.from(m);
for (let i = 0; i < ar.length; ++i)
for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]);
}
return { vars: expressions.size, clauses: cnf };
}
exports.booleanCnf = booleanCnf;
common/filter-parser.js → common/expression-parser.js
+112 -115
View File
@@ -22,6 +22,21 @@ function interpretEscapes(str) {
});
}
function map(exp, callback) {
if (!Array.isArray(exp)) return callback(exp);
let clone;
for (let i = 1; i < exp.length; ++i) {
let sub = map(exp[i], callback);
if (sub !== exp[i]) {
clone = clone || exp.slice();
clone[i] = sub;
}
}
return callback(clone || exp);
}
function binaryLeft(operatorsParser, nextParser) {
return parsimmon.seqMap(
nextParser,
@@ -29,8 +44,9 @@ function binaryLeft(operatorsParser, nextParser) {
(first, rest) =>
rest.reduce((acc, ch) => {
let [op, another] = ch;
if (op === acc[0]) return acc.concat([another]);
if (op === another[0]) return [op, acc].concat(another.slice(1));
if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
if (Array.isArray(another) && op === another[0])
return [op, acc].concat(another.slice(1));
return [op, acc, another];
}, first)
);
@@ -61,12 +77,14 @@ const lang = parsimmon.createLanguage({
.result("IS NOT NULL")
.desc("IS NOT NULL")
)
.notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
.skip(parsimmon.optWhitespace);
},
NotOperator: function() {
return parsimmon
.regexp(/not/i)
.result("NOT")
.notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
.skip(parsimmon.optWhitespace)
.desc("NOT");
},
@@ -74,6 +92,7 @@ const lang = parsimmon.createLanguage({
return parsimmon
.regexp(/and/i)
.result("AND")
.notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
.skip(parsimmon.optWhitespace)
.desc("AND");
},
@@ -81,6 +100,7 @@ const lang = parsimmon.createLanguage({
return parsimmon
.regexp(/or/i)
.result("OR")
.notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
.skip(parsimmon.optWhitespace)
.desc("OR");
},
@@ -88,13 +108,13 @@ const lang = parsimmon.createLanguage({
return parsimmon
.alt(
parsimmon.regexp(/[a-zA-Z0-9_.*]+/),
r.ValueExpression.wrap(
r.Expression.wrap(
parsimmon.string("{").skip(parsimmon.optWhitespace),
parsimmon.string("}")
)
)
.atLeast(1)
.map(x => (x.length > 1 ? ["||"].concat(x) : x[0]))
.map(x => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
.skip(parsimmon.optWhitespace)
.desc("parameter");
},
@@ -120,13 +140,38 @@ const lang = parsimmon.createLanguage({
.map(Number)
.desc("number");
},
BooleanValue: function() {
return parsimmon
.alt(
parsimmon
.regexp(/true/i)
.result(true)
.desc("TRUE"),
parsimmon
.regexp(/false/i)
.result(false)
.desc("FALSE")
)
.notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
.skip(parsimmon.optWhitespace);
},
NullValue: function() {
return parsimmon
.regexp(/null/i)
.notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
.skip(parsimmon.optWhitespace)
.result(null)
.desc("NULL");
},
FuncValue: function(r) {
return parsimmon.seqMap(
parsimmon
.regexp(/([a-zA-Z0-0_]+)/, 1)
.regexp(/([a-zA-Z0-9_]+)/, 1)
.skip(parsimmon.optWhitespace)
.desc("function"),
r.Value.sepBy(parsimmon.string(",").skip(parsimmon.optWhitespace)).wrap(
r.ValueExpression.sepBy(
parsimmon.string(",").skip(parsimmon.optWhitespace)
).wrap(
parsimmon.string("(").skip(parsimmon.optWhitespace),
parsimmon.string(")").skip(parsimmon.optWhitespace)
),
@@ -135,6 +180,8 @@ const lang = parsimmon.createLanguage({
},
Value: function(r) {
return parsimmon.alt(
r.NullValue,
r.BooleanValue,
r.NumberValue,
r.StringValueSql,
r.StringValueJs,
@@ -143,7 +190,7 @@ const lang = parsimmon.createLanguage({
},
ValueExpression: function(r) {
return binaryLeft(
parsimmon.string("||"),
parsimmon.string("||").skip(parsimmon.optWhitespace),
binaryLeft(
parsimmon
.alt(parsimmon.string("+"), parsimmon.string("-"))
@@ -154,7 +201,8 @@ const lang = parsimmon.createLanguage({
.skip(parsimmon.optWhitespace),
parsimmon.alt(
r.Value,
r.ValueExpression.wrap(
r.Parameter,
r.Expression.wrap(
parsimmon.string("(").skip(parsimmon.optWhitespace),
parsimmon.string(")").skip(parsimmon.optWhitespace)
)
@@ -167,7 +215,7 @@ const lang = parsimmon.createLanguage({
return parsimmon.alt(
parsimmon.seqMap(r.Parameter, r.IsNullOperator, (p, o) => [o, p]),
parsimmon.seqMap(
r.Parameter,
r.ValueExpression,
r.ComparisonOperator,
r.ValueExpression,
(p, o, v) => [o, p, v]
@@ -183,129 +231,78 @@ const lang = parsimmon.createLanguage({
r.OrOperator,
binaryLeft(
r.AndOperator,
unary(
r.NotOperator,
r.Comparison.or(
r.Expression.wrap(
parsimmon.string("("),
parsimmon.string(")").skip(parsimmon.optWhitespace)
)
)
)
unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
)
).trim(parsimmon.optWhitespace);
}
});
function parse(filter) {
return lang.Expression.tryParse(filter);
function parse(str) {
if (!str) return null;
return lang.Expression.tryParse(str);
}
function parseExpression(exp) {
return lang.ValueExpression.tryParse(exp);
}
function stringify(exp) {
if (!Array.isArray(exp)) return JSON.stringify(exp);
function evaluateExpressions(ast, funcCallback) {
return map(ast, exp => {
if (exp[0] === "FUNC") {
return funcCallback(exp);
} else if (exp[0] === "*") {
let v = exp[1];
for (let i = 2; i < exp.length; ++i) v *= exp[i];
return v;
} else if (exp[0] === "/") {
let v = exp[1];
for (let i = 2; i < exp.length; ++i) v /= exp[i];
return v;
} else if (exp[0] === "+") {
let v = exp[1];
for (let i = 2; i < exp.length; ++i) v += exp[i];
return v;
} else if (exp[0] === "-") {
let v = exp[1];
for (let i = 2; i < exp.length; ++i) v -= exp[i];
return v;
} else if (exp[0] === "||") {
return exp.slice(1).join("");
}
});
}
const opLevels = {
OR: 10,
AND: 11,
NOT: 12,
"=": 20,
"<>": 21,
">": 22,
">=": 23,
"<": 24,
"<=": 25,
"||": 30,
"+": 31,
"-": 31,
"*": 32,
"/": 32
};
function parseParameter(p) {
return lang.Parameter.tryParse(p);
}
const op = exp[0].toUpperCase();
function stringify(filter) {
function value(v) {
if (Array.isArray(v))
if (v[0].toUpperCase() === "FUNC") {
return `${v[1]}(${v
.slice(2)
.map(e => value(e))
.join(", ")})`;
} else {
return v
.slice(1)
.map(e => value(e))
.join(` ${v[0]} `);
}
return JSON.stringify(v);
}
function expressions(v) {
return v.map(e => {
let str = stringify(e);
if (e[0] === "AND" || e[0] === "OR") return `(${str})`;
else return str;
});
}
function parameter(v) {
if (Array.isArray(v) && v[0] === "||")
return v
if (op === "FUNC") {
return `${exp[1]}(${exp
.slice(2)
.map(e => stringify(e))
.join(", ")})`;
} else if (op === "PARAM") {
if (typeof exp[1] === "string") return exp[1];
else if (Array.isArray(exp[1]) && exp[1][0] === "||")
return exp[1]
.slice(1)
.map(x => {
if (Array.isArray(x)) return `{${value(x)}}`;
else return x;
.map(p => {
if (typeof p === "string") return p;
else return `{${stringify(p)}}`;
})
.join("");
else return `{${stringify(exp[1])}}`;
} else if (op === "IS NULL" || op === "IS NOT NULL") {
return `${stringify(exp[1])} ${op}`;
} else if (op in opLevels) {
const parts = exp.slice(1).map((e, i) => {
if (
Array.isArray(e) &&
(opLevels[e[0]] < opLevels[exp[0]] ||
(opLevels[e[0]] < opLevels[exp[0]] && i > 0) ||
(opLevels[exp[0]] >= 20 &&
opLevels[e[0]] >= 20 &&
opLevels[e[0]] < 30))
)
return `(${stringify(e)})`;
else return stringify(e);
});
return v;
if (op === "NOT") return `${op} ${parts[0]}`;
else return parts.join(` ${op} `);
} else {
throw new Error(`Unrecognized operator ${exp[0]}`);
}
const op = filter[0].toUpperCase();
if (["AND", "OR"].includes(op))
return expressions(filter.slice(1)).join(` ${op} `);
else if (op === "NOT") return `NOT ${expressions(filter.slice(1))[0]}`;
else if ([">=", "<>", "<=", "=", ">", "<"].includes(op))
return `${parameter(filter[1])} ${op} ${value(filter[2])}`;
else if (["IS NULL", "IS NOT NULL"].includes(op))
return `${parameter(filter[1])} ${op}`;
else throw new Error("Unrecognized operator");
}
function map(filter, callback) {
let clone;
for (let i = 1; i < filter.length; ++i)
if (Array.isArray(filter[i])) {
let sub = map(filter[i], callback);
if (sub != null && sub !== filter[i]) {
clone = clone || filter.slice();
clone[i] = sub;
}
}
let r = callback(clone || filter);
if (r != null) return r;
return clone || filter;
}
exports.parse = parse;
exports.parseExpression = parseExpression;
exports.evaluateExpressions = evaluateExpressions;
exports.parseParameter = parseParameter;
exports.stringify = stringify;
exports.map = map;
common/expression.js
+188
View File
@@ -0,0 +1,188 @@
"use strict";
const expressionParser = require("./expression-parser");
const expressionCnf = require("./expression-cnf");
const satSolver = require("./sat-solver");
const isArray = Array.isArray;
function* permute(arr) {
if (arr.length <= 1) {
for (let i = 0; i < arr[0]; ++i) yield [i];
return;
}
let slc = arr.slice(1);
for (let i = 0; i < arr[0]; ++i)
for (let innerArr of permute(slc)) yield [i].concat(innerArr);
}
function reduce(exp, callback) {
let loop = true;
while (loop) {
loop = false;
for (let i = 2; i < exp.length; ++i) {
let res = callback(exp[i - 1], exp[i]);
if (res !== undefined) {
loop = true;
exp = exp.slice();
exp.splice(i - 1, 2, res);
}
}
}
if (exp.length === 2) return exp[1];
return exp;
}
function evaluate(exp, obj, now) {
return expressionParser.map(exp, e => {
if (!isArray(e)) return e;
if (e[0] === "FUNC") {
if (e[1] === "NOW") {
if (now) return now;
} else if (e[1] === "UPPER") {
if (e[2] == null) return null;
if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
} else if (e[1] === "LOWER") {
if (e[12] == null) return null;
if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
}
} else if (e[0] === "PARAM") {
if (e[1] == null) return null;
if (obj && !isArray(e[1]))
if (typeof obj[e[1]] === "object")
return obj[e[1]].value ? obj[e[1]].value[0] : null;
else return obj[e[1]];
} else if (e[0] === "AND") {
return reduce(e, (a, b) => {
if (!isArray(a)) return a ? b : a;
if (!isArray(b)) return b ? a : b;
});
} else if (e[0] === "OR") {
return reduce(e, (a, b) => {
if (!isArray(a)) return a ? a : b;
if (!isArray(b)) return b ? b : a;
});
} else if (e[0] === "NOT") {
if (!isArray(e[1])) return !e[1];
else if (e[1][0] === "NOT") return e[1][1];
} else if (e[0] === "IS NULL") {
if (isArray(e[1])) return e;
return e[1] == null;
} else if (e[0] === "IS NOT NULL") {
if (isArray(e[1])) return e;
return e[1] != null;
} else if (e[0] === "=") {
if (isArray(e[1]) || isArray(e[2])) return e;
if (e[1] == null || e[2] == null) return null;
return e[1] === e[2];
} else if (e[0] === "<>") {
if (isArray(e[1]) || isArray(e[2])) return e;
if (e[1] == null || e[2] == null) return null;
return e[1] !== e[2];
} else if (e[0] === ">") {
if (isArray(e[1]) || isArray(e[2])) return e;
if (e[1] == null || e[2] == null) return null;
return e[1] > e[2];
} else if (e[0] === ">=") {
if (isArray(e[1]) || isArray(e[2])) return e;
if (e[1] == null || e[2] == null) return null;
return e[1] >= e[2];
} else if (e[0] === "<") {
if (isArray(e[1]) || isArray(e[2])) return e;
if (e[1] == null || e[2] == null) return null;
return e[1] < e[2];
} else if (e[0] === "<=") {
if (isArray(e[1]) || isArray(e[2])) return e;
if (e[1] == null || e[2] == null) return null;
return e[1] <= e[2];
} else if (e[0] === "*") {
return reduce(e, (a, b) => {
if (!isArray(a) && !isArray(b)) {
if (a == null || b == null) return null;
return a * b;
}
});
} else if (e[0] === "/") {
return reduce(e, (a, b) => {
if (!isArray(a) && !isArray(b)) {
if (a == null || b == null) return null;
return a / b;
}
});
} else if (e[0] === "+") {
return reduce(e, (a, b) => {
if (!isArray(a) && !isArray(b)) {
if (a == null || b == null) return null;
return a + b;
}
});
} else if (e[0] === "-") {
return reduce(e, (a, b) => {
if (!isArray(a) && !isArray(b)) {
if (a == null || b == null) return null;
return a - b;
}
});
} else if (e[0] === "||") {
return reduce(e, (a, b) => {
if (!isArray(a) && !isArray(b)) {
if (a == null || b == null) return null;
return `${a}${b}`;
}
});
}
return e;
});
}
function and(exp1, exp2) {
if (!isArray(exp1)) return exp1 ? exp2 : exp1;
if (!isArray(exp2)) return exp2 ? exp1 : exp2;
let res = ["AND"];
if (exp1[0] === "AND") res = res.concat(exp1.slice(1));
else res.push(exp1);
if (exp2[0] === "AND") res = res.concat(exp2.slice(1));
else res.push(exp2);
return res;
}
function or(exp1, exp2) {
if (!isArray(exp1)) return exp1 ? exp1 : exp2;
if (!isArray(exp2)) return exp2 ? exp2 : exp1;
let res = ["OR"];
if (exp1[0] === "OR") res = res.concat(exp1.slice(1));
else res.push(exp1);
if (exp2[0] === "OR") res = res.concat(exp2.slice(1));
else res.push(exp2);
return res;
}
function not(exp) {
if (isArray(exp) && exp[0] === "NOT") return exp[1];
return ["NOT", exp];
}
function subset(exp1, exp2) {
const e = evaluate(["NOT", ["OR", ["NOT", exp1], exp2]]);
if (!isArray(e)) return !e;
const { vars, clauses } = expressionCnf.booleanCnf(e);
return !satSolver.naiveDpll(clauses, vars);
}
exports.evaluate = evaluate;
exports.and = and;
exports.or = or;
exports.not = not;
exports.subset = subset;
exports.parse = expressionParser.parse;
exports.stringify = expressionParser.stringify;
common/filter-cnf.js
-138
View File
@@ -1,138 +0,0 @@
"use strict";
function collectRanges(filter, ranges = new Map()) {
if (["AND", "OR", "NOT"].includes(filter[0])) {
for (let i = 1; i < filter.length; ++i) collectRanges(filter[i], ranges);
} else if ([">", ">=", "<", "<=", "=", "<>"].includes(filter[0])) {
const key = `${filter[1]}:${typeof filter[2]}`;
let s = ranges.get(key);
if (!s) ranges.set(key, (s = new Set()));
s.add(filter[2]);
}
return ranges;
}
function or(cnf1, cnf2) {
if (cnf1.length === 0) return cnf2;
else if (cnf2.length === 0) return cnf1;
const res = [];
for (let i = 0; i < cnf1.length; ++i)
for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j]));
return res;
}
function booleanCnf(filter) {
const ranges = collectRanges(filter);
for (let [k, v] of ranges)
if (k.endsWith(":number"))
ranges.set(k, Array.from(v).sort((a, b) => a - b));
else ranges.set(k, Array.from(v).sort());
const expressions = new Map();
const mutuallyExclusive = new Map();
function getIsNullVariable(param) {
const key = `${param}:null`;
let f = expressions.get(key);
if (!f) {
expressions.set(key, (f = expressions.size + 1));
let m = mutuallyExclusive.get(param);
if (!m) mutuallyExclusive.set(param, (m = new Set()));
m.add(f);
}
return f;
}
function getComparisonVariables(param, from, fromInclusive, to, toInclusive) {
const type = from != null ? typeof from : typeof to;
const range = ranges.get(`${param}:${type}`);
const vars = [];
if (from == null) vars.push(`${param}:--${range[0]}`);
if (to == null) vars.push(`${param}:${range[range.length - 1]}--`);
if (fromInclusive) vars.push(`${param}:${from}`);
if (toInclusive) vars.push(`${param}:${to}`);
for (let i = 1; i < range.length; ++i)
if ((from == null || range[i] > from) && (to == null || range[i] <= to)) {
vars.push(`${param}:${range[i - 1]}`);
vars.push(`${param}:${range[i]}`);
vars.push(`${param}:${range[i - 1]}--${range[i]}`);
}
return vars.map(key => {
let f = expressions.get(key);
if (!f) {
expressions.set(key, (f = expressions.size + 1));
let m = mutuallyExclusive.get(param);
if (!m) mutuallyExclusive.set(param, (m = new Set()));
m.add(f);
}
return f;
});
}
const comparisonOperators = {
"=": v => [v, true, v, true],
">": v => [v, false, null, null],
">=": v => [v, true, null, null],
"<": v => [null, null, v, false],
"<=": v => [null, null, v, true]
};
function recursive(clause, negate) {
const op = clause[0];
if ((op === "AND" && !negate) || (op === "OR" && negate)) {
let res = [];
for (let i = 1; i < clause.length; ++i)
res = res.concat(recursive(clause[i], negate));
return res;
} else if ((op === "OR" && !negate) || (op === "AND" && negate)) {
let res = [];
for (let i = 1; i < clause.length; ++i)
res = or(res, recursive(clause[i], negate));
return res;
} else if (op === "NOT") {
return recursive(clause[1], !negate);
} else if (op in comparisonOperators) {
const vars = getComparisonVariables(
clause[1],
...comparisonOperators[op](clause[2])
);
if (negate) return vars.map(x => [0 - x]);
else return [vars];
} else if (op === "<>") {
const vars = getComparisonVariables(
clause[1],
...comparisonOperators["="](clause[2])
);
if (!negate) return vars.map(x => [0 - x]);
else return [vars];
} else if (op === "IS NULL") {
const v = getIsNullVariable(clause[1]);
if (negate) return [[0 - v]];
else return [[v]];
} else if (op === "IS NOT NULL") {
const v = getIsNullVariable(clause[1]);
if (!negate) return [[0 - v]];
else return [[v]];
} else {
throw new Error(`Unknown operator ${op}`);
}
}
const cnf = recursive(filter, false);
for (let m of mutuallyExclusive.values()) {
let ar = Array.from(m);
for (let i = 0; i < ar.length; ++i)
for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]);
}
return { vars: expressions.size, clauses: cnf };
}
exports.booleanCnf = booleanCnf;
common/filter.js
-145
View File
@@ -1,145 +0,0 @@
"use strict";
const filterParser = require("./filter-parser");
const filterCnf = require("./filter-cnf");
const satSolver = require("./sat-solver");
const funcCache = require("./func-cache");
function* permute(arr) {
if (arr.length <= 1) {
for (let i = 0; i < arr[0]; ++i) yield [i];
return;
}
let slc = arr.slice(1);
for (let i = 0; i < arr[0]; ++i)
for (let innerArr of permute(slc)) yield [i].concat(innerArr);
}
class Filter {
constructor(str) {
if (!str) this.ast = null;
else if (Array.isArray(str)) this.ast = str;
else if (str instanceof Filter) this.ast = str.ast;
else if (Array.isArray(str.ast) || str.ast === null) this.ast = str.ast;
else this.ast = filterParser.parse(str);
}
static parse(str) {
return new Filter(str);
}
toString() {
if (!this.string)
if (this.ast) this.string = filterParser.stringify(this.ast);
else this.string = "";
return this.string;
}
or(fltr) {
if (!fltr || !this.ast) return this;
else if (!fltr.ast) return fltr;
let f = ["OR"];
if (this.ast[0] === "OR") f = f.concat(this.ast.slice(1));
else f.push(this.ast);
if (fltr.ast[0] === "OR") f = f.concat(fltr.ast.slice(1));
else f.push(fltr.ast);
return new Filter(f);
}
and(fltr) {
if (!fltr || !fltr.ast) return this;
else if (!this.ast) return fltr;
let f = ["AND"];
if (this.ast[0] === "AND") f = f.concat(this.ast.slice(1));
else f.push(this.ast);
if (fltr.ast[0] === "AND") f = f.concat(fltr.ast.slice(1));
else f.push(fltr.ast);
return new Filter(f);
}
not() {
if (this.ast[0] === "NOT") return new Filter(this.ast[1]);
else return new Filter(["NOT", this.ast]);
}
evaluateExpressions(now = Date.now()) {
if (!this.ast) return this;
let ast = filterParser.evaluateExpressions(this.ast, exp => {
if (exp[0] === "FUNC" && exp[1] === "NOW") return now;
});
if (ast === this.ast) return this;
else return new Filter(ast);
}
subset(fltr) {
if (!fltr.ast) return true;
else if (!this.ast) return false;
const f = this.not()
.or(fltr)
.not();
const { vars, clauses } = filterCnf.booleanCnf(f.ast);
return !satSolver.naiveDpll(clauses, vars);
}
test(obj) {
if (!this.ast) return true;
return filterParser.map(this.ast, exp => {
const op = exp[0];
if (op === "AND") {
for (let i = 1; i < exp.length; ++i) if (!exp[i]) return false;
return true;
} else if (op === "OR") {
for (let i = 1; i < exp.length; ++i) if (exp[i]) return true;
return false;
} else if (op === "NOT") {
return !exp[1];
} else if (op === "=") {
const val =
typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
return val === exp[2];
} else if (op === "<>") {
const val =
typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
return val !== exp[2];
} else if (op === ">=") {
const val =
typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
return typeof val === typeof exp[2] && val >= exp[2];
} else if (op === ">") {
const val =
typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
return typeof val === typeof exp[2] && val > exp[2];
} else if (op === "<=") {
const val =
typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
return typeof val === typeof exp[2] && val <= exp[2];
} else if (op === "<") {
const val =
typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
return typeof val === typeof exp[2] && val < exp[2];
} else if (op === "IS NULL") {
return obj[exp[1]] == null;
} else if (op === "IS NOT NULL") {
return obj[exp[1]] != null;
} else {
throw new Error(`Unsupported operator ${op}`);
}
});
}
}
module.exports = Filter;
common/func-cache.js
+2
View File
@@ -22,6 +22,8 @@ function purge() {
cache1 = new WeakMap();
}
setInterval(purge, 120000);
exports.get = get;
exports.getter = getter;
exports.purge = purge;
server/api-functions.js
+46 -37
View File
@@ -7,20 +7,22 @@ const querystring = require("querystring");
const config = require("./config");
const device = require("./device");
const filterParser = require("../common/filter-parser");
const Filter = require("../common/filter");
const expressionParser = require("../common/expression-parser");
const expression = require("../common/expression");
const commaAscii = ",".charCodeAt(0);
const newLineAscii = "\n".charCodeAt(0);
function unpackTimestamps(filter) {
return filterParser.map(filter, exp => {
if (!Array.isArray(filter)) return filter;
return expressionParser.map(filter, exp => {
if (["=", "<>", ">=", "<=", "<", ">"].includes(exp[0]))
if (typeof exp[2] === "number") {
if (Array.isArray(exp) && typeof exp[2] === "number") {
let alt = exp.slice();
alt[2] = new Date(alt[2]).toJSON();
return ["OR", exp, alt];
}
return exp;
});
}
@@ -39,37 +41,42 @@ function filterToMongoQuery(filter, negate = false, res = {}) {
} else if (op === "NOT") {
filterToMongoQuery(filter[1], !negate, res);
} else if (op === "=") {
let p = (res[filter[1]] = res[filter[1]] || {});
const param = filter[1][1];
let p = (res[param] = res[param] || {});
if (negate) p["$ne"] = filter[2];
else p["$eq"] = filter[2];
} else if (op === "<>") {
let p = (res[filter[1]] = res[filter[1]] || {});
if (negate) {
p["$eq"] = filter[2];
} else {
p["$ne"] = filter[2];
p["$exists"] = true;
}
const param = filter[1][1];
let p = (res[param] = res[param] || {});
if (negate) p = p["$not"] = p["$not"] || {};
p["$ne"] = filter[2];
p["$exists"] = true;
} else if (op === ">") {
let p = (res[filter[1]] = res[filter[1]] || {});
const param = filter[1][1];
let p = (res[param] = res[param] || {});
if (negate) p = p["$not"] = p["$not"] || {};
p["$gt"] = filter[2];
} else if (op === ">=") {
let p = (res[filter[1]] = res[filter[1]] || {});
const param = filter[1][1];
let p = (res[param] = res[param] || {});
if (negate) p = p["$not"] = p["$not"] || {};
p["$gte"] = filter[2];
} else if (op === "<") {
let p = (res[filter[1]] = res[filter[1]] || {});
const param = filter[1][1];
let p = (res[param] = res[param] || {});
if (negate) p = p["$not"] = p["$not"] || {};
p["$lt"] = filter[2];
} else if (op === "<=") {
let p = (res[filter[1]] = res[filter[1]] || {});
const param = filter[1][1];
let p = (res[param] = res[param] || {});
if (negate) p = p["$not"] = p["$not"] || {};
p["$lte"] = filter[2];
} else if (op === "IS NULL") {
res[filter[1]] = { $exists: negate };
const param = filter[1][1];
res[param] = { $exists: negate };
} else if (op === "IS NOT NULL") {
res[filter[1]] = { $exists: !negate };
const param = filter[1][1];
res[param] = { $exists: !negate };
} else {
throw new Error(`Unrecognized operator ${op}`);
}
@@ -80,14 +87,15 @@ function filterToMongoQuery(filter, negate = false, res = {}) {
function count(resource, filter, limit) {
return new Promise((resolve, reject) => {
let qs = {};
if (filter && filter.ast) {
filter = filter.evaluateExpressions();
let ast = unpackTimestamps(filter.ast);
filter = expression.evaluate(filter, null, Date.now());
filter = unpackTimestamps(filter);
let q = filterToMongoQuery(ast);
if (resource === "devices") q = device.transposeQuery(q);
qs.query = JSON.stringify(q);
}
let q = {};
if (Array.isArray(filter)) q = filterToMongoQuery(filter);
else if (filter != null && !filter) return resolve(0);
if (resource === "devices") q = device.transposeQuery(q);
qs.query = JSON.stringify(q);
if (limit) qs.limit = limit;
@@ -117,15 +125,15 @@ function query(resource, filter, limit, skip, projection, callback) {
let ret;
if (!callback) ret = [];
let qs = {};
if (filter && filter.ast) {
filter = filter.evaluateExpressions();
let ast = unpackTimestamps(filter.ast);
filter = expression.evaluate(filter, null, Date.now());
filter = unpackTimestamps(filter);
let q = filterToMongoQuery(ast);
if (resource === "devices") q = device.transposeQuery(q);
let q = {};
if (Array.isArray(filter)) q = filterToMongoQuery(filter);
else if (filter != null && !filter) return resolve(ret);
qs.query = JSON.stringify(q);
}
if (resource === "devices") q = device.transposeQuery(q);
qs.query = JSON.stringify(q);
if (limit) qs.limit = limit;
if (skip) qs.skip = skip;
@@ -308,12 +316,13 @@ function postTasks(deviceId, tasks) {
let promises2 = [];
for (let s of statuses) {
promises2.push(query("tasks", new Filter(["=", "_id", s._id])));
promises2.push(query("tasks", ["=", ["PARAM", "_id"], s._id]));
promises2.push(
query(
"faults",
new Filter(["=", "_id", `${deviceId}:task_${s._id}`])
)
query("faults", [
"=",
["PARAM", "_id"],
`${deviceId}:task_${s._id}`
])
);
}
server/api.js
+24 -18
View File
@@ -4,7 +4,7 @@ const Router = require("koa-router");
const db = require("./db");
const apiFunctions = require("./api-functions");
const Filter = require("../common/filter");
const expression = require("../common/expression");
const router = new Router();
@@ -33,17 +33,20 @@ const resources = {
for (let [resource, flags] of Object.entries(resources)) {
router.head(`/${resource}`, async (ctx, next) => {
let filter, limit;
if (ctx.request.query.filter) filter = new Filter(ctx.request.query.filter);
let filter = true;
let limit;
if (ctx.request.query.filter)
filter = expression.parse(ctx.request.query.filter);
if (ctx.request.query.limit) limit = +ctx.request.query.limit;
if (!ctx.state.authorizer.hasAccess(resource, 1)) return next();
// Exclude temporary tasks and faults
if (resource === "tasks" || resource === "faults") {
let f = new Filter(["NOT", ["<", "expiry", Date.now() + 60000]]);
filter = f.and(filter);
}
if (resource === "tasks" || resource === "faults")
filter = expression.and(filter, [
"NOT",
["<", ["PARAM", "expiry"], Date.now() + 60000]
]);
let count;
if (flags & RESOURCE_DB) count = await db.count(resource, filter, limit);
@@ -54,8 +57,10 @@ for (let [resource, flags] of Object.entries(resources)) {
});
router.get(`/${resource}`, async (ctx, next) => {
let filter, limit, skip, projection;
if (ctx.request.query.filter) filter = new Filter(ctx.request.query.filter);
let filter = true;
let limit, skip, projection;
if (ctx.request.query.filter)
filter = expression.parse(ctx.request.query.filter);
if (ctx.request.query.limit) limit = +ctx.request.query.limit;
if (ctx.request.query.skip) skip = +ctx.request.query.skip;
if (ctx.request.query.projection) projection = ctx.request.query.projection;
@@ -63,10 +68,11 @@ for (let [resource, flags] of Object.entries(resources)) {
if (!ctx.state.authorizer.hasAccess(resource, 2)) return next();
// Exclude temporary tasks and faults
if (resource === "tasks" || resource === "faults") {
let f = new Filter(["NOT", ["<", "expiry", Date.now() + 60000]]);
filter = f.and(filter);
}
if (resource === "tasks" || resource === "faults")
filter = expression.and(filter, [
"NOT",
["<", ["PARAM", "expiry"], Date.now() + 60000]
]);
let res;
if (flags & RESOURCE_DB)
@@ -78,7 +84,7 @@ for (let [resource, flags] of Object.entries(resources)) {
});
router.head(`/${resource}/:id`, async (ctx, next) => {
let filter = new Filter(`${RESOURCE_IDS[resource]} = "${ctx.params.id}"`);
let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
if (!ctx.state.authorizer.hasAccess(resource, 2)) return next();
let res;
@@ -90,7 +96,7 @@ for (let [resource, flags] of Object.entries(resources)) {
});
router.get(`/${resource}/:id`, async (ctx, next) => {
let filter = new Filter(`${RESOURCE_IDS[resource]} = "${ctx.params.id}"`);
let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
if (!ctx.state.authorizer.hasAccess(resource, 2)) return next();
let res;
@@ -104,7 +110,7 @@ for (let [resource, flags] of Object.entries(resources)) {
if (flags & RESOURCE_DELETE && !(flags & RESOURCE_DB))
router.delete(`/${resource}/:id`, async (ctx, next) => {
const authorizer = ctx.state.authorizer;
let filter = new Filter(`${RESOURCE_IDS[resource]} = "${ctx.params.id}"`);
let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
if (!authorizer.hasAccess(resource, 2)) return next();
let res = await apiFunctions.query(resource, filter);
if (!res.length) return next();
@@ -119,7 +125,7 @@ for (let [resource, flags] of Object.entries(resources)) {
router.post("/devices/:id/tasks", async (ctx, next) => {
const authorizer = ctx.state.authorizer;
let filter = new Filter(`DeviceID.ID = "${ctx.params.id}"`);
let filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
if (!authorizer.hasAccess("devices", 2)) return next();
let devices = await apiFunctions.query("devices", filter);
if (!devices.length) return next();
@@ -135,7 +141,7 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
router.post("/devices/:id/tags", async (ctx, next) => {
const authorizer = ctx.state.authorizer;
let filter = new Filter(`DeviceID.ID = "${ctx.params.id}"`);
let filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
if (!authorizer.hasAccess("devices", 2)) return next();
let res = await apiFunctions.query("devices", filter);
if (!res.length) return next();
server/db.js
+4 -2
View File
@@ -20,7 +20,8 @@ function getClient() {
function query(resource, filter, limit, skip, callback) {
let q;
if (filter && filter.ast) q = apiFunctions.filterToMongoQuery(filter.ast);
if (Array.isArray(filter)) q = apiFunctions.filterToMongoQuery(filter);
else if (filter != null && !filter) return Promise.resolve([]);
return new Promise((resolve, reject) => {
getClient().then(client => {
@@ -50,7 +51,8 @@ function query(resource, filter, limit, skip, callback) {
function count(resource, filter) {
let q;
if (filter && filter.ast) q = apiFunctions.filterToMongoQuery(filter.ast);
if (Array.isArray(filter)) q = apiFunctions.filterToMongoQuery(filter);
else if (filter != null && !filter) return Promise.resolve([]);
return new Promise((resolve, reject) => {
getClient().then(client => {
server/server.js
+15 -1
View File
@@ -11,6 +11,7 @@ const jwt = require("jsonwebtoken");
const config = require("./config");
const api = require("./api");
const Authorizer = require("../common/authorizer");
const expression = require("../common/expression");
const koa = new Koa();
const router = new Router();
@@ -20,7 +21,20 @@ const JWT_COOKIE = "genieacs-ui-jwt";
function getPermissionSets(roles) {
const allPermissions = config.permissions;
const permissionSets = roles.map(r => Object.values(allPermissions[r] || {}));
const permissionSets = roles.map(role => {
return Object.values(allPermissions[role] || {}).map(p => {
p = Object.assign({}, p);
for (let [k, v] of Object.entries(p)) {
p[k] = v = Object.assign({}, v);
if (["devices", "faults", "files"].includes(k))
if (v.filter == null || v.filter === "") v.filter = true;
else {
v.filter = expression.parse(v.filter);
}
}
return p;
});
});
return permissionSets;
}