Replace Filter class with expression structure

This is in order to be able to support UPPER() and LOWER() functions in
queries which are needed for case-insensitive search.

client/app.js

@@ -11,7 +11,6 @@ import * as devicePage from "./device-page";
 import * as errorPage from "./error-page";
 import * as faultsPage from "./faults-page";
 import Authorizer from "../common/authorizer";
-import * as funcCache from "../common/func-cache";
 
 window.authorizer = new Authorizer(window.permissionSets);
 
@@ -69,8 +68,6 @@ function pagify(pageName, page) {
   return component;
 }
 
-setInterval(funcCache.purge, 120000);
-
 m.route(document.body, "/overview", {
   "/login": pagify("login", loginPage),
   "/overview": pagify("overview", overviewPage),

client/components/container.js

@@ -3,16 +3,14 @@
 import m from "mithril";
 import * as components from "../components";
 import * as store from "../store";
-import Filter from "../../common/filter";
+import * as expression from "../../common/expression";
 import * as funcCache from "../../common/func-cache";
 
 const component = {
   view: vnode => {
     if (vnode.attrs.filter) {
-      const filter = store.unpackFilter(
+      const filter = funcCache.get(expression.parse, vnode.attrs.filter);
-        funcCache.get(Filter.parse, vnode.attrs.filter)
+      if (!store.evaluateExpression(filter, vnode.attrs.device)) return;
-      );
-      if (!filter.test(vnode.attrs.device)) return;
     }
 
     let children = Object.values(vnode.attrs.components).map(c => {

client/components/device-faults.js

@@ -2,21 +2,17 @@
 
 import m from "mithril";
 import * as store from "../store";
-import Filter from "../../common/filter";
 import * as notifications from "../notifications";
 
 const component = {
   view: vnode => {
     const device = vnode.attrs.device;
     const deviceId = device["DeviceID.ID"].value[0];
-    const faults = store.fetch(
+    const faults = store.fetch("faults", [
-      "faults",
-      new Filter([
       "AND",
-        [">", "_id", `${deviceId}:`],
+      [">", ["PARAM", "_id"], `${deviceId}:`],
-        ["<", "_id", `${deviceId}:zzzz`]
+      ["<", ["PARAM", "_id"], `${deviceId}:zzzz`]
-      ])
+    ]);
-    );
 
     const headers = ["Channel", "Code", "Message", "Retries", "Timestamp"].map(
       l => m("th", l)

client/components/overview-dot.js

@@ -4,7 +4,7 @@ import m from "mithril";
 import config from "../config";
 import * as store from "../store";
 import * as funcCache from "../../common/func-cache";
-import Filter from "../../common/filter";
+import * as expression from "../../common/expression";
 
 const CHARTS = config.ui.overview.charts;
 
@@ -13,10 +13,8 @@ const component = {
     const device = vnode.attrs.device;
     const chart = CHARTS[vnode.attrs.chart];
     for (let slice of Object.values(chart.slices)) {
-      const filter = store.unpackFilter(
+      const filter = funcCache.get(expression.parse, slice.filter);
-        funcCache.get(Filter.parse, slice.filter)
+      if (store.evaluateExpression(filter, device)) {
-      );
-      if (filter.test(device)) {
         const dot = m(
           "svg",
           {

client/components/parameter-list.js

@@ -3,14 +3,21 @@
 import m from "mithril";
 import * as components from "../components";
 import * as store from "../store";
-import * as filterParser from "../../common/filter-parser";
+import * as funcCache from "../../common/func-cache";
+import * as expression from "../../common/expression";
+
+const parseParameter = funcCache.getter(p => {
+  p = expression.parse(p);
+  if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
+  return p;
+});
 
 const component = {
   oninit: vnode => {
     vnode.state.parameters = Object.values(vnode.attrs.parameters).map(
       parameter =>
         Object.assign({}, parameter, {
-          parameter: filterParser.parseParameter(parameter.parameter)
+          parameter: parseParameter(parameter.parameter)
         })
     );
   },

client/components/parameter-table.js

@@ -4,15 +4,23 @@ import m from "mithril";
 import * as components from "../components";
 import * as taskQueue from "../task-queue";
 import * as store from "../store";
-import * as filterParser from "../../common/filter-parser";
+import * as expression from "../../common/expression";
+import * as funcCache from "../../common/func-cache";
+
+const parseParameter = funcCache.getter(p => {
+  p = expression.parse(p);
+  if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
+  return p;
+});
 
 const component = {
   oninit: vnode => {
-    vnode.state.object = filterParser.parseParameter(vnode.attrs.parameter);
+    vnode.state.object = parseParameter(vnode.attrs.parameter);
     vnode.state.parameters = Object.values(vnode.attrs.childParameters).map(
       parameter => {
-        let p = filterParser.parseParameter(parameter.parameter);
+        return Object.assign({}, parameter, {
-        return Object.assign({}, parameter, { parameter: p });
+          parameter: parseParameter(parameter.parameter)
+        });
       }
     );
   },

client/components/summon-button.js

@@ -4,12 +4,19 @@ import m from "mithril";
 import * as taskQueue from "../task-queue";
 import * as store from "../store";
 import * as notifications from "../notifications";
-import * as filterParser from "../../common/filter-parser";
+import * as funcCache from "../../common/func-cache";
+import * as expression from "../../common/expression";
+
+const parseParameter = funcCache.getter(p => {
+  p = expression.parse(p);
+  if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
+  return p;
+});
 
 const component = {
   oninit: vnode => {
     vnode.state.parameters = Object.values(vnode.attrs.parameters).map(
-      parameter => filterParser.parseParameter(parameter)
+      parameter => parseParameter(parameter)
     );
   },
   view: vnode => {

client/device-page.js

@@ -3,7 +3,6 @@
 import m from "mithril";
 
 import config from "./config";
-import Filter from "../common/filter";
 import * as store from "./store";
 import * as components from "./components";
 
@@ -15,7 +14,7 @@ function init(args) {
 
   return Promise.resolve({
     deviceId: args.id,
-    deviceFilter: new Filter(["=", "DeviceID.ID", args.id])
+    deviceFilter: ["=", ["PARAM", "DeviceID.ID"], args.id]
   });
 }
 

client/devices-page.js

@@ -4,24 +4,24 @@ import m from "mithril";
 
 import config from "./config";
 import filterComponent from "./filter-component";
-import * as filterParser from "../common/filter-parser";
-import Filter from "../common/filter";
 import * as store from "./store";
 import * as components from "./components";
 import * as taskQueue from "./task-queue";
 import * as notifications from "./notifications";
+import * as expression from "../common/expression";
 
 function init(args) {
   return new Promise((resolve, reject) => {
     if (!window.authorizer.hasAccess("devices", 2))
       return reject(new Error("You are not authorized to view this page"));
 
-    let filter = new Filter(args.filter);
+    let filter = null;
-    let indexParameters = Object.values(config.ui.index).map(p =>
+    if (args.filter != null) filter = expression.parse(`${args.filter}`);
-      Object.assign({}, p, {
+    let indexParameters = Object.values(config.ui.index).map(p => {
-        parameter: filterParser.parseParameter(p.parameter)
+      let param = expression.parse(p.parameter);
-      })
+      if (Array.isArray(param) && param[0] === "PARAM") param = param[1];
-    );
+      return Object.assign({}, p, { parameter: param });
+    });
     resolve({ filter: filter, indexParameters: indexParameters });
   });
 }
@@ -281,16 +281,19 @@ const component = {
 
     function onFilterChanged(filter) {
       let params = {};
-      if (filter) params.filter = filterParser.stringify(filter);
+      if (filter != null) params.filter = expression.stringify(filter);
       m.route.set("/devices", params);
     }
 
     let devs = store.fetch(
       "devices",
-      vnode.attrs.filter,
+      vnode.attrs.filter == null ? true : vnode.attrs.filter,
       vnode.state.showCount || 10
     );
-    let count = store.count("devices", vnode.attrs.filter);
+    let count = store.count(
+      "devices",
+      vnode.attrs.filter == null ? true : vnode.attrs.filter
+    );
 
     let selected = new Set();
     if (vnode.state.selected)
@@ -303,7 +306,7 @@ const component = {
       m("h1", "Listing devices"),
       m(filterComponent, {
         predefined: Object.values(config.ui.filters),
-        filter: vnode.attrs.filter ? vnode.attrs.filter.ast : null,
+        filter: vnode.attrs.filter,
         onChange: onFilterChanged
       }),
       renderTable(

client/drawer-component.js

@@ -5,7 +5,6 @@ import m from "mithril";
 import * as taskQueue from "./task-queue";
 import * as store from "./store";
 import * as notifications from "./notifications";
-import Filter from "../common/filter";
 
 function mparam(param) {
   return m("span.parameter", { title: param }, `${param}`);
@@ -41,7 +40,7 @@ function renderStagingSpv(task, queueFunc, cancelFunc) {
 
 function renderStagingDownload(task) {
   task.invalid = !task.fileName || !task.fileType;
-  const files = store.fetch("files", new Filter());
+  const files = store.fetch("files", true);
   const idParts = task.device.split("-");
   const oui = decodeURIComponent(idParts[0]);
   const productClass =

client/faults-page.js

@@ -3,10 +3,9 @@
 import m from "mithril";
 
 import filterComponent from "./filter-component";
-import * as filterParser from "../common/filter-parser";
-import Filter from "../common/filter";
 import * as store from "./store";
 import * as notifications from "./notifications";
+import * as expression from "../common/expression";
 
 function init(args) {
   if (!window.authorizer.hasAccess("faults", 2))
@@ -14,7 +13,8 @@ function init(args) {
       new Error("You are not authorized to view this page")
     );
 
-  let filter = new Filter(args.filter);
+  let filter = null;
+  if (args.filter != null) filter = expression.parse(`${args.filter}`);
   return Promise.resolve({ filter: filter });
 }
 
@@ -153,16 +153,19 @@ const component = {
 
     function onFilterChanged(filter) {
       let params = {};
-      if (filter) params.filter = filterParser.stringify(filter);
+      if (filter != null) params.filter = expression.stringify(filter);
       m.route.set("/faults", params);
     }
 
     let faults = store.fetch(
       "faults",
-      vnode.attrs.filter,
+      vnode.attrs.filter == null ? true : vnode.attrs.filter,
       vnode.state.showCount || 10
     );
-    let count = store.count("faults", vnode.attrs.filter);
+    let count = store.count(
+      "faults",
+      vnode.attrs.filter == null ? true : vnode.attrs.filter
+    );
 
     let selected = new Set();
     if (vnode.state.selected)
@@ -180,7 +183,7 @@ const component = {
           { parameter: "retries" },
           { parameter: "timestamp" }
         ],
-        filter: vnode.attrs.filter ? vnode.attrs.filter.ast : null,
+        filter: vnode.attrs.filter,
         onChange: onFilterChanged
       }),
       renderTable(faults, count.value, selected, showMore)

client/filter-component.js

@@ -2,7 +2,7 @@
 
 import m from "mithril";
 
-import * as filterParser from "../common/filter-parser";
+import * as expression from "../common/expression";
 
 const component = {
   view: vnode => {
@@ -15,16 +15,14 @@ const component = {
     if (vnode.attrs.filter !== vnode.state.filter) {
       vnode.state.filterList = [];
       vnode.state.filter = vnode.attrs.filter;
-      if (vnode.attrs.filter)
+      if (vnode.attrs.filter != null)
         if (vnode.attrs.filter[0] === "AND")
           for (let i = 1; i < vnode.attrs.filter.length; ++i)
             vnode.state.filterList.push(
-              filterParser.stringify(vnode.attrs.filter[i])
+              expression.stringify(vnode.attrs.filter[i])
             );
         else
-          vnode.state.filterList.push(
+          vnode.state.filterList.push(expression.stringify(vnode.attrs.filter));
-            filterParser.stringify(vnode.attrs.filter)
-          );
       vnode.state.filterList.push("");
     }
 
@@ -33,18 +31,18 @@ const component = {
       vnode.state.filterList = vnode.state.filterList.filter(
         f => f && f.trim()
       );
-      let filterAst = vnode.state.filterList.map((f, idx) => {
+      let filter = vnode.state.filterList.map((f, idx) => {
         try {
-          return filterParser.parse(f);
+          return expression.parse(f);
         } catch (err) {
           vnode.state.filterInvalid |= 1 << idx;
         }
       });
-      if (filterAst.length) filterAst = ["AND"].concat(filterAst);
+      if (filter.length) filter = ["AND"].concat(filter);
-      else filterAst = null;
+      else filter = null;
 
       vnode.state.filterList.push("");
-      if (!vnode.state.filterInvalid) vnode.attrs.onChange(filterAst);
+      if (!vnode.state.filterInvalid) vnode.attrs.onChange(filter);
     }
 
     return m(

client/overview-page.js

@@ -4,8 +4,8 @@ import m from "mithril";
 import config from "./config";
 import * as store from "./store";
 import pieChartComponent from "./pie-chart-component";
-import Filter from "../common/filter";
 import * as funcCache from "../common/func-cache";
+import * as expression from "../common/expression";
 
 const GROUPS = config.ui.overview.groups;
 const CHARTS = {};
@@ -19,7 +19,7 @@ function queryCharts(charts) {
     charts[chartName] = chart = Object.assign({}, chart);
     chart.slices = Object.assign({}, chart.slices);
     for (let [sliceName, slice] of Object.entries(chart.slices)) {
-      const filter = funcCache.get(Filter.parse, slice.filter);
+      const filter = funcCache.get(expression.parse, slice.filter);
       chart.slices[sliceName] = slice = Object.assign({}, slice);
       slice.count = store.count("devices", filter);
     }

client/store.js

@@ -1,15 +1,15 @@
 "use strict";
 
 import m from "mithril";
-import Filter from "../common/filter";
 import config from "./config";
-import * as filterParser from "../common/filter-parser";
+import * as expressionParser from "../common/expression-parser";
+import * as expression from "../common/expression";
 import * as funcCache from "../common/func-cache";
 
 let fulfillTimestamp = 0;
 
-let unpackedFiltersCache = new WeakMap();
+let unpackExpressionCache = new WeakMap();
-let expressionsCache = new WeakMap();
+let evaluateExpressionCache = new WeakMap();
 
 const queries = {
   filter: new WeakMap(),
@@ -47,46 +47,54 @@ class QueryResponse {
   }
 }
 
-function unpackFilter(filter) {
+function unpackExpression(exp) {
-  let f = unpackedFiltersCache.get(filter);
+  if (!Array.isArray(exp)) return exp;
-  if (!f) {
+  let e = unpackExpressionCache.get(exp);
-    f = f.evaluateExpressions(fulfillTimestamp);
+  if (e === undefined) {
-    unpackedFiltersCache.set(filter, f);
+    e = expression.evaluate(e, null, fulfillTimestamp);
+    unpackExpressionCache.set(exp, e);
   }
-  return f;
+  return e;
 }
 
 function count(resourceType, filter) {
-  let queryResponse = resources[resourceType].count.get(filter.toString());
+  const filterStr = funcCache.get(expression.stringify, filter);
+  let queryResponse = resources[resourceType].count.get(filterStr);
   if (queryResponse) return queryResponse;
 
   queryResponse = new QueryResponse();
 
-  resources[resourceType].count.set(filter.toString(), queryResponse);
+  resources[resourceType].count.set(filterStr, queryResponse);
   queries.filter.set(queryResponse, filter);
   return queryResponse;
 }
 
 function limitFilter(resourceType, filter, last) {
   if (resourceType === "devices")
-    return filter.and(new Filter(["<=", "DeviceID.ID", last]));
+    return expression.and(filter, ["<=", "DeviceID.ID", last]);
-  else return filter.and(new Filter(["<=", "_id", last]));
+  else return expression.and(filter, ["<=", ["PARAM", "_id"], last]);
 }
 
 function findMatches(resourceType, filter, limit) {
   // Handle "tag =" and "tag <>" special cases
-  if (resourceType === "devices" && filter.ast) {
+  if (resourceType === "devices")
-    const ast = filterParser.map(filter.ast, e => {
+    filter = expressionParser.map(filter, e => {
-      if (e[1] === "tag")
+      if (
-        if (e[0] === "=") return ["=", `Tags.${e[2]}`, true];
+        Array.isArray(e) &&
-        else if (e[0] === "<>") return ["NOT", ["=", `Tags.${e[2]}`, true]];
+        Array.isArray(e[1]) &&
+        e[1][0] === "PARAM" &&
+        e[1][0] === "tag"
+      )
+        if (e[0] === "=") return ["=", ["PARAM", `Tags.${e[2]}`], true];
+        else if (e[0] === "<>")
+          return ["NOT", ["=", ["PARAM", `Tags.${e[2]}`], true]];
+      return e;
     });
-    filter = new Filter(ast);
-  }
 
   let value = [];
   for (let [id, obj] of resources[resourceType].objects.entries())
-    if (filter.test(obj)) value.push(id);
+    if (expression.evaluate(filter, obj, fulfillTimestamp))
+      value.push(id);
 
   value = value.sort();
   if (limit) value = value.slice(0, limit);
@@ -97,13 +105,13 @@ function findMatches(resourceType, filter, limit) {
 function inferQuery(resourceType, queryResponse) {
   const limit = queries.limit.get(queryResponse);
   let filter = queries.filter.get(queryResponse);
-  filter = unpackFilter(filter);
+  filter = unpackExpression(filter);
   let last = queries.last.get(queryResponse);
   if (last || !limit) {
     if (last) filter = limitFilter(resourceType, filter, last);
     if (
       resources[resourceType].combinedFilter &&
-      filter.subset(resources[resourceType].combinedFilter)
+      expression.subset(filter, resources[resourceType].combinedFilter)
     )
       queries.fulfilled.set(queryResponse, fulfillTimestamp);
   }
@@ -112,7 +120,8 @@ function inferQuery(resourceType, queryResponse) {
 }
 
 function fetch(resourceType, filter, limit = 0) {
-  const key = `${limit}:${filter.toString()}`;
+  const filterStr = funcCache.get(expression.stringify, filter);
+  const key = `${limit}:${filterStr}`;
   let queryResponse = resources[resourceType].fetch.get(key);
   if (queryResponse) return queryResponse;
 
@@ -131,8 +140,8 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
   let updated = false;
 
   if (_fulfillTimestamp > fulfillTimestamp) {
-    unpackedFiltersCache = new WeakMap();
+    unpackExpressionCache = new WeakMap();
-    expressionsCache = new WeakMap();
+    evaluateExpressionCache = new WeakMap();
 
     for (let resource of Object.values(resources))
       resource.combinedFilter = null;
@@ -156,14 +165,14 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
           new Promise((resolve, reject) => {
             updated = true;
             let filter = queries.filter.get(queryResponse);
-            filter = unpackFilter(filter);
+            filter = unpackExpression(filter);
             m
               .request({
                 method: "HEAD",
                 url:
                   `/api/${resourceType}/?` +
                   m.buildQueryString({
-                    filter: filter.toString()
+                    filter: funcCache.get(expression.stringify, filter)
                   }),
                 extract: xhr => +xhr.getResponseHeader("x-total-count"),
                 background: true
@@ -201,14 +210,14 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
             new Promise((resolve, reject) => {
               updated = true;
               let filter = queries.filter.get(queryResponse);
-              filter = unpackFilter(filter);
+              filter = unpackExpression(filter);
               m
                 .request({
                   method: "GET",
                   url:
                     `/api/${resourceType}/?` +
                     m.buildQueryString({
-                      filter: filter.toString(),
+                      filter: funcCache.get(expression.stringify, filter),
                       limit: 1,
                       skip: limit - 1,
                       projection: "_id"
@@ -240,13 +249,13 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
 
           toFetch = toFetch.filter(queryResponse => {
             let filter = queries.filter.get(queryResponse);
-            filter = unpackFilter(filter);
+            filter = unpackExpression(filter);
             let last = queries.last.get(queryResponse);
             if (last) filter = limitFilter(resourceType, filter, last);
 
             if (
               resources[resourceType].combinedFilter &&
-              filter.subset(resources[resourceType].combinedFilter)
+              expression.subset(filter, resources[resourceType].combinedFilter)
             ) {
               queries.value.set(
                 queryResponse,
@@ -257,11 +266,11 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
               return false;
             }
 
-            combinedFilter = filter.or(combinedFilter);
+            combinedFilter = expression.or(combinedFilter, filter);
             return true;
           });
 
-          if (!combinedFilter) return resolve(updated);
+          if (combinedFilter == null) continue;
 
           updated = true;
           let deleted = new Set();
@@ -269,10 +278,12 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
             deleted = new Set(resources[resourceType].objects.keys());
           let combinedFilterDiff = combinedFilter;
           if (resources[resourceType].combinedFilter)
-            combinedFilterDiff = combinedFilterDiff.and(
+            combinedFilterDiff = expression.and(
-              resources[resourceType].combinedFilter.not()
+              combinedFilterDiff,
+              expression.not(resources[resourceType].combinedFilter)
             );
-          resources[resourceType].combinedFilter = combinedFilter.or(
+          resources[resourceType].combinedFilter = expression.or(
+            combinedFilter,
             resources[resourceType].combinedFilter
           );
 
@@ -284,7 +295,10 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
                   url:
                     `/api/${resourceType}/?` +
                     m.buildQueryString({
-                      filter: combinedFilterDiff.toString()
+                      filter: funcCache.get(
+                        expression.stringify,
+                        combinedFilterDiff
+                      )
                     })
                 })
                 .then(res => {
@@ -299,13 +313,19 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
 
                   for (let d of deleted) {
                     const obj = resources[resourceType].objects.get(d);
-                    if (combinedFilterDiff.test(obj))
+                    if (
+                      expression.evaluate(
+                        combinedFilterDiff,
+                        obj,
+                        fulfillTimestamp
+                      )
+                    )
                       resources[resourceType].objects.delete(d);
                   }
 
                   for (let queryResponse of toFetch) {
                     let filter = queries.filter.get(queryResponse);
-                    filter = unpackFilter(filter);
+                    filter = unpackExpression(filter);
                     let last = queries.last.get(queryResponse);
                     if (last) filter = limitFilter(resourceType, filter, last);
 
@@ -379,20 +399,18 @@ function deleteResource(resourceType, id) {
   });
 }
 
-function evaluateExpression(exp, device) {
+function evaluateExpression(exp, obj) {
   if (!Array.isArray(exp)) return exp;
 
-  let exps = expressionsCache.get(exp);
+  let exps = evaluateExpressionCache.get(exp);
-  if (!exps) expressionsCache.set(exp, (exps = new WeakMap()));
+  if (!exps) evaluateExpressionCache.set(exp, (exps = new WeakMap()));
 
-  if (!exps.has(device)) {
+  if (!exps.has(obj)) {
-    let v = filterParser.evaluateExpressions(exp, e => {
+    let v = expression.evaluate(exp, obj, fulfillTimestamp);
-      if (e[0] === "FUNC" && e[1] === "NOW") return fulfillTimestamp;
+    exps.set(obj, v);
-    });
-    exps.set(device, v);
   }
 
-  return exps.get(device);
+  return exps.get(obj);
 }
 
 function logIn(username, password) {
@@ -421,7 +439,7 @@ export {
   count,
   fetch,
   fulfill,
-  unpackFilter,
+  unpackExpression,
   getTimestamp,
   postTasks,
   updateTags,

common/authorizer.js

@@ -1,5 +1,7 @@
 "use strict";
 
+const expression = require("./expression");
+
 const validators = {
   pass: require("./validators/pass"),
   test: require("./validators/test")
@@ -18,16 +20,14 @@ class Authorizer {
       return this.hasAccessCache[cacheKey];
 
     let has = false;
-    for (const permissionSet of this.permissionSets) {
+    for (let permissionSet of this.permissionSets) {
-      for (const perm of permissionSet) {
+      for (let perm of permissionSet)
-        if (perm[resourceType]) {
+        if (perm[resourceType])
           if (perm[resourceType].access >= access) {
             has = true;
             break;
           }
     }
-      }
-    }
 
     this.hasAccessCache[cacheKey] = has;
     return has;

common/expression-cnf.js

@@ -0,0 +1,172 @@
+"use strict";
+
+function collectRanges(exp, ranges = new Map()) {
+  if (["AND", "OR", "NOT"].includes(exp[0]))
+    for (let i = 1; i < exp.length; ++i) collectRanges(exp[i], ranges);
+  else if ([">", ">=", "<", "<=", "=", "<>"].includes(exp[0]))
+    if (!Array.isArray(exp[2])) {
+      const key = `${JSON.stringify(exp[1])}:${typeof exp[2]}`;
+      let s = ranges.get(key);
+      if (!s) ranges.set(key, (s = new Set()));
+      s.add(exp[2]);
+    } else if (!Array.isArray(exp[1])) {
+      const key = `${JSON.stringify(exp[2])}:${typeof exp[1]}`;
+      let s = ranges.get(key);
+      if (!s) ranges.set(key, (s = new Set()));
+      s.add(exp[2]);
+    }
+
+  return ranges;
+}
+
+function or(cnf1, cnf2) {
+  if (cnf1.length === 0) return cnf2;
+  else if (cnf2.length === 0) return cnf1;
+
+  const res = [];
+  for (let i = 0; i < cnf1.length; ++i)
+    for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j]));
+
+  return res;
+}
+
+function booleanCnf(exp) {
+  const ranges = collectRanges(exp);
+  for (let [k, v] of ranges)
+    if (k.endsWith(":number"))
+      ranges.set(k, Array.from(v).sort((a, b) => a - b));
+    else ranges.set(k, Array.from(v).sort());
+
+  const expressions = new Map();
+  const mutuallyExclusive = new Map();
+
+  function getVariable(lhs, op, rhs = "") {
+    const key = `${lhs}:${op}:${rhs}`;
+    let f = expressions.get(key);
+    if (!f) {
+      expressions.set(key, (f = expressions.size + 1));
+      let m = mutuallyExclusive.get(lhs);
+      if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
+      m.add(f);
+      if (rhs || key !== "null") m.add(getVariable(lhs, "null"));
+    }
+    return f;
+  }
+
+  function getComparisonVariables(lhs, from, fromInclusive, to, toInclusive) {
+    const type = from != null ? typeof from : typeof to;
+    const range = ranges.get(`${lhs}:${type}`);
+    const vars = [];
+
+    if (from == null) vars.push(`${lhs}:--${range[0]}`);
+    if (to == null) vars.push(`${lhs}:${range[range.length - 1]}--`);
+    if (fromInclusive) vars.push(`${lhs}:${from}`);
+    if (toInclusive) vars.push(`${lhs}:${to}`);
+
+    for (let i = 1; i < range.length; ++i)
+      if ((from == null || range[i] > from) && (to == null || range[i] <= to)) {
+        vars.push(`${lhs}:${range[i - 1]}`);
+        vars.push(`${lhs}:${range[i]}`);
+        vars.push(`${lhs}:${range[i - 1]}--${range[i]}`);
+      }
+
+    return vars.map(key => {
+      let f = expressions.get(key);
+      if (!f) {
+        expressions.set(key, (f = expressions.size + 1));
+        let m = mutuallyExclusive.get(lhs);
+        if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
+        m.add(f);
+      }
+      return f;
+    });
+  }
+
+  const comparisonOperators = {
+    "=": v => [v, true, v, true],
+    ">": v => [v, false, null, null],
+    ">=": v => [v, true, null, null],
+    "<": v => [null, null, v, false],
+    "<=": v => [null, null, v, true]
+  };
+
+  function recursive(clause, negate) {
+    let op = clause[0];
+
+    if ((op === "AND" && !negate) || (op === "OR" && negate)) {
+      let res = [];
+      for (let i = 1; i < clause.length; ++i)
+        res = res.concat(recursive(clause[i], negate));
+      return res;
+    } else if ((op === "OR" && !negate) || (op === "AND" && negate)) {
+      let res = [];
+      for (let i = 1; i < clause.length; ++i)
+        res = or(res, recursive(clause[i], negate));
+      return res;
+    } else if (op === "NOT") {
+      return recursive(clause[1], !negate);
+    } else if (op === "<>") {
+      const vars = getComparisonVariables(
+        JSON.stringify(clause[1]),
+        ...comparisonOperators["="](clause[2])
+      );
+      if (!negate) return vars.map(x => [0 - x]);
+      else return [vars];
+    } else if (op === "IS NULL") {
+      const v = getVariable(JSON.stringify(clause[1]), "null");
+      if (negate) return [[0 - v]];
+      else return [[v]];
+    } else if (op === "IS NOT NULL") {
+      const v = getVariable(JSON.stringify(clause[1]), "null");
+      if (!negate) return [[0 - v]];
+      else return [[v]];
+    } else if (op in comparisonOperators) {
+      if (!Array.isArray(clause[2])) {
+        const vars = getComparisonVariables(
+          JSON.stringify(clause[1]),
+          ...comparisonOperators[op](clause[2])
+        );
+        if (negate) return vars.map(x => [0 - x]);
+        else return [vars];
+      } else if (!Array.isArray(clause[1])) {
+        op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
+        const vars = getComparisonVariables(
+          JSON.stringify(clause[2]),
+          ...comparisonOperators[op](clause[1])
+        );
+        if (negate) return vars.map(x => [0 - x]);
+        else return [vars];
+      } else {
+        let lhs = JSON.stringify(clause[1]);
+        let rhs = JSON.stringify(clause[2]);
+        // For consistency
+        if (rhs > lhs) {
+          let t = rhs;
+          rhs = lhs;
+          lhs = t;
+          op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
+        }
+
+        const v = getVariable(lhs, op, rhs);
+        if (negate) return [[0 - v]];
+        else return [[v]];
+      }
+    } else {
+      const v = getVariable(JSON.stringify(clause), "");
+      if (negate) return [[0 - v]];
+      else return [[v]];
+    }
+  }
+
+  const cnf = recursive(exp, false);
+
+  for (let m of mutuallyExclusive.values()) {
+    let ar = Array.from(m);
+    for (let i = 0; i < ar.length; ++i)
+      for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]);
+  }
+
+  return { vars: expressions.size, clauses: cnf };
+}
+
+exports.booleanCnf = booleanCnf;

common/expression-parser.js

@@ -22,6 +22,21 @@ function interpretEscapes(str) {
   });
 }
 
+function map(exp, callback) {
+  if (!Array.isArray(exp)) return callback(exp);
+
+  let clone;
+  for (let i = 1; i < exp.length; ++i) {
+    let sub = map(exp[i], callback);
+    if (sub !== exp[i]) {
+      clone = clone || exp.slice();
+      clone[i] = sub;
+    }
+  }
+
+  return callback(clone || exp);
+}
+
 function binaryLeft(operatorsParser, nextParser) {
   return parsimmon.seqMap(
     nextParser,
@@ -29,8 +44,9 @@ function binaryLeft(operatorsParser, nextParser) {
     (first, rest) =>
       rest.reduce((acc, ch) => {
         let [op, another] = ch;
-        if (op === acc[0]) return acc.concat([another]);
+        if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
-        if (op === another[0]) return [op, acc].concat(another.slice(1));
+        if (Array.isArray(another) && op === another[0])
+          return [op, acc].concat(another.slice(1));
         return [op, acc, another];
       }, first)
   );
@@ -61,12 +77,14 @@ const lang = parsimmon.createLanguage({
           .result("IS NOT NULL")
           .desc("IS NOT NULL")
       )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
   },
   NotOperator: function() {
     return parsimmon
       .regexp(/not/i)
       .result("NOT")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace)
       .desc("NOT");
   },
@@ -74,6 +92,7 @@ const lang = parsimmon.createLanguage({
     return parsimmon
       .regexp(/and/i)
       .result("AND")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace)
       .desc("AND");
   },
@@ -81,6 +100,7 @@ const lang = parsimmon.createLanguage({
     return parsimmon
       .regexp(/or/i)
       .result("OR")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace)
       .desc("OR");
   },
@@ -88,13 +108,13 @@ const lang = parsimmon.createLanguage({
     return parsimmon
       .alt(
         parsimmon.regexp(/[a-zA-Z0-9_.*]+/),
-        r.ValueExpression.wrap(
+        r.Expression.wrap(
           parsimmon.string("{").skip(parsimmon.optWhitespace),
           parsimmon.string("}")
         )
       )
       .atLeast(1)
-      .map(x => (x.length > 1 ? ["||"].concat(x) : x[0]))
+      .map(x => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
       .skip(parsimmon.optWhitespace)
       .desc("parameter");
   },
@@ -120,13 +140,38 @@ const lang = parsimmon.createLanguage({
       .map(Number)
       .desc("number");
   },
+  BooleanValue: function() {
+    return parsimmon
+      .alt(
+        parsimmon
+          .regexp(/true/i)
+          .result(true)
+          .desc("TRUE"),
+        parsimmon
+          .regexp(/false/i)
+          .result(false)
+          .desc("FALSE")
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  NullValue: function() {
+    return parsimmon
+      .regexp(/null/i)
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .result(null)
+      .desc("NULL");
+  },
   FuncValue: function(r) {
     return parsimmon.seqMap(
       parsimmon
-        .regexp(/([a-zA-Z0-0_]+)/, 1)
+        .regexp(/([a-zA-Z0-9_]+)/, 1)
         .skip(parsimmon.optWhitespace)
         .desc("function"),
-      r.Value.sepBy(parsimmon.string(",").skip(parsimmon.optWhitespace)).wrap(
+      r.ValueExpression.sepBy(
+        parsimmon.string(",").skip(parsimmon.optWhitespace)
+      ).wrap(
         parsimmon.string("(").skip(parsimmon.optWhitespace),
         parsimmon.string(")").skip(parsimmon.optWhitespace)
       ),
@@ -135,6 +180,8 @@ const lang = parsimmon.createLanguage({
   },
   Value: function(r) {
     return parsimmon.alt(
+      r.NullValue,
+      r.BooleanValue,
       r.NumberValue,
       r.StringValueSql,
       r.StringValueJs,
@@ -143,7 +190,7 @@ const lang = parsimmon.createLanguage({
   },
   ValueExpression: function(r) {
     return binaryLeft(
-      parsimmon.string("||"),
+      parsimmon.string("||").skip(parsimmon.optWhitespace),
       binaryLeft(
         parsimmon
           .alt(parsimmon.string("+"), parsimmon.string("-"))
@@ -154,7 +201,8 @@ const lang = parsimmon.createLanguage({
             .skip(parsimmon.optWhitespace),
           parsimmon.alt(
             r.Value,
-            r.ValueExpression.wrap(
+            r.Parameter,
+            r.Expression.wrap(
               parsimmon.string("(").skip(parsimmon.optWhitespace),
               parsimmon.string(")").skip(parsimmon.optWhitespace)
             )
@@ -167,7 +215,7 @@ const lang = parsimmon.createLanguage({
     return parsimmon.alt(
       parsimmon.seqMap(r.Parameter, r.IsNullOperator, (p, o) => [o, p]),
       parsimmon.seqMap(
-        r.Parameter,
+        r.ValueExpression,
         r.ComparisonOperator,
         r.ValueExpression,
         (p, o, v) => [o, p, v]
@@ -183,129 +231,78 @@ const lang = parsimmon.createLanguage({
       r.OrOperator,
       binaryLeft(
         r.AndOperator,
-        unary(
+        unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
-          r.NotOperator,
-          r.Comparison.or(
-            r.Expression.wrap(
-              parsimmon.string("("),
-              parsimmon.string(")").skip(parsimmon.optWhitespace)
-            )
-          )
-        )
       )
     ).trim(parsimmon.optWhitespace);
   }
 });
 
-function parse(filter) {
+function parse(str) {
-  return lang.Expression.tryParse(filter);
+  if (!str) return null;
+  return lang.Expression.tryParse(str);
 }
 
-function parseExpression(exp) {
+function stringify(exp) {
-  return lang.ValueExpression.tryParse(exp);
+  if (!Array.isArray(exp)) return JSON.stringify(exp);
-}
 
-function evaluateExpressions(ast, funcCallback) {
+  const opLevels = {
-  return map(ast, exp => {
+    OR: 10,
-    if (exp[0] === "FUNC") {
+    AND: 11,
-      return funcCallback(exp);
+    NOT: 12,
-    } else if (exp[0] === "*") {
+    "=": 20,
-      let v = exp[1];
+    "<>": 21,
-      for (let i = 2; i < exp.length; ++i) v *= exp[i];
+    ">": 22,
-      return v;
+    ">=": 23,
-    } else if (exp[0] === "/") {
+    "<": 24,
-      let v = exp[1];
+    "<=": 25,
-      for (let i = 2; i < exp.length; ++i) v /= exp[i];
+    "||": 30,
-      return v;
+    "+": 31,
-    } else if (exp[0] === "+") {
+    "-": 31,
-      let v = exp[1];
+    "*": 32,
-      for (let i = 2; i < exp.length; ++i) v += exp[i];
+    "/": 32
-      return v;
+  };
-    } else if (exp[0] === "-") {
-      let v = exp[1];
-      for (let i = 2; i < exp.length; ++i) v -= exp[i];
-      return v;
-    } else if (exp[0] === "||") {
-      return exp.slice(1).join("");
-    }
-  });
-}
 
-function parseParameter(p) {
+  const op = exp[0].toUpperCase();
-  return lang.Parameter.tryParse(p);
-}
 
-function stringify(filter) {
+  if (op === "FUNC") {
-  function value(v) {
+    return `${exp[1]}(${exp
-    if (Array.isArray(v))
-      if (v[0].toUpperCase() === "FUNC") {
-        return `${v[1]}(${v
       .slice(2)
-          .map(e => value(e))
+      .map(e => stringify(e))
       .join(", ")})`;
-      } else {
+  } else if (op === "PARAM") {
-        return v
+    if (typeof exp[1] === "string") return exp[1];
+    else if (Array.isArray(exp[1]) && exp[1][0] === "||")
+      return exp[1]
         .slice(1)
-          .map(e => value(e))
+        .map(p => {
-          .join(` ${v[0]} `);
+          if (typeof p === "string") return p;
-      }
+          else return `{${stringify(p)}}`;
-
-    return JSON.stringify(v);
-  }
-
-  function expressions(v) {
-    return v.map(e => {
-      let str = stringify(e);
-      if (e[0] === "AND" || e[0] === "OR") return `(${str})`;
-      else return str;
-    });
-  }
-
-  function parameter(v) {
-    if (Array.isArray(v) && v[0] === "||")
-      return v
-        .slice(1)
-        .map(x => {
-          if (Array.isArray(x)) return `{${value(x)}}`;
-          else return x;
         })
         .join("");
+    else return `{${stringify(exp[1])}}`;
+  } else if (op === "IS NULL" || op === "IS NOT NULL") {
+    return `${stringify(exp[1])} ${op}`;
+  } else if (op in opLevels) {
+    const parts = exp.slice(1).map((e, i) => {
+      if (
+        Array.isArray(e) &&
+        (opLevels[e[0]] < opLevels[exp[0]] ||
+          (opLevels[e[0]] < opLevels[exp[0]] && i > 0) ||
+          (opLevels[exp[0]] >= 20 &&
+            opLevels[e[0]] >= 20 &&
+            opLevels[e[0]] < 30))
+      )
+        return `(${stringify(e)})`;
+      else return stringify(e);
+    });
 
-    return v;
+    if (op === "NOT") return `${op} ${parts[0]}`;
+    else return parts.join(` ${op} `);
+  } else {
+    throw new Error(`Unrecognized operator ${exp[0]}`);
   }
-
-  const op = filter[0].toUpperCase();
-
-  if (["AND", "OR"].includes(op))
-    return expressions(filter.slice(1)).join(` ${op} `);
-  else if (op === "NOT") return `NOT ${expressions(filter.slice(1))[0]}`;
-  else if ([">=", "<>", "<=", "=", ">", "<"].includes(op))
-    return `${parameter(filter[1])} ${op} ${value(filter[2])}`;
-  else if (["IS NULL", "IS NOT NULL"].includes(op))
-    return `${parameter(filter[1])} ${op}`;
-  else throw new Error("Unrecognized operator");
-}
-
-function map(filter, callback) {
-  let clone;
-  for (let i = 1; i < filter.length; ++i)
-    if (Array.isArray(filter[i])) {
-      let sub = map(filter[i], callback);
-      if (sub != null && sub !== filter[i]) {
-        clone = clone || filter.slice();
-        clone[i] = sub;
-      }
-    }
-
-  let r = callback(clone || filter);
-  if (r != null) return r;
-
-  return clone || filter;
 }
 
 exports.parse = parse;
-exports.parseExpression = parseExpression;
-exports.evaluateExpressions = evaluateExpressions;
-exports.parseParameter = parseParameter;
 exports.stringify = stringify;
 exports.map = map;

common/expression.js

@@ -0,0 +1,188 @@
+"use strict";
+
+const expressionParser = require("./expression-parser");
+const expressionCnf = require("./expression-cnf");
+const satSolver = require("./sat-solver");
+
+const isArray = Array.isArray;
+
+function* permute(arr) {
+  if (arr.length <= 1) {
+    for (let i = 0; i < arr[0]; ++i) yield [i];
+    return;
+  }
+
+  let slc = arr.slice(1);
+  for (let i = 0; i < arr[0]; ++i)
+    for (let innerArr of permute(slc)) yield [i].concat(innerArr);
+}
+
+function reduce(exp, callback) {
+  let loop = true;
+  while (loop) {
+    loop = false;
+    for (let i = 2; i < exp.length; ++i) {
+      let res = callback(exp[i - 1], exp[i]);
+      if (res !== undefined) {
+        loop = true;
+        exp = exp.slice();
+        exp.splice(i - 1, 2, res);
+      }
+    }
+  }
+  if (exp.length === 2) return exp[1];
+  return exp;
+}
+
+function evaluate(exp, obj, now) {
+  return expressionParser.map(exp, e => {
+    if (!isArray(e)) return e;
+
+    if (e[0] === "FUNC") {
+      if (e[1] === "NOW") {
+        if (now) return now;
+      } else if (e[1] === "UPPER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
+      } else if (e[1] === "LOWER") {
+        if (e[12] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
+      }
+    } else if (e[0] === "PARAM") {
+      if (e[1] == null) return null;
+      if (obj && !isArray(e[1]))
+        if (typeof obj[e[1]] === "object")
+          return obj[e[1]].value ? obj[e[1]].value[0] : null;
+        else return obj[e[1]];
+    } else if (e[0] === "AND") {
+      return reduce(e, (a, b) => {
+        if (!isArray(a)) return a ? b : a;
+        if (!isArray(b)) return b ? a : b;
+      });
+    } else if (e[0] === "OR") {
+      return reduce(e, (a, b) => {
+        if (!isArray(a)) return a ? a : b;
+        if (!isArray(b)) return b ? b : a;
+      });
+    } else if (e[0] === "NOT") {
+      if (!isArray(e[1])) return !e[1];
+      else if (e[1][0] === "NOT") return e[1][1];
+    } else if (e[0] === "IS NULL") {
+      if (isArray(e[1])) return e;
+      return e[1] == null;
+    } else if (e[0] === "IS NOT NULL") {
+      if (isArray(e[1])) return e;
+      return e[1] != null;
+    } else if (e[0] === "=") {
+      if (isArray(e[1]) || isArray(e[2])) return e;
+      if (e[1] == null || e[2] == null) return null;
+      return e[1] === e[2];
+    } else if (e[0] === "<>") {
+      if (isArray(e[1]) || isArray(e[2])) return e;
+      if (e[1] == null || e[2] == null) return null;
+      return e[1] !== e[2];
+    } else if (e[0] === ">") {
+      if (isArray(e[1]) || isArray(e[2])) return e;
+      if (e[1] == null || e[2] == null) return null;
+      return e[1] > e[2];
+    } else if (e[0] === ">=") {
+      if (isArray(e[1]) || isArray(e[2])) return e;
+      if (e[1] == null || e[2] == null) return null;
+      return e[1] >= e[2];
+    } else if (e[0] === "<") {
+      if (isArray(e[1]) || isArray(e[2])) return e;
+      if (e[1] == null || e[2] == null) return null;
+      return e[1] < e[2];
+    } else if (e[0] === "<=") {
+      if (isArray(e[1]) || isArray(e[2])) return e;
+      if (e[1] == null || e[2] == null) return null;
+      return e[1] <= e[2];
+    } else if (e[0] === "*") {
+      return reduce(e, (a, b) => {
+        if (!isArray(a) && !isArray(b)) {
+          if (a == null || b == null) return null;
+          return a * b;
+        }
+      });
+    } else if (e[0] === "/") {
+      return reduce(e, (a, b) => {
+        if (!isArray(a) && !isArray(b)) {
+          if (a == null || b == null) return null;
+          return a / b;
+        }
+      });
+    } else if (e[0] === "+") {
+      return reduce(e, (a, b) => {
+        if (!isArray(a) && !isArray(b)) {
+          if (a == null || b == null) return null;
+          return a + b;
+        }
+      });
+    } else if (e[0] === "-") {
+      return reduce(e, (a, b) => {
+        if (!isArray(a) && !isArray(b)) {
+          if (a == null || b == null) return null;
+          return a - b;
+        }
+      });
+    } else if (e[0] === "||") {
+      return reduce(e, (a, b) => {
+        if (!isArray(a) && !isArray(b)) {
+          if (a == null || b == null) return null;
+          return `${a}${b}`;
+        }
+      });
+    }
+    return e;
+  });
+}
+
+function and(exp1, exp2) {
+  if (!isArray(exp1)) return exp1 ? exp2 : exp1;
+  if (!isArray(exp2)) return exp2 ? exp1 : exp2;
+
+  let res = ["AND"];
+
+  if (exp1[0] === "AND") res = res.concat(exp1.slice(1));
+  else res.push(exp1);
+
+  if (exp2[0] === "AND") res = res.concat(exp2.slice(1));
+  else res.push(exp2);
+
+  return res;
+}
+
+function or(exp1, exp2) {
+  if (!isArray(exp1)) return exp1 ? exp1 : exp2;
+  if (!isArray(exp2)) return exp2 ? exp2 : exp1;
+
+  let res = ["OR"];
+
+  if (exp1[0] === "OR") res = res.concat(exp1.slice(1));
+  else res.push(exp1);
+
+  if (exp2[0] === "OR") res = res.concat(exp2.slice(1));
+  else res.push(exp2);
+
+  return res;
+}
+
+function not(exp) {
+  if (isArray(exp) && exp[0] === "NOT") return exp[1];
+  return ["NOT", exp];
+}
+
+function subset(exp1, exp2) {
+  const e = evaluate(["NOT", ["OR", ["NOT", exp1], exp2]]);
+  if (!isArray(e)) return !e;
+  const { vars, clauses } = expressionCnf.booleanCnf(e);
+  return !satSolver.naiveDpll(clauses, vars);
+}
+
+exports.evaluate = evaluate;
+exports.and = and;
+exports.or = or;
+exports.not = not;
+exports.subset = subset;
+exports.parse = expressionParser.parse;
+exports.stringify = expressionParser.stringify;

common/filter-cnf.js

@@ -1,138 +0,0 @@
-"use strict";
-
-function collectRanges(filter, ranges = new Map()) {
-  if (["AND", "OR", "NOT"].includes(filter[0])) {
-    for (let i = 1; i < filter.length; ++i) collectRanges(filter[i], ranges);
-  } else if ([">", ">=", "<", "<=", "=", "<>"].includes(filter[0])) {
-    const key = `${filter[1]}:${typeof filter[2]}`;
-    let s = ranges.get(key);
-    if (!s) ranges.set(key, (s = new Set()));
-    s.add(filter[2]);
-  }
-  return ranges;
-}
-
-function or(cnf1, cnf2) {
-  if (cnf1.length === 0) return cnf2;
-  else if (cnf2.length === 0) return cnf1;
-
-  const res = [];
-  for (let i = 0; i < cnf1.length; ++i)
-    for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j]));
-
-  return res;
-}
-
-function booleanCnf(filter) {
-  const ranges = collectRanges(filter);
-  for (let [k, v] of ranges)
-    if (k.endsWith(":number"))
-      ranges.set(k, Array.from(v).sort((a, b) => a - b));
-    else ranges.set(k, Array.from(v).sort());
-
-  const expressions = new Map();
-  const mutuallyExclusive = new Map();
-
-  function getIsNullVariable(param) {
-    const key = `${param}:null`;
-    let f = expressions.get(key);
-    if (!f) {
-      expressions.set(key, (f = expressions.size + 1));
-      let m = mutuallyExclusive.get(param);
-      if (!m) mutuallyExclusive.set(param, (m = new Set()));
-      m.add(f);
-    }
-    return f;
-  }
-
-  function getComparisonVariables(param, from, fromInclusive, to, toInclusive) {
-    const type = from != null ? typeof from : typeof to;
-    const range = ranges.get(`${param}:${type}`);
-    const vars = [];
-
-    if (from == null) vars.push(`${param}:--${range[0]}`);
-    if (to == null) vars.push(`${param}:${range[range.length - 1]}--`);
-    if (fromInclusive) vars.push(`${param}:${from}`);
-    if (toInclusive) vars.push(`${param}:${to}`);
-
-    for (let i = 1; i < range.length; ++i)
-      if ((from == null || range[i] > from) && (to == null || range[i] <= to)) {
-        vars.push(`${param}:${range[i - 1]}`);
-        vars.push(`${param}:${range[i]}`);
-        vars.push(`${param}:${range[i - 1]}--${range[i]}`);
-      }
-
-    return vars.map(key => {
-      let f = expressions.get(key);
-      if (!f) {
-        expressions.set(key, (f = expressions.size + 1));
-        let m = mutuallyExclusive.get(param);
-        if (!m) mutuallyExclusive.set(param, (m = new Set()));
-        m.add(f);
-      }
-      return f;
-    });
-  }
-
-  const comparisonOperators = {
-    "=": v => [v, true, v, true],
-    ">": v => [v, false, null, null],
-    ">=": v => [v, true, null, null],
-    "<": v => [null, null, v, false],
-    "<=": v => [null, null, v, true]
-  };
-
-  function recursive(clause, negate) {
-    const op = clause[0];
-
-    if ((op === "AND" && !negate) || (op === "OR" && negate)) {
-      let res = [];
-      for (let i = 1; i < clause.length; ++i)
-        res = res.concat(recursive(clause[i], negate));
-      return res;
-    } else if ((op === "OR" && !negate) || (op === "AND" && negate)) {
-      let res = [];
-      for (let i = 1; i < clause.length; ++i)
-        res = or(res, recursive(clause[i], negate));
-      return res;
-    } else if (op === "NOT") {
-      return recursive(clause[1], !negate);
-    } else if (op in comparisonOperators) {
-      const vars = getComparisonVariables(
-        clause[1],
-        ...comparisonOperators[op](clause[2])
-      );
-      if (negate) return vars.map(x => [0 - x]);
-      else return [vars];
-    } else if (op === "<>") {
-      const vars = getComparisonVariables(
-        clause[1],
-        ...comparisonOperators["="](clause[2])
-      );
-      if (!negate) return vars.map(x => [0 - x]);
-      else return [vars];
-    } else if (op === "IS NULL") {
-      const v = getIsNullVariable(clause[1]);
-      if (negate) return [[0 - v]];
-      else return [[v]];
-    } else if (op === "IS NOT NULL") {
-      const v = getIsNullVariable(clause[1]);
-      if (!negate) return [[0 - v]];
-      else return [[v]];
-    } else {
-      throw new Error(`Unknown operator ${op}`);
-    }
-  }
-
-  const cnf = recursive(filter, false);
-
-  for (let m of mutuallyExclusive.values()) {
-    let ar = Array.from(m);
-    for (let i = 0; i < ar.length; ++i)
-      for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]);
-  }
-
-  return { vars: expressions.size, clauses: cnf };
-}
-
-exports.booleanCnf = booleanCnf;

common/filter.js

@@ -1,145 +0,0 @@
-"use strict";
-
-const filterParser = require("./filter-parser");
-const filterCnf = require("./filter-cnf");
-const satSolver = require("./sat-solver");
-const funcCache = require("./func-cache");
-
-function* permute(arr) {
-  if (arr.length <= 1) {
-    for (let i = 0; i < arr[0]; ++i) yield [i];
-    return;
-  }
-
-  let slc = arr.slice(1);
-  for (let i = 0; i < arr[0]; ++i)
-    for (let innerArr of permute(slc)) yield [i].concat(innerArr);
-}
-
-class Filter {
-  constructor(str) {
-    if (!str) this.ast = null;
-    else if (Array.isArray(str)) this.ast = str;
-    else if (str instanceof Filter) this.ast = str.ast;
-    else if (Array.isArray(str.ast) || str.ast === null) this.ast = str.ast;
-    else this.ast = filterParser.parse(str);
-  }
-
-  static parse(str) {
-    return new Filter(str);
-  }
-
-  toString() {
-    if (!this.string)
-      if (this.ast) this.string = filterParser.stringify(this.ast);
-      else this.string = "";
-
-    return this.string;
-  }
-
-  or(fltr) {
-    if (!fltr || !this.ast) return this;
-    else if (!fltr.ast) return fltr;
-
-    let f = ["OR"];
-
-    if (this.ast[0] === "OR") f = f.concat(this.ast.slice(1));
-    else f.push(this.ast);
-
-    if (fltr.ast[0] === "OR") f = f.concat(fltr.ast.slice(1));
-    else f.push(fltr.ast);
-
-    return new Filter(f);
-  }
-
-  and(fltr) {
-    if (!fltr || !fltr.ast) return this;
-    else if (!this.ast) return fltr;
-
-    let f = ["AND"];
-
-    if (this.ast[0] === "AND") f = f.concat(this.ast.slice(1));
-    else f.push(this.ast);
-
-    if (fltr.ast[0] === "AND") f = f.concat(fltr.ast.slice(1));
-    else f.push(fltr.ast);
-
-    return new Filter(f);
-  }
-
-  not() {
-    if (this.ast[0] === "NOT") return new Filter(this.ast[1]);
-    else return new Filter(["NOT", this.ast]);
-  }
-
-  evaluateExpressions(now = Date.now()) {
-    if (!this.ast) return this;
-
-    let ast = filterParser.evaluateExpressions(this.ast, exp => {
-      if (exp[0] === "FUNC" && exp[1] === "NOW") return now;
-    });
-
-    if (ast === this.ast) return this;
-    else return new Filter(ast);
-  }
-
-  subset(fltr) {
-    if (!fltr.ast) return true;
-    else if (!this.ast) return false;
-
-    const f = this.not()
-      .or(fltr)
-      .not();
-    const { vars, clauses } = filterCnf.booleanCnf(f.ast);
-    return !satSolver.naiveDpll(clauses, vars);
-  }
-
-  test(obj) {
-    if (!this.ast) return true;
-    return filterParser.map(this.ast, exp => {
-      const op = exp[0];
-
-      if (op === "AND") {
-        for (let i = 1; i < exp.length; ++i) if (!exp[i]) return false;
-        return true;
-      } else if (op === "OR") {
-        for (let i = 1; i < exp.length; ++i) if (exp[i]) return true;
-        return false;
-      } else if (op === "NOT") {
-        return !exp[1];
-      } else if (op === "=") {
-        const val =
-          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
-        return val === exp[2];
-      } else if (op === "<>") {
-        const val =
-          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
-        return val !== exp[2];
-      } else if (op === ">=") {
-        const val =
-          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
-        return typeof val === typeof exp[2] && val >= exp[2];
-      } else if (op === ">") {
-        const val =
-          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
-        return typeof val === typeof exp[2] && val > exp[2];
-      } else if (op === "<=") {
-        const val =
-          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
-        return typeof val === typeof exp[2] && val <= exp[2];
-      } else if (op === "<") {
-        const val =
-          typeof obj[exp[1]] === "object" ? obj[exp[1]].value[0] : obj[exp[1]];
-        return typeof val === typeof exp[2] && val < exp[2];
-      } else if (op === "IS NULL") {
-        return obj[exp[1]] == null;
-      } else if (op === "IS NOT NULL") {
-        return obj[exp[1]] != null;
-      } else {
-        throw new Error(`Unsupported operator ${op}`);
-      }
-    });
-  }
-}
-
-module.exports = Filter;

common/func-cache.js

@@ -22,6 +22,8 @@ function purge() {
   cache1 = new WeakMap();
 }
 
+setInterval(purge, 120000);
+
 exports.get = get;
 exports.getter = getter;
 exports.purge = purge;

server/api-functions.js

@@ -7,20 +7,22 @@ const querystring = require("querystring");
 
 const config = require("./config");
 const device = require("./device");
-const filterParser = require("../common/filter-parser");
+const expressionParser = require("../common/expression-parser");
-const Filter = require("../common/filter");
+const expression = require("../common/expression");
 
 const commaAscii = ",".charCodeAt(0);
 const newLineAscii = "\n".charCodeAt(0);
 
 function unpackTimestamps(filter) {
-  return filterParser.map(filter, exp => {
+  if (!Array.isArray(filter)) return filter;
+  return expressionParser.map(filter, exp => {
     if (["=", "<>", ">=", "<=", "<", ">"].includes(exp[0]))
-      if (typeof exp[2] === "number") {
+      if (Array.isArray(exp) && typeof exp[2] === "number") {
         let alt = exp.slice();
         alt[2] = new Date(alt[2]).toJSON();
         return ["OR", exp, alt];
       }
+    return exp;
   });
 }
 
@@ -39,37 +41,42 @@ function filterToMongoQuery(filter, negate = false, res = {}) {
   } else if (op === "NOT") {
     filterToMongoQuery(filter[1], !negate, res);
   } else if (op === "=") {
-    let p = (res[filter[1]] = res[filter[1]] || {});
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
     if (negate) p["$ne"] = filter[2];
     else p["$eq"] = filter[2];
   } else if (op === "<>") {
-    let p = (res[filter[1]] = res[filter[1]] || {});
+    const param = filter[1][1];
-    if (negate) {
+    let p = (res[param] = res[param] || {});
-      p["$eq"] = filter[2];
+    if (negate) p = p["$not"] = p["$not"] || {};
-    } else {
     p["$ne"] = filter[2];
     p["$exists"] = true;
-    }
   } else if (op === ">") {
-    let p = (res[filter[1]] = res[filter[1]] || {});
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
     if (negate) p = p["$not"] = p["$not"] || {};
     p["$gt"] = filter[2];
   } else if (op === ">=") {
-    let p = (res[filter[1]] = res[filter[1]] || {});
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
     if (negate) p = p["$not"] = p["$not"] || {};
     p["$gte"] = filter[2];
   } else if (op === "<") {
-    let p = (res[filter[1]] = res[filter[1]] || {});
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
     if (negate) p = p["$not"] = p["$not"] || {};
     p["$lt"] = filter[2];
   } else if (op === "<=") {
-    let p = (res[filter[1]] = res[filter[1]] || {});
+    const param = filter[1][1];
+    let p = (res[param] = res[param] || {});
     if (negate) p = p["$not"] = p["$not"] || {};
     p["$lte"] = filter[2];
   } else if (op === "IS NULL") {
-    res[filter[1]] = { $exists: negate };
+    const param = filter[1][1];
+    res[param] = { $exists: negate };
   } else if (op === "IS NOT NULL") {
-    res[filter[1]] = { $exists: !negate };
+    const param = filter[1][1];
+    res[param] = { $exists: !negate };
   } else {
     throw new Error(`Unrecognized operator ${op}`);
   }
@@ -80,14 +87,15 @@ function filterToMongoQuery(filter, negate = false, res = {}) {
 function count(resource, filter, limit) {
   return new Promise((resolve, reject) => {
     let qs = {};
-    if (filter && filter.ast) {
+    filter = expression.evaluate(filter, null, Date.now());
-      filter = filter.evaluateExpressions();
+    filter = unpackTimestamps(filter);
-      let ast = unpackTimestamps(filter.ast);
+
+    let q = {};
+    if (Array.isArray(filter)) q = filterToMongoQuery(filter);
+    else if (filter != null && !filter) return resolve(0);
 
-      let q = filterToMongoQuery(ast);
     if (resource === "devices") q = device.transposeQuery(q);
     qs.query = JSON.stringify(q);
-    }
 
     if (limit) qs.limit = limit;
 
@@ -117,15 +125,15 @@ function query(resource, filter, limit, skip, projection, callback) {
     let ret;
     if (!callback) ret = [];
     let qs = {};
-    if (filter && filter.ast) {
+    filter = expression.evaluate(filter, null, Date.now());
-      filter = filter.evaluateExpressions();
+    filter = unpackTimestamps(filter);
-      let ast = unpackTimestamps(filter.ast);
+
+    let q = {};
+    if (Array.isArray(filter)) q = filterToMongoQuery(filter);
+    else if (filter != null && !filter) return resolve(ret);
 
-      let q = filterToMongoQuery(ast);
     if (resource === "devices") q = device.transposeQuery(q);
-
     qs.query = JSON.stringify(q);
-    }
 
     if (limit) qs.limit = limit;
     if (skip) qs.skip = skip;
@@ -308,12 +316,13 @@ function postTasks(deviceId, tasks) {
 
         let promises2 = [];
         for (let s of statuses) {
-          promises2.push(query("tasks", new Filter(["=", "_id", s._id])));
+          promises2.push(query("tasks", ["=", ["PARAM", "_id"], s._id]));
           promises2.push(
-            query(
+            query("faults", [
-              "faults",
+              "=",
-              new Filter(["=", "_id", `${deviceId}:task_${s._id}`])
+              ["PARAM", "_id"],
-            )
+              `${deviceId}:task_${s._id}`
+            ])
           );
         }
 

server/api.js

@@ -4,7 +4,7 @@ const Router = require("koa-router");
 
 const db = require("./db");
 const apiFunctions = require("./api-functions");
-const Filter = require("../common/filter");
+const expression = require("../common/expression");
 
 const router = new Router();
 
@@ -33,17 +33,20 @@ const resources = {
 
 for (let [resource, flags] of Object.entries(resources)) {
   router.head(`/${resource}`, async (ctx, next) => {
-    let filter, limit;
+    let filter = true;
-    if (ctx.request.query.filter) filter = new Filter(ctx.request.query.filter);
+    let limit;
+    if (ctx.request.query.filter)
+      filter = expression.parse(ctx.request.query.filter);
     if (ctx.request.query.limit) limit = +ctx.request.query.limit;
 
     if (!ctx.state.authorizer.hasAccess(resource, 1)) return next();
 
     // Exclude temporary tasks and faults
-    if (resource === "tasks" || resource === "faults") {
+    if (resource === "tasks" || resource === "faults")
-      let f = new Filter(["NOT", ["<", "expiry", Date.now() + 60000]]);
+      filter = expression.and(filter, [
-      filter = f.and(filter);
+        "NOT",
-    }
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
 
     let count;
     if (flags & RESOURCE_DB) count = await db.count(resource, filter, limit);
@@ -54,8 +57,10 @@ for (let [resource, flags] of Object.entries(resources)) {
   });
 
   router.get(`/${resource}`, async (ctx, next) => {
-    let filter, limit, skip, projection;
+    let filter = true;
-    if (ctx.request.query.filter) filter = new Filter(ctx.request.query.filter);
+    let limit, skip, projection;
+    if (ctx.request.query.filter)
+      filter = expression.parse(ctx.request.query.filter);
     if (ctx.request.query.limit) limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) skip = +ctx.request.query.skip;
     if (ctx.request.query.projection) projection = ctx.request.query.projection;
@@ -63,10 +68,11 @@ for (let [resource, flags] of Object.entries(resources)) {
     if (!ctx.state.authorizer.hasAccess(resource, 2)) return next();
 
     // Exclude temporary tasks and faults
-    if (resource === "tasks" || resource === "faults") {
+    if (resource === "tasks" || resource === "faults")
-      let f = new Filter(["NOT", ["<", "expiry", Date.now() + 60000]]);
+      filter = expression.and(filter, [
-      filter = f.and(filter);
+        "NOT",
-    }
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+      ]);
 
     let res;
     if (flags & RESOURCE_DB)
@@ -78,7 +84,7 @@ for (let [resource, flags] of Object.entries(resources)) {
   });
 
   router.head(`/${resource}/:id`, async (ctx, next) => {
-    let filter = new Filter(`${RESOURCE_IDS[resource]} = "${ctx.params.id}"`);
+    let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
     if (!ctx.state.authorizer.hasAccess(resource, 2)) return next();
 
     let res;
@@ -90,7 +96,7 @@ for (let [resource, flags] of Object.entries(resources)) {
   });
 
   router.get(`/${resource}/:id`, async (ctx, next) => {
-    let filter = new Filter(`${RESOURCE_IDS[resource]} = "${ctx.params.id}"`);
+    let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
     if (!ctx.state.authorizer.hasAccess(resource, 2)) return next();
 
     let res;
@@ -104,7 +110,7 @@ for (let [resource, flags] of Object.entries(resources)) {
   if (flags & RESOURCE_DELETE && !(flags & RESOURCE_DB))
     router.delete(`/${resource}/:id`, async (ctx, next) => {
       const authorizer = ctx.state.authorizer;
-      let filter = new Filter(`${RESOURCE_IDS[resource]} = "${ctx.params.id}"`);
+      let filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
       if (!authorizer.hasAccess(resource, 2)) return next();
       let res = await apiFunctions.query(resource, filter);
       if (!res.length) return next();
@@ -119,7 +125,7 @@ for (let [resource, flags] of Object.entries(resources)) {
 
 router.post("/devices/:id/tasks", async (ctx, next) => {
   const authorizer = ctx.state.authorizer;
-  let filter = new Filter(`DeviceID.ID = "${ctx.params.id}"`);
+  let filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
   if (!authorizer.hasAccess("devices", 2)) return next();
   let devices = await apiFunctions.query("devices", filter);
   if (!devices.length) return next();
@@ -135,7 +141,7 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
 
 router.post("/devices/:id/tags", async (ctx, next) => {
   const authorizer = ctx.state.authorizer;
-  let filter = new Filter(`DeviceID.ID = "${ctx.params.id}"`);
+  let filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
   if (!authorizer.hasAccess("devices", 2)) return next();
   let res = await apiFunctions.query("devices", filter);
   if (!res.length) return next();

server/db.js

@@ -20,7 +20,8 @@ function getClient() {
 
 function query(resource, filter, limit, skip, callback) {
   let q;
-  if (filter && filter.ast) q = apiFunctions.filterToMongoQuery(filter.ast);
+  if (Array.isArray(filter)) q = apiFunctions.filterToMongoQuery(filter);
+  else if (filter != null && !filter) return Promise.resolve([]);
 
   return new Promise((resolve, reject) => {
     getClient().then(client => {
@@ -50,7 +51,8 @@ function query(resource, filter, limit, skip, callback) {
 
 function count(resource, filter) {
   let q;
-  if (filter && filter.ast) q = apiFunctions.filterToMongoQuery(filter.ast);
+  if (Array.isArray(filter)) q = apiFunctions.filterToMongoQuery(filter);
+  else if (filter != null && !filter) return Promise.resolve([]);
 
   return new Promise((resolve, reject) => {
     getClient().then(client => {

server/server.js

@@ -11,6 +11,7 @@ const jwt = require("jsonwebtoken");
 const config = require("./config");
 const api = require("./api");
 const Authorizer = require("../common/authorizer");
+const expression = require("../common/expression");
 
 const koa = new Koa();
 const router = new Router();
@@ -20,7 +21,20 @@ const JWT_COOKIE = "genieacs-ui-jwt";
 
 function getPermissionSets(roles) {
   const allPermissions = config.permissions;
-  const permissionSets = roles.map(r => Object.values(allPermissions[r] || {}));
+  const permissionSets = roles.map(role => {
+    return Object.values(allPermissions[role] || {}).map(p => {
+      p = Object.assign({}, p);
+      for (let [k, v] of Object.entries(p)) {
+        p[k] = v = Object.assign({}, v);
+        if (["devices", "faults", "files"].includes(k))
+          if (v.filter == null || v.filter === "") v.filter = true;
+          else {
+            v.filter = expression.parse(v.filter);
+          }
+      }
+      return p;
+    });
+  });
   return permissionSets;
 }