Replace Filter class with expression structure

This is in order to be able to support UPPER() and LOWER() functions in
queries which are needed for case-insensitive search.

client/app.js

@@ -11,7 +11,6 @@ import * as devicePage from "./device-page";
 import * as errorPage from "./error-page";
 import * as faultsPage from "./faults-page";
 import Authorizer from "../common/authorizer";
-import * as funcCache from "../common/func-cache";
 
 window.authorizer = new Authorizer(window.permissionSets);
 
@@ -69,8 +68,6 @@ function pagify(pageName, page) {
   return component;
 }
 
-setInterval(funcCache.purge, 120000);
-
 m.route(document.body, "/overview", {
   "/login": pagify("login", loginPage),
   "/overview": pagify("overview", overviewPage),

client/components/container.js

@@ -3,16 +3,14 @@
 import m from "mithril";
 import * as components from "../components";
 import * as store from "../store";
-import Filter from "../../common/filter";
+import * as expression from "../../common/expression";
 import * as funcCache from "../../common/func-cache";
 
 const component = {
   view: vnode => {
     if (vnode.attrs.filter) {
-      const filter = store.unpackFilter(
-        funcCache.get(Filter.parse, vnode.attrs.filter)
-      );
-      if (!filter.test(vnode.attrs.device)) return;
+      const filter = funcCache.get(expression.parse, vnode.attrs.filter);
+      if (!store.evaluateExpression(filter, vnode.attrs.device)) return;
     }
 
     let children = Object.values(vnode.attrs.components).map(c => {

client/components/device-faults.js

@@ -2,21 +2,17 @@
 
 import m from "mithril";
 import * as store from "../store";
-import Filter from "../../common/filter";
 import * as notifications from "../notifications";
 
 const component = {
   view: vnode => {
     const device = vnode.attrs.device;
     const deviceId = device["DeviceID.ID"].value[0];
-    const faults = store.fetch(
-      "faults",
-      new Filter([
-        "AND",
-        [">", "_id", `${deviceId}:`],
-        ["<", "_id", `${deviceId}:zzzz`]
-      ])
-    );
+    const faults = store.fetch("faults", [
+      "AND",
+      [">", ["PARAM", "_id"], `${deviceId}:`],
+      ["<", ["PARAM", "_id"], `${deviceId}:zzzz`]
+    ]);
 
     const headers = ["Channel", "Code", "Message", "Retries", "Timestamp"].map(
       l => m("th", l)

client/components/overview-dot.js

@@ -4,7 +4,7 @@ import m from "mithril";
 import config from "../config";
 import * as store from "../store";
 import * as funcCache from "../../common/func-cache";
-import Filter from "../../common/filter";
+import * as expression from "../../common/expression";
 
 const CHARTS = config.ui.overview.charts;
 
@@ -13,10 +13,8 @@ const component = {
     const device = vnode.attrs.device;
     const chart = CHARTS[vnode.attrs.chart];
     for (let slice of Object.values(chart.slices)) {
-      const filter = store.unpackFilter(
-        funcCache.get(Filter.parse, slice.filter)
-      );
-      if (filter.test(device)) {
+      const filter = funcCache.get(expression.parse, slice.filter);
+      if (store.evaluateExpression(filter, device)) {
         const dot = m(
           "svg",
           {

client/components/parameter-list.js

@@ -3,14 +3,21 @@
 import m from "mithril";
 import * as components from "../components";
 import * as store from "../store";
-import * as filterParser from "../../common/filter-parser";
+import * as funcCache from "../../common/func-cache";
+import * as expression from "../../common/expression";
+
+const parseParameter = funcCache.getter(p => {
+  p = expression.parse(p);
+  if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
+  return p;
+});
 
 const component = {
   oninit: vnode => {
     vnode.state.parameters = Object.values(vnode.attrs.parameters).map(
       parameter =>
         Object.assign({}, parameter, {
-          parameter: filterParser.parseParameter(parameter.parameter)
+          parameter: parseParameter(parameter.parameter)
         })
     );
   },

client/components/parameter-table.js

@@ -4,15 +4,23 @@ import m from "mithril";
 import * as components from "../components";
 import * as taskQueue from "../task-queue";
 import * as store from "../store";
-import * as filterParser from "../../common/filter-parser";
+import * as expression from "../../common/expression";
+import * as funcCache from "../../common/func-cache";
+
+const parseParameter = funcCache.getter(p => {
+  p = expression.parse(p);
+  if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
+  return p;
+});
 
 const component = {
   oninit: vnode => {
-    vnode.state.object = filterParser.parseParameter(vnode.attrs.parameter);
+    vnode.state.object = parseParameter(vnode.attrs.parameter);
     vnode.state.parameters = Object.values(vnode.attrs.childParameters).map(
       parameter => {
-        let p = filterParser.parseParameter(parameter.parameter);
-        return Object.assign({}, parameter, { parameter: p });
+        return Object.assign({}, parameter, {
+          parameter: parseParameter(parameter.parameter)
+        });
       }
     );
   },

client/components/summon-button.js

@@ -4,12 +4,19 @@ import m from "mithril";
 import * as taskQueue from "../task-queue";
 import * as store from "../store";
 import * as notifications from "../notifications";
-import * as filterParser from "../../common/filter-parser";
+import * as funcCache from "../../common/func-cache";
+import * as expression from "../../common/expression";
+
+const parseParameter = funcCache.getter(p => {
+  p = expression.parse(p);
+  if (Array.isArray(p) && p[0] === "PARAM") p = p[1];
+  return p;
+});
 
 const component = {
   oninit: vnode => {
     vnode.state.parameters = Object.values(vnode.attrs.parameters).map(
-      parameter => filterParser.parseParameter(parameter)
+      parameter => parseParameter(parameter)
     );
   },
   view: vnode => {

client/device-page.js

@@ -3,7 +3,6 @@
 import m from "mithril";
 
 import config from "./config";
-import Filter from "../common/filter";
 import * as store from "./store";
 import * as components from "./components";
 
@@ -15,7 +14,7 @@ function init(args) {
 
   return Promise.resolve({
     deviceId: args.id,
-    deviceFilter: new Filter(["=", "DeviceID.ID", args.id])
+    deviceFilter: ["=", ["PARAM", "DeviceID.ID"], args.id]
   });
 }
 

client/devices-page.js

@@ -4,24 +4,24 @@ import m from "mithril";
 
 import config from "./config";
 import filterComponent from "./filter-component";
-import * as filterParser from "../common/filter-parser";
-import Filter from "../common/filter";
 import * as store from "./store";
 import * as components from "./components";
 import * as taskQueue from "./task-queue";
 import * as notifications from "./notifications";
+import * as expression from "../common/expression";
 
 function init(args) {
   return new Promise((resolve, reject) => {
     if (!window.authorizer.hasAccess("devices", 2))
       return reject(new Error("You are not authorized to view this page"));
 
-    let filter = new Filter(args.filter);
-    let indexParameters = Object.values(config.ui.index).map(p =>
-      Object.assign({}, p, {
-        parameter: filterParser.parseParameter(p.parameter)
-      })
-    );
+    let filter = null;
+    if (args.filter != null) filter = expression.parse(`${args.filter}`);
+    let indexParameters = Object.values(config.ui.index).map(p => {
+      let param = expression.parse(p.parameter);
+      if (Array.isArray(param) && param[0] === "PARAM") param = param[1];
+      return Object.assign({}, p, { parameter: param });
+    });
     resolve({ filter: filter, indexParameters: indexParameters });
   });
 }
@@ -281,16 +281,19 @@ const component = {
 
     function onFilterChanged(filter) {
       let params = {};
-      if (filter) params.filter = filterParser.stringify(filter);
+      if (filter != null) params.filter = expression.stringify(filter);
       m.route.set("/devices", params);
     }
 
     let devs = store.fetch(
       "devices",
-      vnode.attrs.filter,
+      vnode.attrs.filter == null ? true : vnode.attrs.filter,
       vnode.state.showCount || 10
     );
-    let count = store.count("devices", vnode.attrs.filter);
+    let count = store.count(
+      "devices",
+      vnode.attrs.filter == null ? true : vnode.attrs.filter
+    );
 
     let selected = new Set();
     if (vnode.state.selected)
@@ -303,7 +306,7 @@ const component = {
       m("h1", "Listing devices"),
       m(filterComponent, {
         predefined: Object.values(config.ui.filters),
-        filter: vnode.attrs.filter ? vnode.attrs.filter.ast : null,
+        filter: vnode.attrs.filter,
         onChange: onFilterChanged
       }),
       renderTable(

client/drawer-component.js

@@ -5,7 +5,6 @@ import m from "mithril";
 import * as taskQueue from "./task-queue";
 import * as store from "./store";
 import * as notifications from "./notifications";
-import Filter from "../common/filter";
 
 function mparam(param) {
   return m("span.parameter", { title: param }, `${param}`);
@@ -41,7 +40,7 @@ function renderStagingSpv(task, queueFunc, cancelFunc) {
 
 function renderStagingDownload(task) {
   task.invalid = !task.fileName || !task.fileType;
-  const files = store.fetch("files", new Filter());
+  const files = store.fetch("files", true);
   const idParts = task.device.split("-");
   const oui = decodeURIComponent(idParts[0]);
   const productClass =

client/faults-page.js

@@ -3,10 +3,9 @@
 import m from "mithril";
 
 import filterComponent from "./filter-component";
-import * as filterParser from "../common/filter-parser";
-import Filter from "../common/filter";
 import * as store from "./store";
 import * as notifications from "./notifications";
+import * as expression from "../common/expression";
 
 function init(args) {
   if (!window.authorizer.hasAccess("faults", 2))
@@ -14,7 +13,8 @@ function init(args) {
       new Error("You are not authorized to view this page")
     );
 
-  let filter = new Filter(args.filter);
+  let filter = null;
+  if (args.filter != null) filter = expression.parse(`${args.filter}`);
   return Promise.resolve({ filter: filter });
 }
 
@@ -153,16 +153,19 @@ const component = {
 
     function onFilterChanged(filter) {
       let params = {};
-      if (filter) params.filter = filterParser.stringify(filter);
+      if (filter != null) params.filter = expression.stringify(filter);
       m.route.set("/faults", params);
     }
 
     let faults = store.fetch(
       "faults",
-      vnode.attrs.filter,
+      vnode.attrs.filter == null ? true : vnode.attrs.filter,
       vnode.state.showCount || 10
     );
-    let count = store.count("faults", vnode.attrs.filter);
+    let count = store.count(
+      "faults",
+      vnode.attrs.filter == null ? true : vnode.attrs.filter
+    );
 
     let selected = new Set();
     if (vnode.state.selected)
@@ -180,7 +183,7 @@ const component = {
           { parameter: "retries" },
           { parameter: "timestamp" }
         ],
-        filter: vnode.attrs.filter ? vnode.attrs.filter.ast : null,
+        filter: vnode.attrs.filter,
         onChange: onFilterChanged
       }),
       renderTable(faults, count.value, selected, showMore)

client/filter-component.js

@@ -2,7 +2,7 @@
 
 import m from "mithril";
 
-import * as filterParser from "../common/filter-parser";
+import * as expression from "../common/expression";
 
 const component = {
   view: vnode => {
@@ -15,16 +15,14 @@ const component = {
     if (vnode.attrs.filter !== vnode.state.filter) {
       vnode.state.filterList = [];
       vnode.state.filter = vnode.attrs.filter;
-      if (vnode.attrs.filter)
+      if (vnode.attrs.filter != null)
         if (vnode.attrs.filter[0] === "AND")
           for (let i = 1; i < vnode.attrs.filter.length; ++i)
             vnode.state.filterList.push(
-              filterParser.stringify(vnode.attrs.filter[i])
+              expression.stringify(vnode.attrs.filter[i])
             );
         else
-          vnode.state.filterList.push(
-            filterParser.stringify(vnode.attrs.filter)
-          );
+          vnode.state.filterList.push(expression.stringify(vnode.attrs.filter));
       vnode.state.filterList.push("");
     }
 
@@ -33,18 +31,18 @@ const component = {
       vnode.state.filterList = vnode.state.filterList.filter(
         f => f && f.trim()
       );
-      let filterAst = vnode.state.filterList.map((f, idx) => {
+      let filter = vnode.state.filterList.map((f, idx) => {
         try {
-          return filterParser.parse(f);
+          return expression.parse(f);
         } catch (err) {
           vnode.state.filterInvalid |= 1 << idx;
         }
       });
-      if (filterAst.length) filterAst = ["AND"].concat(filterAst);
-      else filterAst = null;
+      if (filter.length) filter = ["AND"].concat(filter);
+      else filter = null;
 
       vnode.state.filterList.push("");
-      if (!vnode.state.filterInvalid) vnode.attrs.onChange(filterAst);
+      if (!vnode.state.filterInvalid) vnode.attrs.onChange(filter);
     }
 
     return m(

client/overview-page.js

@@ -4,8 +4,8 @@ import m from "mithril";
 import config from "./config";
 import * as store from "./store";
 import pieChartComponent from "./pie-chart-component";
-import Filter from "../common/filter";
 import * as funcCache from "../common/func-cache";
+import * as expression from "../common/expression";
 
 const GROUPS = config.ui.overview.groups;
 const CHARTS = {};
@@ -19,7 +19,7 @@ function queryCharts(charts) {
     charts[chartName] = chart = Object.assign({}, chart);
     chart.slices = Object.assign({}, chart.slices);
     for (let [sliceName, slice] of Object.entries(chart.slices)) {
-      const filter = funcCache.get(Filter.parse, slice.filter);
+      const filter = funcCache.get(expression.parse, slice.filter);
       chart.slices[sliceName] = slice = Object.assign({}, slice);
       slice.count = store.count("devices", filter);
     }

client/store.js

@@ -1,15 +1,15 @@
 "use strict";
 
 import m from "mithril";
-import Filter from "../common/filter";
 import config from "./config";
-import * as filterParser from "../common/filter-parser";
+import * as expressionParser from "../common/expression-parser";
+import * as expression from "../common/expression";
 import * as funcCache from "../common/func-cache";
 
 let fulfillTimestamp = 0;
 
-let unpackedFiltersCache = new WeakMap();
-let expressionsCache = new WeakMap();
+let unpackExpressionCache = new WeakMap();
+let evaluateExpressionCache = new WeakMap();
 
 const queries = {
   filter: new WeakMap(),
@@ -47,46 +47,54 @@ class QueryResponse {
   }
 }
 
-function unpackFilter(filter) {
-  let f = unpackedFiltersCache.get(filter);
-  if (!f) {
-    f = f.evaluateExpressions(fulfillTimestamp);
-    unpackedFiltersCache.set(filter, f);
+function unpackExpression(exp) {
+  if (!Array.isArray(exp)) return exp;
+  let e = unpackExpressionCache.get(exp);
+  if (e === undefined) {
+    e = expression.evaluate(e, null, fulfillTimestamp);
+    unpackExpressionCache.set(exp, e);
   }
-  return f;
+  return e;
 }
 
 function count(resourceType, filter) {
-  let queryResponse = resources[resourceType].count.get(filter.toString());
+  const filterStr = funcCache.get(expression.stringify, filter);
+  let queryResponse = resources[resourceType].count.get(filterStr);
   if (queryResponse) return queryResponse;
 
   queryResponse = new QueryResponse();
 
-  resources[resourceType].count.set(filter.toString(), queryResponse);
+  resources[resourceType].count.set(filterStr, queryResponse);
   queries.filter.set(queryResponse, filter);
   return queryResponse;
 }
 
 function limitFilter(resourceType, filter, last) {
   if (resourceType === "devices")
-    return filter.and(new Filter(["<=", "DeviceID.ID", last]));
-  else return filter.and(new Filter(["<=", "_id", last]));
+    return expression.and(filter, ["<=", "DeviceID.ID", last]);
+  else return expression.and(filter, ["<=", ["PARAM", "_id"], last]);
 }
 
 function findMatches(resourceType, filter, limit) {
   // Handle "tag =" and "tag <>" special cases
-  if (resourceType === "devices" && filter.ast) {
-    const ast = filterParser.map(filter.ast, e => {
-      if (e[1] === "tag")
-        if (e[0] === "=") return ["=", `Tags.${e[2]}`, true];
-        else if (e[0] === "<>") return ["NOT", ["=", `Tags.${e[2]}`, true]];
+  if (resourceType === "devices")
+    filter = expressionParser.map(filter, e => {
+      if (
+        Array.isArray(e) &&
+        Array.isArray(e[1]) &&
+        e[1][0] === "PARAM" &&
+        e[1][0] === "tag"
+      )
+        if (e[0] === "=") return ["=", ["PARAM", `Tags.${e[2]}`], true];
+        else if (e[0] === "<>")
+          return ["NOT", ["=", ["PARAM", `Tags.${e[2]}`], true]];
+      return e;
     });
-    filter = new Filter(ast);
-  }
 
   let value = [];
   for (let [id, obj] of resources[resourceType].objects.entries())
-    if (filter.test(obj)) value.push(id);
+    if (expression.evaluate(filter, obj, fulfillTimestamp))
+      value.push(id);
 
   value = value.sort();
   if (limit) value = value.slice(0, limit);
@@ -97,13 +105,13 @@ function findMatches(resourceType, filter, limit) {
 function inferQuery(resourceType, queryResponse) {
   const limit = queries.limit.get(queryResponse);
   let filter = queries.filter.get(queryResponse);
-  filter = unpackFilter(filter);
+  filter = unpackExpression(filter);
   let last = queries.last.get(queryResponse);
   if (last || !limit) {
     if (last) filter = limitFilter(resourceType, filter, last);
     if (
       resources[resourceType].combinedFilter &&
-      filter.subset(resources[resourceType].combinedFilter)
+      expression.subset(filter, resources[resourceType].combinedFilter)
     )
       queries.fulfilled.set(queryResponse, fulfillTimestamp);
   }
@@ -112,7 +120,8 @@ function inferQuery(resourceType, queryResponse) {
 }
 
 function fetch(resourceType, filter, limit = 0) {
-  const key = `${limit}:${filter.toString()}`;
+  const filterStr = funcCache.get(expression.stringify, filter);
+  const key = `${limit}:${filterStr}`;
   let queryResponse = resources[resourceType].fetch.get(key);
   if (queryResponse) return queryResponse;
 
@@ -131,8 +140,8 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
   let updated = false;
 
   if (_fulfillTimestamp > fulfillTimestamp) {
-    unpackedFiltersCache = new WeakMap();
-    expressionsCache = new WeakMap();
+    unpackExpressionCache = new WeakMap();
+    evaluateExpressionCache = new WeakMap();
 
     for (let resource of Object.values(resources))
       resource.combinedFilter = null;
@@ -156,14 +165,14 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
           new Promise((resolve, reject) => {
             updated = true;
             let filter = queries.filter.get(queryResponse);
-            filter = unpackFilter(filter);
+            filter = unpackExpression(filter);
             m
               .request({
                 method: "HEAD",
                 url:
                   `/api/${resourceType}/?` +
                   m.buildQueryString({
-                    filter: filter.toString()
+                    filter: funcCache.get(expression.stringify, filter)
                   }),
                 extract: xhr => +xhr.getResponseHeader("x-total-count"),
                 background: true
@@ -201,14 +210,14 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
             new Promise((resolve, reject) => {
               updated = true;
               let filter = queries.filter.get(queryResponse);
-              filter = unpackFilter(filter);
+              filter = unpackExpression(filter);
               m
                 .request({
                   method: "GET",
                   url:
                     `/api/${resourceType}/?` +
                     m.buildQueryString({
-                      filter: filter.toString(),
+                      filter: funcCache.get(expression.stringify, filter),
                       limit: 1,
                       skip: limit - 1,
                       projection: "_id"
@@ -240,13 +249,13 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
 
           toFetch = toFetch.filter(queryResponse => {
             let filter = queries.filter.get(queryResponse);
-            filter = unpackFilter(filter);
+            filter = unpackExpression(filter);
             let last = queries.last.get(queryResponse);
             if (last) filter = limitFilter(resourceType, filter, last);
 
             if (
               resources[resourceType].combinedFilter &&
-              filter.subset(resources[resourceType].combinedFilter)
+              expression.subset(filter, resources[resourceType].combinedFilter)
             ) {
               queries.value.set(
                 queryResponse,
@@ -257,11 +266,11 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
               return false;
             }
 
-            combinedFilter = filter.or(combinedFilter);
+            combinedFilter = expression.or(combinedFilter, filter);
             return true;
           });
 
-          if (!combinedFilter) return resolve(updated);
+          if (combinedFilter == null) continue;
 
           updated = true;
           let deleted = new Set();
@@ -269,10 +278,12 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
             deleted = new Set(resources[resourceType].objects.keys());
           let combinedFilterDiff = combinedFilter;
           if (resources[resourceType].combinedFilter)
-            combinedFilterDiff = combinedFilterDiff.and(
-              resources[resourceType].combinedFilter.not()
+            combinedFilterDiff = expression.and(
+              combinedFilterDiff,
+              expression.not(resources[resourceType].combinedFilter)
             );
-          resources[resourceType].combinedFilter = combinedFilter.or(
+          resources[resourceType].combinedFilter = expression.or(
+            combinedFilter,
             resources[resourceType].combinedFilter
           );
 
@@ -284,7 +295,10 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
                   url:
                     `/api/${resourceType}/?` +
                     m.buildQueryString({
-                      filter: combinedFilterDiff.toString()
+                      filter: funcCache.get(
+                        expression.stringify,
+                        combinedFilterDiff
+                      )
                     })
                 })
                 .then(res => {
@@ -299,13 +313,19 @@ function fulfill(accessTimestamp, _fulfillTimestamp) {
 
                   for (let d of deleted) {
                     const obj = resources[resourceType].objects.get(d);
-                    if (combinedFilterDiff.test(obj))
+                    if (
+                      expression.evaluate(
+                        combinedFilterDiff,
+                        obj,
+                        fulfillTimestamp
+                      )
+                    )
                       resources[resourceType].objects.delete(d);
                   }
 
                   for (let queryResponse of toFetch) {
                     let filter = queries.filter.get(queryResponse);
-                    filter = unpackFilter(filter);
+                    filter = unpackExpression(filter);
                     let last = queries.last.get(queryResponse);
                     if (last) filter = limitFilter(resourceType, filter, last);
 
@@ -379,20 +399,18 @@ function deleteResource(resourceType, id) {
   });
 }
 
-function evaluateExpression(exp, device) {
+function evaluateExpression(exp, obj) {
   if (!Array.isArray(exp)) return exp;
 
-  let exps = expressionsCache.get(exp);
-  if (!exps) expressionsCache.set(exp, (exps = new WeakMap()));
+  let exps = evaluateExpressionCache.get(exp);
+  if (!exps) evaluateExpressionCache.set(exp, (exps = new WeakMap()));
 
-  if (!exps.has(device)) {
-    let v = filterParser.evaluateExpressions(exp, e => {
-      if (e[0] === "FUNC" && e[1] === "NOW") return fulfillTimestamp;
-    });
-    exps.set(device, v);
+  if (!exps.has(obj)) {
+    let v = expression.evaluate(exp, obj, fulfillTimestamp);
+    exps.set(obj, v);
   }
 
-  return exps.get(device);
+  return exps.get(obj);
 }
 
 function logIn(username, password) {
@@ -421,7 +439,7 @@ export {
   count,
   fetch,
   fulfill,
-  unpackFilter,
+  unpackExpression,
   getTimestamp,
   postTasks,
   updateTags,