Refactor expression to Mongo query translation

The previous approach was simplistic and could result in inefficient queries
in certain edge cases, notably causing MongoDB to not utilize existing indexes
when querying by last inform timestamps given upper and lower bounds. The new
implementation leverages our existing expression minimizer code and can
therefore generate more efficient queries or at least queries that can take
advantage of existing indexes.

ui/components/all-parameters.ts

@@ -20,7 +20,7 @@
 import { ClosureComponent, Component } from "mithril";
 import { m } from "../components";
 import * as taskQueue from "../task-queue";
-import { parse } from "../../lib/common/expression-parser";
+import { parse } from "../../lib/common/expression/parser";
 import memoize from "../../lib/common/memoize";
 import { getIcon } from "../icons";
 import { evaluateExpression } from "../store";

ui/components/container.ts

@@ -18,7 +18,7 @@
  */
 
 import { Attributes, ClosureComponent, Component } from "mithril";
-import { map } from "../../lib/common/expression-parser";
+import { map } from "../../lib/common/expression/parser";
 import memoize from "../../lib/common/memoize";
 import { Expression } from "../../lib/types";
 import { m } from "../components";

ui/components/parameter-table.ts

@@ -21,7 +21,7 @@ import { ClosureComponent, Component } from "mithril";
 import { m } from "../components";
 import * as taskQueue from "../task-queue";
 import { evaluateExpression } from "../store";
-import * as expressionParser from "../../lib/common/expression-parser";
+import * as expressionParser from "../../lib/common/expression/parser";
 import { getIcon } from "../icons";
 
 const component: ClosureComponent = (): Component => {

ui/components/parameter.ts

@@ -21,14 +21,14 @@ import { ClosureComponent, Component, VnodeDOM } from "mithril";
 import { m } from "../components";
 import * as taskQueue from "../task-queue";
 import * as store from "../store";
-import * as expression from "../../lib/common/expression";
+import { evaluate } from "../../lib/common/expression/util";
 import memoize from "../../lib/common/memoize";
 import timeAgo from "../timeago";
 import { getIcon } from "../icons";
 
 const evaluateParam = memoize((exp, obj, now: number) => {
   let timestamp = now;
-  exp = expression.evaluate(exp, null, now, (e) => {
+  exp = evaluate(exp, null, now, (e) => {
     if (!Array.isArray(e)) return e;
     for (let i = 1; i < e.length; ++i) {
       if (

ui/config-page.ts

@@ -23,7 +23,7 @@ import * as notifications from "./notifications";
 import putFormComponent from "./put-form-component";
 import uiConfigComponent from "./ui-config-component";
 import * as overlay from "./overlay";
-import { parse, stringify } from "../lib/common/expression-parser";
+import { parse, stringify } from "../lib/common/expression/parser";
 import { loadCodeMirror, loadYaml } from "./dynamic-loader";
 import { ClosureComponent, Component, Children } from "mithril";
 import { getIcon } from "./icons";

ui/devices-page.ts

@@ -25,11 +25,10 @@ import filterComponent from "./filter-component";
 import * as store from "./store";
 import { queueTask, stageDownload } from "./task-queue";
 import * as notifications from "./notifications";
-import { parse, stringify } from "../lib/common/expression-parser";
-import { evaluate, extractParams } from "../lib/common/expression";
+import { parse, stringify, map } from "../lib/common/expression/parser";
+import { evaluate, extractParams } from "../lib/common/expression/util";
 import memoize from "../lib/common/memoize";
 import * as smartQuery from "./smart-query";
-import * as expressionParser from "../lib/common/expression-parser";
 
 const PAGE_SIZE = config.ui.pageSize || 10;
 
@@ -54,7 +53,7 @@ const getDownloadUrl = memoize((filter, indexParameters) => {
 });
 
 const unpackSmartQuery = memoize((query) => {
-  return expressionParser.map(query, (e) => {
+  return map(query, (e) => {
     if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "Q")
       return smartQuery.unpack("devices", e[2], e[3]);
     return e;

ui/faults-page.ts

@@ -26,7 +26,7 @@ import * as store from "./store";
 import * as notifications from "./notifications";
 import memoize from "../lib/common/memoize";
 import * as smartQuery from "./smart-query";
-import { map, parse, stringify } from "../lib/common/expression-parser";
+import { map, parse, stringify } from "../lib/common/expression/parser";
 import { stringify as yamlStringify } from "../lib/common/yaml";
 
 const PAGE_SIZE = config.ui.pageSize || 10;

ui/files-page.ts

@@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component";
 import indexTableComponent from "./index-table-component";
 import * as overlay from "./overlay";
 import * as smartQuery from "./smart-query";
-import { map, parse, stringify } from "../lib/common/expression-parser";
+import { map, parse, stringify } from "../lib/common/expression/parser";
 
 const PAGE_SIZE = config.ui.pageSize || 10;
 

ui/filter-component.ts

@@ -18,13 +18,12 @@
  */
 
 import m, { ClosureComponent } from "mithril";
-import { parse, stringify, map } from "../lib/common/expression-parser";
+import { parse, stringify, map } from "../lib/common/expression/parser";
 import memoize from "../lib/common/memoize";
 import Autocomplete from "./autocomplete-compnent";
 import * as smartQuery from "./smart-query";
-import { filterToMongoQuery } from "../lib/mongodb-functions";
+import { validQuery } from "../lib/mongodb-functions";
 import { Expression } from "../lib/types";
-import { normalize } from "../lib/common/boolean-expression";
 
 const getAutocomplete = memoize((resource) => {
   const labels = smartQuery.getLabels(resource);
@@ -42,16 +41,6 @@ const getAutocomplete = memoize((resource) => {
   return autocomplete;
 });
 
-function validateQuery(q: Expression): void {
-  q = normalize(q);
-  if (Array.isArray(q) && q[0] === "CASE") {
-    q = q.slice(1).filter((a) => Array.isArray(a));
-    if (q.length > 1) q = ["AND", ...q];
-    else q = q[0];
-  }
-  if (Array.isArray(q)) filterToMongoQuery(q);
-}
-
 function parseFilter(resource, f): Expression {
   let exp;
   if (/^[\s0-9a-zA-Z]+:/.test(f)) {
@@ -62,16 +51,16 @@ function parseFilter(resource, f): Expression {
     exp = parse(f);
   }
 
+  const unpacked = map(exp, (e) => {
+    if (Array.isArray(e) && e[0] === "FUNC") {
+      if (e[1] === "Q") return smartQuery.unpack(resource, e[2], e[3]);
+      else if (e[1] === "NOW") return Date.now();
+    }
+    return e;
+  });
+
   // Throws exception if invalid Mongo query
-  validateQuery(
-    map(exp, (e) => {
-      if (Array.isArray(e) && e[0] === "FUNC") {
-        if (e[1] === "Q") return smartQuery.unpack(resource, e[2], e[3]);
-        else if (e[1] === "NOW") return Date.now();
-      }
-      return e;
-    })
-  );
+  validQuery(unpacked, resource);
 
   return exp;
 }

ui/permissions-page.ts

@@ -26,7 +26,7 @@ import putFormComponent from "./put-form-component";
 import indexTableComponent from "./index-table-component";
 import * as overlay from "./overlay";
 import * as smartQuery from "./smart-query";
-import { map, parse, stringify } from "../lib/common/expression-parser";
+import { map, parse, stringify } from "../lib/common/expression/parser";
 import filterComponent from "./filter-component";
 import { Children, ClosureComponent, Component } from "mithril";
 import { getIcon } from "./icons";

ui/pie-chart-component.ts

@@ -19,7 +19,7 @@
 
 import { ClosureComponent, Component, Children } from "mithril";
 import { m } from "./components";
-import { stringify } from "../lib/common/expression-parser";
+import { stringify } from "../lib/common/expression/parser";
 import memoize from "../lib/common/memoize";
 
 const memoizedStringify = memoize(stringify);

ui/presets-page.ts

@@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component";
 import indexTableComponent from "./index-table-component";
 import memoize from "../lib/common/memoize";
 import * as smartQuery from "./smart-query";
-import { map, parse, stringify } from "../lib/common/expression-parser";
+import { map, parse, stringify } from "../lib/common/expression/parser";
 
 const PAGE_SIZE = config.ui.pageSize || 10;
 

ui/provisions-page.ts

@@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component";
 import indexTableComponent from "./index-table-component";
 import * as overlay from "./overlay";
 import * as smartQuery from "./smart-query";
-import { map, parse, stringify } from "../lib/common/expression-parser";
+import { map, parse, stringify } from "../lib/common/expression/parser";
 import { loadCodeMirror } from "./dynamic-loader";
 
 const PAGE_SIZE = config.ui.pageSize || 10;

ui/store.ts

@@ -18,15 +18,15 @@
  */
 
 import m from "mithril";
-import { stringify } from "../lib/common/expression-parser";
-import { or, and, evaluate } from "../lib/common/expression";
+import { stringify } from "../lib/common/expression/parser";
+import { or, and, evaluate } from "../lib/common/expression/util";
 import memoize from "../lib/common/memoize";
 import { QueryOptions, Expression } from "../lib/types";
 import * as notifications from "./notifications";
 import { configSnapshot, genieacsVersion } from "./config";
 import { QueueTask } from "./task-queue";
 import { PingResult } from "../lib/ping";
-import { unionDiff, covers } from "../lib/common/boolean-expression";
+import { unionDiff, covers } from "../lib/common/expression/synth";
 
 const memoizedStringify = memoize(stringify);
 const memoizedEvaluate = memoize(evaluate);

ui/ui-config-component.ts

@@ -23,7 +23,7 @@ import * as store from "./store";
 import { yaml } from "./dynamic-loader";
 import * as configFunctions from "./config-functions";
 import codeEditorComponent from "./code-editor-component";
-import { parse } from "../lib/common/expression-parser";
+import { parse } from "../lib/common/expression/parser";
 
 function putActionHandler(prefix: string[], dataYaml: string): Promise<any> {
   return new Promise((resolve, reject) => {

ui/users-page.ts

@@ -26,7 +26,7 @@ import putFormComponent from "./put-form-component";
 import indexTableComponent from "./index-table-component";
 import * as overlay from "./overlay";
 import * as smartQuery from "./smart-query";
-import { map, parse, stringify } from "../lib/common/expression-parser";
+import { map, parse, stringify } from "../lib/common/expression/parser";
 import filterComponent from "./filter-component";
 import changePasswordComponent from "./change-password-component";
 import { Children, ClosureComponent, Component } from "mithril";

ui/virtual-parameters-page.ts

@@ -28,7 +28,7 @@ import putFormComponent from "./put-form-component";
 import indexTableComponent from "./index-table-component";
 import * as overlay from "./overlay";
 import * as smartQuery from "./smart-query";
-import { map, parse, stringify } from "../lib/common/expression-parser";
+import { map, parse, stringify } from "../lib/common/expression/parser";
 import { loadCodeMirror } from "./dynamic-loader";
 
 const PAGE_SIZE = config.ui.pageSize || 10;