1.2.13

To allow enough time for running unindexed queries in large deployments.

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Change Log
 
+## 1.2.13 (2024-06-06)
+
+- Increase connection timeout for UI and NBI from 30 to 120 seconds to avoid
+  timeouts when running unindexed queries in large deployments.
+
+- Fix race condition causing 503 error when deleting multiple faults at once.
+
+- Fix some UI config options not being evaluated as dynamic expressions.
+
+- Fix an issue where certain edge-case query expressions were not being
+  correctly converted to MongoDB queries, resulting in inaccurate search
+  results.
+
 ## 1.2.12 (2024-03-28)
 
 - Fix broken XMPP support in the previous release.

bin/genieacs-nbi.ts

@@ -64,7 +64,7 @@ if (!cluster.worker) {
     port: SERVICE_PORT,
     host: SERVICE_ADDRESS,
     ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
+    timeout: 120000,
   };
 
   // Need this for Node < 15

bin/genieacs-ui.ts

@@ -64,7 +64,7 @@ if (!cluster.worker) {
     port: SERVICE_PORT,
     host: SERVICE_ADDRESS,
     ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
+    timeout: 120000,
   };
 
   // Need this for Node < 15

build/spellcheck-dict.pws

@@ -177,3 +177,4 @@ CIDR
 XMPP
 PEM
 JID
+unindexed

lib/db/synth.ts

@@ -578,6 +578,8 @@ class MongoSynthContext extends SynthContextBase<Clause, MongoClause> {
   toQuery(minterms: Minterm[]): Filter<unknown> {
     const or = [];
 
+    const ejsonOps = ["$oid", "$date", "$regularExpression"];
+
     for (const minterm of minterms) {
       const query = {};
       loop: for (const clause of minterm) {
@@ -588,8 +590,6 @@ class MongoSynthContext extends SynthContextBase<Clause, MongoClause> {
         if (Object.keys(q).length !== 1)
           throw new Error("Invalid query expression");
         const [param, value] = Object.entries(q)[0];
-        if (Object.getPrototypeOf(value).constructor !== Object)
-          throw new Error("Invalid query expression");
         const dests = [query, ...(query["$and"] ?? [])];
         for (const dest of dests) {
           if (!(param in dest)) {
@@ -599,17 +599,29 @@ class MongoSynthContext extends SynthContextBase<Clause, MongoClause> {
 
           let src = value;
           let dst = dest[param];
+          if (Object.getPrototypeOf(src).constructor !== Object) continue;
+          if (Object.getPrototypeOf(dst).constructor !== Object) continue;
+          let srcKeys = Object.keys(src);
+          let dstKeys = Object.keys(dst);
 
-          if (src["$not"] && dst["$not"]) {
+          if ([...srcKeys, ...dstKeys].every((k) => k === "$not")) {
             src = src["$not"];
             dst = dst["$not"];
+            if (Object.getPrototypeOf(src).constructor !== Object) continue;
+            if (Object.getPrototypeOf(dst).constructor !== Object) continue;
+            srcKeys = Object.keys(src);
+            dstKeys = Object.keys(dst);
           }
 
-          if (!Object.keys(src).some((k) => k in dst)) {
+          if (srcKeys.some((k) => dstKeys.includes(k))) continue;
+
+          // Don't mix regular operators with EJSON special operators
+          if (srcKeys.some((k) => ejsonOps.includes(k))) continue;
+          if (dstKeys.some((k) => ejsonOps.includes(k))) continue;
+
           Object.assign(dst, src);
           continue loop;
         }
-        }
 
         query["$and"] ??= [];
         query["$and"].push({ [param]: value });

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.12",
+  "version": "1.2.13",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -29,9 +29,9 @@
     "bson": "^4.7.2",
     "espresso-iisojs": "^1.0.8",
     "iconv-lite": "^0.6.3",
-    "ipaddr.js": "^2.0.1",
+    "ipaddr.js": "^2.2.0",
     "jsonwebtoken": "^9.0.2",
-    "koa": "^2.15.2",
+    "koa": "^2.15.3",
     "koa-bodyparser": "^4.3.0",
     "koa-compress": "^5.1.0",
     "koa-jwt": "^4.0.3",
@@ -49,20 +49,20 @@
     "@types/koa-router": "^7.4.8",
     "@types/koa-static": "^4.0.4",
     "@types/mithril": "^2.2.6",
-    "@types/node": "^20.11.30",
+    "@types/node": "^20.14.2",
     "@types/parsimmon": "^1.10.9",
     "@types/seedrandom": "^3.0.8",
-    "@typescript-eslint/eslint-plugin": "^7.3.1",
-    "@typescript-eslint/parser": "^7.3.1",
+    "@typescript-eslint/eslint-plugin": "^7.12.0",
+    "@typescript-eslint/parser": "^7.12.0",
     "codemirror": "^5.65.16",
-    "esbuild": "^0.20.2",
+    "esbuild": "^0.21.4",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "mithril": "^2.2.2",
-    "prettier": "^3.2.5",
-    "sql.js": "^1.10.2",
-    "svgo": "^3.2.0",
-    "typescript": "^5.4.3",
+    "prettier": "^3.3.1",
+    "sql.js": "^1.10.3",
+    "svgo": "^3.3.2",
+    "typescript": "^5.4.5",
     "yaml": "^1.10.2"
   },
   "engines": {

ui/components/overview-dot.ts

@@ -27,10 +27,14 @@ const component: ClosureComponent = (): Component => {
               cx: "0.5em",
               cy: "0.5em",
               r: "0.4em",
-              fill: slice["color"],
+              fill: evaluateExpression(slice["color"], null),
             }),
           );
-          return m("span.overview-dot", dot, `${slice["label"]}`);
+          return m(
+            "span.overview-dot",
+            dot,
+            evaluateExpression(slice["label"], null),
+          );
         }
       }
       return null;

ui/device-page.ts

@@ -39,7 +39,11 @@ export const component: ClosureComponent = (): Component => {
 
       for (const c of Object.values(conf)) {
         cmps.push(
-          m.context({ device: dev.value[0], deviceQuery: dev }, c["type"], c),
+          m.context(
+            { device: dev.value[0], deviceQuery: dev },
+            store.evaluateExpression(c["type"], null) as string,
+            c,
+          ),
         );
       }
 

ui/devices-page.ts

@@ -26,7 +26,10 @@ const memoizedGetSortable = memoize((p) => {
 
 const getDownloadUrl = memoize((filter, indexParameters) => {
   const columns = {};
-  for (const p of indexParameters) columns[p.label] = stringify(p.parameter);
+  for (const p of indexParameters)
+    columns[store.evaluateExpression(p.label, null) as string] = stringify(
+      p.parameter,
+    );
   return `api/devices.csv?${m.buildQueryString({
     filter: stringify(filter),
     columns: JSON.stringify(columns),
@@ -282,7 +285,11 @@ export const component: ClosureComponent = (): Component => {
       };
 
       const attrs = {};
-      attrs["attributes"] = attributes;
+      attrs["attributes"] = attributes.map((a) => ({
+        ...a,
+        label: store.evaluateExpression(a.label, null),
+        type: store.evaluateExpression(a.type, null),
+      }));
       attrs["data"] = devs.value;
       attrs["total"] = count.value;
       attrs["showMoreCallback"] = showMore;

ui/faults-page.ts

@@ -46,6 +46,18 @@ const unpackSmartQuery = memoize((query) => {
   });
 });
 
+async function deleteFaults(faults: Iterable<string>): Promise<void> {
+  const proms: Map<string, Promise<void>> = new Map();
+  for (const f of faults) {
+    const deviceId = f.split(":", 1)[0];
+    let p = proms.get(deviceId);
+    if (p == null) p = store.deleteResource("faults", f);
+    else p = p.then(() => store.deleteResource("faults", f));
+    proms.set(deviceId, p);
+  }
+  await Promise.all(proms.values());
+}
+
 export function init(
   args: Record<string, unknown>,
 ): Promise<Record<string, unknown>> {
@@ -154,16 +166,10 @@ export const component: ClosureComponent = (): Component => {
                 if (!confirm(`Deleting ${selected.size} faults. Are you sure?`))
                   return;
 
-                Promise.all(
-                  Array.from(selected).map((id) =>
-                    store.deleteResource("faults", id),
-                  ),
-                )
-                  .then((res) => {
-                    notifications.push(
-                      "success",
-                      `${res.length} faults deleted`,
-                    );
+                const c = selected.size;
+                deleteFaults(selected)
+                  .then(() => {
+                    notifications.push("success", `${c} faults deleted`);
                     store.setTimestamp(Date.now());
                   })
                   .catch((err) => {

ui/pie-chart-component.ts

@@ -2,6 +2,7 @@ import { ClosureComponent, Component, Children } from "mithril";
 import { m } from "./components.ts";
 import { stringify } from "../lib/common/expression/parser.ts";
 import memoize from "../lib/common/memoize.ts";
+import * as store from "./store.ts";
 
 const memoizedStringify = memoize(stringify);
 
@@ -24,9 +25,9 @@ function drawChart(chartData): Children {
     legend.push(
       m(".legend-line", [
         m("span.color", {
-          style: `background-color: ${slice["color"]} !important;`,
+          style: `background-color: ${store.evaluateExpression(slice["color"], null)} !important;`,
         }),
-        `${slice["label"]}: `,
+        `${store.evaluateExpression(slice["label"], null)}: `,
         m(
           "a",
           {
@@ -57,7 +58,7 @@ function drawChart(chartData): Children {
       paths.push(
         m("path", {
           d: sketch,
-          fill: slice["color"],
+          fill: store.evaluateExpression(slice["color"], null),
         }),
       );