1.2.14

To allow enough time for running unindexed queries in large deployments.

.eslintrc.json

@@ -1,6 +1,6 @@
 {
   "env": {
-    "es6": true,
+    "es2022": true,
     "node": true
   },
   "parser": "@typescript-eslint/parser",
@@ -17,11 +17,9 @@
     "no-shadow": ["error", { "allow": ["err"] }],
     "handle-callback-err": "error",
     "prefer-arrow-callback": "error",
-    "curly": ["error", "multi-or-nest", "consistent"],
     "no-buffer-constructor": "error",
     "prefer-const": ["error", { "destructuring": "all" }],
     "eqeqeq": ["error", "always", { "null": "ignore" }],
-    "no-return-await": "error",
     "@typescript-eslint/no-explicit-any": 0,
     "@typescript-eslint/no-use-before-define": [
       "error",
@@ -32,7 +30,7 @@
       { "allowExpressions": true }
     ],
     "@typescript-eslint/no-floating-promises": "error",
-    "require-atomic-updates": 0,
+    "@typescript-eslint/no-misused-promises": "error",
     "no-prototype-builtins": 0
   }
 }

CHANGELOG.md

@@ -1,5 +1,111 @@
 # Change Log
 
+## 1.2.14 (2026-03-12)
+
+- Prevent UI crash when a malformed URL is sent to the server.
+
+- Fix potential edge-case bugs in expression evaluation and session
+  serialization.
+
+## 1.2.13 (2024-06-06)
+
+- Increase connection timeout for UI and NBI from 30 to 120 seconds to avoid
+  timeouts when running unindexed queries in large deployments.
+
+- Fix race condition causing 503 error when deleting multiple faults at once.
+
+- Fix some UI config options not being evaluated as dynamic expressions.
+
+- Fix an issue where certain edge-case query expressions were not being
+  correctly converted to MongoDB queries, resulting in inaccurate search
+  results.
+
+## 1.2.12 (2024-03-28)
+
+- Fix broken XMPP support in the previous release.
+
+- Fix regression causing CSV downloads to be buffered in memory before being
+  streamed to the client.
+
+## 1.2.11 (2024-03-21)
+
+- Resolved an issue from the previous release that caused incompatibility with
+  Node.js versions 12 through 15.
+
+## 1.2.10 (2024-03-18)
+
+- Add support for XMPP connection requests. Use the environment variables
+  `XMPP_JID` and `XMPP_PASSWORD` to configure the XMPP connection for the ACS.
+
+- The environment variables `CWMP_SSL_CERT` and `CWMP_SSL_KEY`, as well as their
+  counterparts for UI, NBI, and FS, now accept PEM-encoded certificates and keys
+  in addition to file paths.
+
+- The UI no longer requires users to refresh the page after modifying presets,
+  provisions, or virtual parameters. Refreshing is now only necessary for
+  changes to users, permissions, or UI configurations.
+
+- Improved conversion of GenieACS expressions into MongoDB queries for more
+  optimized queries and better index utilization.
+
+- Refactor UI pagination and sorting to fix issues from the previous approach,
+  especially with sorting by rapidly changing parameters such as 'last inform'
+  time.
+
+- The file server now supports HEAD requests, the Range header, and conditional
+  requests.
+
+- Addressed an issue causing file server disconnections for slow clients over
+  HTTPS.
+
+- Fix bug preventing users from changing their own passwords.
+
+- Introduced a new 'locks' collection for database locking, replacing the
+  previous use of the 'cache' collection for this purpose.
+
+- Various other minor fixes and enhancements.
+
+## 1.2.9 (2022-08-22)
+
+- New config option `cwmp.skipRootGpn` to enable a workaround for some
+  problematic CPEs that reject GPN requests on data model root.
+- Stream query results and CSV downloads as data becomes available instead of
+  buffering the entire response.
+- Log HTTP/HTTPS client errors in debug log.
+- Fix occasional lock expired errors after updating presets, etc.
+- Fix bug where queries containing `<>` operator may return incorrect results.
+- Fix a performance hit caused by DB calls containing the entire CPE data model
+  rather then just the updated parameters.
+- Fix bug where tags containing special characters are saved in their encoded
+  form when set via a Provision script.
+- Fix issue causing invalid `xsd:dateTime` values to be saved in DB as `NaN`
+  rather than maintain the original string value.
+- Fix bug using `$regex` operator with a numeric or a datetime string.
+- Fix ping not working on certain platforms.
+- Ping requests are now authenticated.
+- Fix error when the `FORWARDED_HEADER` environment variable contains IPv4 CIDR
+  while the listening interface is IPv6 and vice versa.
+- Fix false warning "unexpected parameter in response" showing in
+  `genieacs-cwmp` logs.
+- Other minor fixes and stability improvements.
+
+## 1.2.8 (2021-10-27)
+
+- Fix a remote code execution security vulnerability in genieacs-ui.
+- All UI components can now be configured using fully dynamic expressions.
+  Previously some component types only accept fixed string values as properties.
+- The `container` UI component can now be configured with an `element` property
+  that's either a string or a nested pair of `tag` and `attributes` properties.
+  The various attributes under the `attributes` property can now make use of a
+  new function `ENCODEURICOMPONENT()` to facilitate creating custom hyperlinks
+  in the UI.
+- Improve sorting buttons' behavior in the device listing page and other pages.
+  Sorting by multiple columns should now feel more intuitive.
+- Support the modulo (%) operator in expressions.
+- Fix a regression in the previous release where the config option 'ui.pageSize'
+  no longer works.
+- Fix process crash when a CPE sends an unsupported ACS method.
+
 ## 1.2.7 (2021-09-18)
 
 - Fix regression causing frequent invalid session errors.

ava.config.js

@@ -1,4 +0,0 @@
-export default {
-  extensions: ["ts"],
-  require: ["esm", "ts-node/register/transpile-only"],
-};

bin/genieacs-cwmp.ts

@@ -1,29 +1,10 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import * as cwmp from "../lib/cwmp";
-import * as db from "../lib/db";
-import * as extensions from "../lib/extensions";
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import * as cwmp from "../lib/cwmp.ts";
+import * as db from "../lib/db/db.ts";
+import * as extensions from "../lib/extensions.ts";
 import { version as VERSION } from "../package.json";
 
 logger.init("cwmp", VERSION);
@@ -37,11 +18,13 @@ function exitWorkerGracefully(): void {
     db.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
-  ]).catch(exitWorkerUngracefully);
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
+  void extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -83,10 +66,17 @@ if (!cluster.worker) {
     host: SERVICE_ADDRESS,
     ssl: key && cert ? { key, cert } : null,
     onConnection: cwmp.onConnection,
+    onClientError: cwmp.onClientError,
     timeout: 30000,
     keepAliveTimeout: 0,
+    requestTimeout: cwmp.REQUEST_TIMEOUT,
   };
 
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
+
   process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
@@ -94,7 +84,7 @@ if (!cluster.worker) {
       exception: err,
       pid: process.pid,
     });
-    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    server.stop(false).then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
   const initPromise = db
@@ -109,14 +99,20 @@ if (!cluster.worker) {
     });
 
   process.on("SIGINT", () => {
-    initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    void initPromise.finally(() => {
+      server
+        .stop(false)
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
-    initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    void initPromise.finally(() => {
+      server
+        .stop(false)
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 }

bin/genieacs-ext.ts

@@ -1,23 +1,4 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { Fault } from "../lib/types";
+import { Fault } from "../lib/types.ts";
 
 const jobs = new Set();
 const fileName = process.argv[2];
@@ -41,12 +22,12 @@ function errorToFault(err: Error): Fault {
     fault.detail["stack"] = err.stack;
     // Trim the stack trace
     const stackTrimIndex = fault.detail["stack"].match(
-      /\s+at\s[^\s]+\s\(.*genieacs-ext:.+\)/
+      /\s+at\s[^\s]+\s\(.*genieacs-ext:.+\)/,
     );
     if (stackTrimIndex) {
       fault.detail["stack"] = fault.detail["stack"].slice(
         0,
-        stackTrimIndex.index
+        stackTrimIndex.index,
       );
     }
   }
@@ -54,6 +35,11 @@ function errorToFault(err: Error): Fault {
   return fault;
 }
 
+// Need this for Node < 15
+process.on("unhandledRejection", (err) => {
+  throw err;
+});
+
 process.on("uncaughtException", (err) => {
   const fault = errorToFault(err);
   jobs.forEach((jobId) => {

bin/genieacs-fs.ts

@@ -1,28 +1,9 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/fs";
-import * as db from "../lib/db";
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import { listener } from "../lib/fs.ts";
+import * as db from "../lib/db/db.ts";
 import { version as VERSION } from "../package.json";
 
 logger.init("fs", VERSION);
@@ -32,9 +13,9 @@ const SERVICE_PORT = config.get("FS_PORT") as number;
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
-  Promise.all([db.disconnect(), cluster.worker.disconnect()]).catch(
-    exitWorkerUngracefully
-  );
+  Promise.all([db.disconnect(), cluster.worker.disconnect()])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
@@ -79,7 +60,10 @@ if (!cluster.worker) {
     timeout: 30000,
   };
 
-  let stopping = false;
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
 
   process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
@@ -88,19 +72,13 @@ if (!cluster.worker) {
       exception: err,
       pid: process.pid,
     });
-    stopping = true;
     server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    void listener(req, res);
-  };
-
   const initPromise = db
     .connect()
     .then(() => {
-      server.start(options, _listener);
+      server.start(options, listener);
     })
     .catch((err) => {
       setTimeout(() => {
@@ -109,15 +87,13 @@ if (!cluster.worker) {
     });
 
   process.on("SIGINT", () => {
-    stopping = true;
-    initPromise.finally(() => {
+    void initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
-    stopping = true;
-    initPromise.finally(() => {
+    void initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });

bin/genieacs-nbi.ts

@@ -1,29 +1,10 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/nbi";
-import * as db from "../lib/db";
-import * as extensions from "../lib/extensions";
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import { listener } from "../lib/nbi.ts";
+import * as db from "../lib/db/db.ts";
+import * as extensions from "../lib/extensions.ts";
 import { version as VERSION } from "../package.json";
 
 logger.init("nbi", VERSION);
@@ -37,11 +18,13 @@ function exitWorkerGracefully(): void {
     db.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
-  ]).catch(exitWorkerUngracefully);
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
+  void extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -81,10 +64,13 @@ if (!cluster.worker) {
     port: SERVICE_PORT,
     host: SERVICE_ADDRESS,
     ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
+    timeout: 120000,
   };
 
-  let stopping = false;
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
 
   process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
@@ -93,19 +79,13 @@ if (!cluster.worker) {
       exception: err,
       pid: process.pid,
     });
-    stopping = true;
     server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
-  };
-
   const initPromise = db
     .connect()
     .then(() => {
-      server.start(options, _listener);
+      server.start(options, listener);
     })
     .catch((err) => {
       setTimeout(() => {
@@ -114,15 +94,13 @@ if (!cluster.worker) {
     });
 
   process.on("SIGINT", () => {
-    stopping = true;
-    initPromise.finally(() => {
+    void initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
-    stopping = true;
-    initPromise.finally(() => {
+    void initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });

bin/genieacs-ui.ts

@@ -1,29 +1,10 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/ui";
-import * as extensions from "../lib/extensions";
-import * as db from "../lib/db";
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import { listener } from "../lib/ui.ts";
+import * as extensions from "../lib/extensions.ts";
+import * as db from "../lib/db/db.ts";
 import { version as VERSION } from "../package.json";
 
 logger.init("ui", VERSION);
@@ -37,11 +18,13 @@ function exitWorkerGracefully(): void {
     db.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
-  ]).catch(exitWorkerUngracefully);
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
+  void extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -81,10 +64,13 @@ if (!cluster.worker) {
     port: SERVICE_PORT,
     host: SERVICE_ADDRESS,
     ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
+    timeout: 120000,
   };
 
-  let stopping = false;
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
 
   process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
@@ -93,19 +79,13 @@ if (!cluster.worker) {
       exception: err,
       pid: process.pid,
     });
-    stopping = true;
     server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
-  };
-
   const initPromise = db
     .connect()
     .then(() => {
-      server.start(options, _listener);
+      server.start(options, async (req, res) => listener(req, res));
     })
     .catch((err) => {
       setTimeout(() => {
@@ -114,15 +94,13 @@ if (!cluster.worker) {
     });
 
   process.on("SIGINT", () => {
-    stopping = true;
-    initPromise.finally(() => {
+    void initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
-    stopping = true;
-    initPromise.finally(() => {
+    void initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });

build/assets.ts

@@ -0,0 +1,5 @@
+export const APP_JS = "app.js";
+export const APP_CSS = "app.css";
+export const ICONS_SVG = "icons.svg";
+export const LOGO_SVG = "logo.svg";
+export const FAVICON_PNG = "favicon.png";

build/build.ts

@@ -1,79 +1,46 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as path from "path";
-import * as fs from "fs";
-import { rollup, WarningHandler } from "rollup";
-import rollupJson from "@rollup/plugin-json";
-import typescript from "@rollup/plugin-typescript";
-import { terser } from "rollup-plugin-terser";
-import nodeResolve from "@rollup/plugin-node-resolve";
-import commonjs from "@rollup/plugin-commonjs";
-import postcss from "postcss";
-import postcssImport from "postcss-import";
-import postcssPresetEnv from "postcss-preset-env";
-import cssnano from "cssnano";
+import path from "node:path";
+import fs from "node:fs";
+import { createHash } from "node:crypto";
+import { promisify } from "node:util";
+import { exec } from "node:child_process";
+import * as esbuild from "esbuild";
 import { optimize } from "svgo";
-import * as xmlParser from "../lib/xml-parser";
+import * as xmlParser from "../lib/xml-parser.ts";
+
+const fsAsync = {
+  readdir: promisify(fs.readdir),
+  readFile: promisify(fs.readFile),
+  writeFile: promisify(fs.writeFile),
+  copyFile: promisify(fs.copyFile),
+  rename: promisify(fs.rename),
+  chmod: promisify(fs.chmod),
+  lstat: promisify(fs.lstat),
+  exists: promisify(fs.exists),
+  rmdir: promisify(fs.rmdir),
+  unlink: promisify(fs.unlink),
+  mkdir: promisify(fs.mkdir),
+};
+
+const execAsync = promisify(exec);
 
 const MODE = process.env["NODE_ENV"] || "production";
 
-const BUILD_METADATA = new Date()
-  .toISOString()
-  .split(".")[0]
-  .replace(/[^0-9]/g, "");
+const INPUT_DIR = process.cwd();
+const OUTPUT_DIR = path.join(INPUT_DIR, "dist");
 
-const INPUT_DIR = path.resolve(__dirname, "..");
-const OUTPUT_DIR = path.resolve(__dirname, "../dist");
-
-const builtins = [
-  "path",
-  "fs",
-  "cluster",
-  "os",
-  "tls",
-  "http",
-  "https",
-  "zlib",
-  "crypto",
-  "util",
-  "vm",
-  "querystring",
-  "child_process",
-  "dgram",
-  "url",
-  "readline",
-  "stream",
-];
-
-function rmDirSync(dirPath): void {
-  if (!fs.existsSync(dirPath)) return;
-  const files = fs.readdirSync(dirPath);
+async function rmDir(dirPath: string): Promise<void> {
+  if (!(await fsAsync.exists(dirPath))) return;
+  const files = await fsAsync.readdir(dirPath);
 
   for (const file of files) {
-    const filePath = `${dirPath}/${file}`;
-    if (fs.statSync(filePath).isFile()) fs.unlinkSync(filePath);
-    else rmDirSync(filePath);
+    const filePath = path.join(dirPath, file);
+    if ((await fsAsync.lstat(filePath)).isDirectory()) await rmDir(filePath);
+    else await fsAsync.unlink(filePath);
   }
-  fs.rmdirSync(dirPath);
+  await fsAsync.rmdir(dirPath);
 }
 
+// For lockfileVersion = 1
 function stripDevDeps(deps): void {
   if (!deps["dependencies"]) return;
   for (const [k, v] of Object.entries(deps["dependencies"])) {
@@ -83,6 +50,15 @@ function stripDevDeps(deps): void {
   if (!Object.keys(deps["dependencies"]).length) delete deps["dependencies"];
 }
 
+// For lockfileVersion = 2
+function stripDevDeps2(deps): void {
+  if (!deps["packages"]) return;
+  for (const [k, v] of Object.entries(deps["packages"])) {
+    delete v["devDependencies"];
+    if (v["dev"]) delete deps["packages"][k];
+  }
+}
+
 function xmlTostring(xml): string {
   const children = [];
   for (const c of xml.children || []) children.push(xmlTostring(c));
@@ -92,6 +68,59 @@ function xmlTostring(xml): string {
     : `<${xml.name} ${xml.attrs}>${children.join("")}</${xml.name}>`;
 }
 
+function assetHash(buffer: Buffer | string): string {
+  return createHash("md5").update(buffer).digest("hex").slice(0, 8);
+}
+
+const ASSETS = {} as {
+  APP_JS?: string;
+  APP_CSS?: string;
+  ICONS_SVG?: string;
+  LOGO_SVG?: string;
+  FAVICON_PNG?: string;
+};
+
+const assetsPlugin = {
+  name: "assets",
+  setup(build) {
+    build.onLoad({ filter: /\/build\/assets.ts$/ }, () => {
+      const lines = Object.entries(ASSETS).map(
+        ([k, v]) => `export const ${k} = ${JSON.stringify(v)};`,
+      );
+      return { contents: lines.join("\n") };
+    });
+  },
+} as esbuild.Plugin;
+
+const packageDotJsonPlugin = {
+  name: "packageDotJson",
+  setup(build) {
+    const sourcePath = path.join(INPUT_DIR, "package.json");
+    build.onResolve({ filter: /\/package.json$/ }, (args) => {
+      const p = path.join(args.resolveDir, args.path);
+      if (p !== sourcePath) return undefined;
+      return { path: path.join(OUTPUT_DIR, "package.json") };
+    });
+  },
+} as esbuild.Plugin;
+
+const inlineDepsPlugin = {
+  name: "inlineDeps",
+  setup(build) {
+    const deps = [
+      "parsimmon",
+      "espresso-iisojs",
+      "codemirror",
+      "mithril",
+      "yaml",
+    ];
+    build.onResolve({ filter: /^[^.]/ }, async (args) => {
+      if (deps.some((d) => args.path.startsWith(d))) return undefined;
+      return { sideEffects: false, external: true };
+    });
+  },
+} as esbuild.Plugin;
+
 function generateSymbol(id: string, svgStr: string): string {
   const xml = xmlParser.parseXml(svgStr);
   const svg = xml.children[0];
@@ -109,42 +138,60 @@ function generateSymbol(id: string, svgStr: string): string {
   return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
 }
 
-async function init(): Promise<string[]> {
-  // Delete any old output directory
-  rmDirSync(OUTPUT_DIR);
+async function getBuildMetadata(): Promise<string> {
+  const date = new Date().toISOString().slice(2, 10).replaceAll("-", "");
 
-  // Create output directory layout
-  fs.mkdirSync(OUTPUT_DIR);
-  fs.mkdirSync(OUTPUT_DIR + "/bin");
-  fs.mkdirSync(OUTPUT_DIR + "/public");
+  const [commit, diff, newFiles] = await Promise.all([
+    execAsync("git rev-parse HEAD"),
+    execAsync("git diff HEAD"),
+    execAsync("git ls-files --others --exclude-standard"),
+  ]).then((res) => res.map((r) => r.stdout.trim()));
 
-  // Create package.json
-  const packageJson = JSON.parse(
-    fs.readFileSync(path.resolve(INPUT_DIR, "package.json")).toString()
+  if (!diff && !newFiles) return date + commit.slice(0, 4);
+
+  const hash = createHash("md5");
+  hash.update(commit).update(diff).update(newFiles);
+  for (const file of newFiles.split("\n").filter((f) => f))
+    hash.update(await fsAsync.readFile(file));
+  return date + hash.digest("hex").slice(0, 4);
+}
+
+async function init(): Promise<void> {
+  const [buildMetadata, packageJsonFile, npmShrinkwrapFile] = await Promise.all(
+    [
+      getBuildMetadata(),
+      fsAsync.readFile(path.join(INPUT_DIR, "package.json")),
+      fsAsync.readFile(path.join(INPUT_DIR, "npm-shrinkwrap.json")),
+    ],
   );
+
+  const packageJson = JSON.parse(packageJsonFile.toString());
   delete packageJson["devDependencies"];
   delete packageJson["private"];
-  packageJson["scripts"] = {
-    install: packageJson["scripts"].install,
-    configure: packageJson["scripts"].configure,
-  };
-  packageJson["version"] = `${packageJson["version"]}+${BUILD_METADATA}`;
-  fs.writeFileSync(
-    path.resolve(OUTPUT_DIR, "package.json"),
-    JSON.stringify(packageJson, null, 2)
-  );
+  delete packageJson["scripts"];
+  packageJson["version"] = `${packageJson["version"]}+${buildMetadata}`;
 
-  // Create npm-shrinkwrap.json
-  const npmShrinkwrapJson = JSON.parse(
-    fs.readFileSync(path.resolve(INPUT_DIR, "npm-shrinkwrap.json")).toString()
-  );
-  npmShrinkwrapJson["version"] = packageJson["version"];
-  stripDevDeps(npmShrinkwrapJson);
-  fs.writeFileSync(
-    path.resolve(OUTPUT_DIR, "npm-shrinkwrap.json"),
-    JSON.stringify(npmShrinkwrapJson, null, 2)
-  );
-  return Object.keys(packageJson["dependencies"]);
+  const npmShrinkwrap = JSON.parse(npmShrinkwrapFile.toString());
+  npmShrinkwrap["version"] = packageJson["version"];
+  stripDevDeps(npmShrinkwrap);
+  stripDevDeps2(npmShrinkwrap);
+
+  await rmDir(OUTPUT_DIR);
+
+  await fsAsync.mkdir(OUTPUT_DIR);
+
+  await Promise.all([
+    fsAsync.mkdir(path.join(OUTPUT_DIR, "bin")),
+    fsAsync.mkdir(path.join(OUTPUT_DIR, "public")),
+    fsAsync.writeFile(
+      path.join(OUTPUT_DIR, "package.json"),
+      JSON.stringify(packageJson, null, 2),
+    ),
+    fsAsync.writeFile(
+      path.join(OUTPUT_DIR, "npm-shrinkwrap.json"),
+      JSON.stringify(npmShrinkwrap, null, 2),
+    ),
+  ]);
 }
 
 async function copyStatic(): Promise<void> {
@@ -156,149 +203,125 @@ async function copyStatic(): Promise<void> {
     "public/favicon.png",
   ];
 
-  for (const file of files) {
-    fs.copyFileSync(
-      path.resolve(INPUT_DIR, file),
-      path.resolve(OUTPUT_DIR, file)
-    );
-  }
+  const [logo, favicon] = await Promise.all([
+    fsAsync.readFile(path.join(INPUT_DIR, "public/logo.svg")),
+    fsAsync.readFile(path.join(INPUT_DIR, "public/favicon.png")),
+  ]);
+
+  ASSETS.LOGO_SVG = `logo-${assetHash(logo)}.svg`;
+  ASSETS.FAVICON_PNG = `favicon-${assetHash(favicon)}.png`;
+
+  const filenames = {} as Record<string, string>;
+  filenames["public/logo.svg"] = path.join("public", ASSETS.LOGO_SVG);
+  filenames["public/favicon.png"] = path.join("public", ASSETS.FAVICON_PNG);
+
+  await Promise.all(
+    files.map((f) =>
+      fsAsync.copyFile(
+        path.join(INPUT_DIR, f),
+        path.join(OUTPUT_DIR, filenames[f] || f),
+      ),
+    ),
+  );
 }
 
 async function generateCss(): Promise<void> {
-  const cssInPath = path.resolve(INPUT_DIR, "ui/css/app.css");
-  const cssOutPath = path.resolve(OUTPUT_DIR, "public/app.css");
-  const cssIn = fs.readFileSync(cssInPath);
-  const cssOut = await postcss([
-    postcssImport,
-    postcssPresetEnv({
-      stage: 3,
-      features: {
-        "nesting-rules": true,
-        "color-mod-function": true,
-      },
-    }),
-    cssnano,
-  ]).process(cssIn, { from: cssInPath, to: cssOutPath });
-  fs.writeFileSync(cssOutPath, cssOut.css);
+  const res = await esbuild.build({
+    bundle: true,
+    absWorkingDir: INPUT_DIR,
+    minify: MODE === "production",
+    sourcemap: "linked",
+    sourcesContent: false,
+    entryPoints: ["ui/css/app.css"],
+    entryNames: "[dir]/[name]-[hash]",
+    outfile: path.join(OUTPUT_DIR, "public/app.css"),
+    target: ["chrome109", "safari15.6", "firefox115", "opera102", "edge118"],
+    metafile: true,
+  });
+
+  for (const [k, v] of Object.entries(res.metafile.outputs)) {
+    if (v.entryPoint === "ui/css/app.css") {
+      ASSETS.APP_CSS = path.relative(
+        path.join(OUTPUT_DIR, "public"),
+        path.join(INPUT_DIR, k),
+      );
+      break;
+    }
+  }
 }
 
-async function generateBackendJs(externals: string[]): Promise<void> {
-  for (const bin of [
+async function generateBackendJs(): Promise<void> {
+  const services = [
     "genieacs-cwmp",
     "genieacs-ext",
     "genieacs-nbi",
     "genieacs-fs",
     "genieacs-ui",
-  ]) {
-    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}.ts`);
-    const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
-    const bundle = await rollup({
-      input: inputFile,
-      external: [...builtins, ...externals],
-      acorn: {
-        allowHashBang: true,
-      },
-      treeshake: {
-        propertyReadSideEffects: false,
-        moduleSideEffects: false,
-      },
-      plugins: [
-        rollupJson({ preferConst: true }),
-        {
-          name: "",
-          resolveId: (importee, importer) => {
-            if (importee.endsWith("/package.json")) {
-              const p = path.resolve(path.dirname(importer), importee);
-              if (p === path.resolve(INPUT_DIR, "package.json"))
-                return path.resolve(OUTPUT_DIR, "package.json");
-            }
-            return null;
-          },
-        },
-        typescript({
-          tsconfig: "./tsconfig.json",
-          include: [`bin/${bin}.ts`, "lib/**/*.ts"],
-        }),
-        MODE === "production" ? terser() : null,
-      ],
-    });
+  ];
 
-    await bundle.write({
-      format: "cjs",
-      preferConst: true,
-      sourcemap: "inline",
-      sourcemapExcludeSources: true,
-      banner: "#!/usr/bin/env node",
-      file: outputFile,
-    });
+  await esbuild.build({
+    bundle: true,
+    absWorkingDir: INPUT_DIR,
+    minify: MODE === "production",
+    sourcemap: "inline",
+    sourcesContent: false,
+    platform: "node",
+    target: "node12.13.0",
+    packages: "external",
+    banner: { js: "#!/usr/bin/env node" },
+    entryPoints: services.map((s) => `bin/${s}.ts`),
+    outdir: path.join(OUTPUT_DIR, "bin"),
+    plugins: [packageDotJsonPlugin, assetsPlugin],
+  });
 
+  for (const bin of services) {
+    const p = path.join(OUTPUT_DIR, "bin", bin);
+    await fsAsync.rename(`${p}.js`, p);
     // Mark as executable
-    const mode = fs.statSync(outputFile).mode;
-    fs.chmodSync(outputFile, mode | 73);
+    const mode = (await fsAsync.lstat(p)).mode;
+    await fsAsync.chmod(p, mode | 73);
   }
 }
 
-async function generateFrontendJs(externals: string[]): Promise<void> {
-  const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
-  const outputDir = path.resolve(OUTPUT_DIR, "public");
-
-  const inlineDeps = ["parsimmon", "espresso-iisojs"];
-  const bundle = await rollup({
-    input: inputFile,
-    external: [
-      ...builtins,
-      ...externals.filter((e) => !inlineDeps.includes(e)),
-    ],
-    plugins: [
-      rollupJson({ preferConst: true }),
-      {
-        name: "",
-        resolveId: function (importee, importer) {
-          if (importee.endsWith("/package.json")) {
-            const p = path.resolve(path.dirname(importer), importee);
-            if (p === path.resolve(INPUT_DIR, "package.json"))
-              return path.resolve(OUTPUT_DIR, "package.json");
-          }
-          return null;
-        },
-      },
-      typescript({ tsconfig: "./tsconfig.json" }),
-      nodeResolve(),
-      commonjs(),
-      MODE === "production" ? terser() : null,
-    ],
-    preserveEntrySignatures: false,
-    treeshake: {
-      propertyReadSideEffects: false,
-      moduleSideEffects: false,
-    },
-    onwarn: ((warning, warn) => {
-      // Ignore circular dependency warnings
-      if (warning.code !== "CIRCULAR_DEPENDENCY") warn(warning);
-    }) as WarningHandler,
+async function generateFrontendJs(): Promise<void> {
+  const res = await esbuild.build({
+    bundle: true,
+    absWorkingDir: INPUT_DIR,
+    splitting: true,
+    minify: MODE === "production",
+    sourcemap: "linked",
+    sourcesContent: false,
+    platform: "browser",
+    format: "esm",
+    target: ["chrome109", "safari15.6", "firefox115", "opera102", "edge118"],
+    entryPoints: ["ui/app.ts"],
+    entryNames: "[dir]/[name]-[hash]",
+    outdir: path.join(OUTPUT_DIR, "public"),
+    plugins: [packageDotJsonPlugin, inlineDepsPlugin, assetsPlugin],
+    metafile: true,
   });
 
-  await bundle.write({
-    manualChunks: (id) => {
-      if (id.includes("node_modules/codemirror")) return "codemirror";
-      else if (id.includes("node_modules/yaml")) return "yaml";
-      return "app";
-    },
-    preferConst: true,
-    format: "es",
-    sourcemap: true,
-    sourcemapExcludeSources: true,
-    dir: outputDir,
-  });
+  for (const [k, v] of Object.entries(res.metafile.outputs)) {
+    for (const imp of v.imports)
+      if (imp.external) throw new Error(`External import found: ${imp.path}`);
+
+    if (v.entryPoint === "ui/app.ts") {
+      ASSETS.APP_JS = path.relative(
+        path.join(OUTPUT_DIR, "public"),
+        path.join(INPUT_DIR, k),
+      );
+    }
+  }
 }
 
 async function generateIconsSprite(): Promise<void> {
-  const symbols = [];
-  const iconsDir = path.resolve(INPUT_DIR, "ui/icons");
-  for (const file of fs.readdirSync(iconsDir)) {
+  const symbols = [] as string[];
+  const iconsDir = path.join(INPUT_DIR, "ui/icons");
+  for (const file of await fsAsync.readdir(iconsDir)) {
     const id = path.parse(file).name;
     const filePath = path.join(iconsDir, file);
-    const { data } = await optimize(fs.readFileSync(filePath).toString(), {
+    const src = (await fsAsync.readFile(filePath)).toString();
+    const { data } = await optimize(src, {
       plugins: [
         {
           name: "preset-default",
@@ -312,28 +335,25 @@ async function generateIconsSprite(): Promise<void> {
     });
     symbols.push(generateSymbol(id, data));
   }
-  fs.writeFileSync(
-    path.resolve(OUTPUT_DIR, "public/icons.svg"),
-    `<svg xmlns="http://www.w3.org/2000/svg">${symbols.join("")}</svg>`
+  const data = `<svg xmlns="http://www.w3.org/2000/svg">${symbols.join(
+    "",
+  )}</svg>`;
+  ASSETS.ICONS_SVG = `icons-${assetHash(data)}.svg`;
+  await fsAsync.writeFile(
+    path.join(OUTPUT_DIR, "public", ASSETS.ICONS_SVG),
+    data,
   );
 }
 
 init()
-  .then((externals) => {
+  .then(() =>
     Promise.all([
-      copyStatic(),
+      Promise.all([generateIconsSprite(), copyStatic()]).then(
+        generateFrontendJs,
+      ),
       generateCss(),
-      generateIconsSprite(),
-      generateBackendJs(externals),
-      generateFrontendJs(externals),
-    ])
-      .then(() => {
-        // Ignore
-      })
-      .catch((err) => {
-        process.stderr.write(err.stack + "\n");
-      });
-  })
+    ]).then(generateBackendJs),
+  )
   .catch((err) => {
     process.stderr.write(err.stack + "\n");
   });

build/lint.ts

@@ -0,0 +1,55 @@
+import { exec } from "node:child_process";
+import { promisify } from "node:util";
+
+const execPromise = promisify(exec);
+
+async function runEslint(): Promise<string> {
+  const CMD =
+    "eslint 'bin/*.ts' 'lib/**/*.ts' 'ui/**/*.ts' 'test/**/*.ts' 'build/**/*.ts'";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  try {
+    const { stdout, stderr } = await execPromise(CMD, { env });
+    if (stderr) throw new Error(stderr);
+    return stdout;
+  } catch (err) {
+    if (err.killed || err.signal || err.stderr || err.code !== 1) throw err;
+    return err.stdout;
+  }
+}
+
+async function runTsc(): Promise<string> {
+  const CMD = "tsc --noEmit";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  const { stdout, stderr } = await execPromise(CMD, { env });
+  if (stderr) throw new Error(stderr);
+  return stdout;
+}
+
+async function runPrettier(): Promise<string> {
+  const CMD = "prettier --prose-wrap always --write .";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  const { stdout, stderr } = await execPromise(CMD, { env });
+  if (stderr) throw new Error(stderr);
+  return stdout;
+}
+
+async function runAll(): Promise<void> {
+  const prom1 = runPrettier();
+  const prom2 = runEslint();
+  const prom3 = runTsc();
+
+  console.log(await prom1);
+  console.log(await prom2);
+  console.log(await prom3);
+}
+
+runAll().catch(console.error);

build/spellcheck-dict.pws

@@ -168,3 +168,13 @@ IPv
 downloadSuccessOnTimeout
 TransferComplete
 nginx
+ENCODEURICOMPONENT
+pageSize
+skipRootGpn
+GPN
+NaN
+CIDR
+XMPP
+PEM
+JID
+unindexed

docs/.readthedocs.yaml

@@ -0,0 +1,18 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+
+sphinx:
+  configuration: docs/conf.py
+
+formats: all
+
+python:
+  install:
+    - requirements: docs/requirements.txt

docs/conf.py

@@ -19,7 +19,7 @@ import json
 # -- Project information -----------------------------------------------------
 
 project = 'GenieACS Documentation'
-copyright = '2019, GenieACS Inc.'
+copyright = '2024, GenieACS Inc.'
 author = 'GenieACS Inc.'
 
 # The full version, including alpha/beta/rc tags
@@ -58,7 +58,7 @@ html_theme_options = {
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 master_doc = 'index'
 

docs/environment-variables.rst

@@ -165,7 +165,7 @@ FS_ACCESS_LOG_FILE
   Default: unset
 
 FS_URL_PREFIX
-  The URL prefix (e.g. 'https://example.com:7657/') to use when generating the
+  The URL prefix (e.g. 'https://example.com:7567/') to use when generating the
   file URL for TR-069 Download requests. Set this if genieacs-fs and
   genieacs-cwmp are behind a proxy or running on different servers.
 

docs/ext-sample.js

@@ -16,7 +16,7 @@ function latlong(args, callback) {
     .get("http://api.open-notify.org/iss-now.json", (res) => {
       if (res.statusCode !== 200)
         return callback(
-          new Error(`Request failed (status code: ${res.statusCode})`)
+          new Error(`Request failed (status code: ${res.statusCode})`),
         );
 
       let rawData = "";

docs/installation-guide.rst

@@ -75,7 +75,12 @@ Configure systemd
     GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
     NODE_OPTIONS=--enable-source-maps
     GENIEACS_EXT_DIR=/opt/genieacs/ext
-    GENIEACS_UI_JWT_SECRET=secret
+
+  Generate a secure JWT secret and append to :file:`/opt/genieacs/genieacs.env`:
+
+  .. code:: bash
+
+    node -e "console.log(\"GENIEACS_UI_JWT_SECRET=\" + require('crypto').randomBytes(128).toString('hex'))" >> /opt/genieacs/genieacs.env
   
   Set file ownership and permissions:
 

docs/requirements.txt

@@ -0,0 +1 @@
+sphinx_rtd_theme==2.0.0

lib/api-functions.ts

@@ -1,49 +1,61 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as db from "./db";
-import * as common from "./common";
-import * as cache from "./cache";
+import { ObjectId } from "mongodb";
+import { collections } from "./db/db.ts";
 import {
-  getCurrentSnapshot,
+  deleteConfig,
+  deleteFault as dbDeleteFault,
+  deleteFile,
+  deletePermission,
+  deletePreset,
+  deleteProvision,
+  deleteTask,
+  deleteUser,
+  deleteVirtualParameter,
+  putConfig,
+  putPermission,
+  putPreset,
+  putProvision,
+  putUser,
+  putVirtualParameter,
+} from "./ui/db.ts";
+import * as common from "./util.ts";
+import * as cache from "./cache.ts";
+import { acquireLock, getToken, releaseLock } from "./lock.ts";
+import {
+  getRevision,
   getConfig,
   getConfigExpression,
-} from "./local-cache";
+  getUsers,
+} from "./ui/local-cache.ts";
 import {
   httpConnectionRequest,
   udpConnectionRequest,
-} from "./connection-request";
-import { Expression, Task } from "./types";
-import { flattenDevice } from "./mongodb-functions";
-import { evaluate } from "./common/expression";
+  xmppConnectionRequest,
+} from "./connection-request.ts";
+import { Expression, Task } from "./types.ts";
+import { evaluate } from "./common/expression/util.ts";
+import { hashPassword } from "./auth.ts";
+import { flattenDevice } from "./ui/db.ts";
+import { ResourceLockedError } from "./common/errors.ts";
+import * as config from "../lib/config.ts";
+
+const XMPP_CONFIGURED = !!config.get("XMPP_JID");
 
 export async function connectionRequest(
   deviceId: string,
-  device?: Record<string, { value?: [boolean | number | string, string] }>
+  device?: Record<string, { value?: [boolean | number | string, string] }>,
 ): Promise<string> {
   if (!device) {
-    const res = await db.devicesCollection.findOne({ _id: deviceId });
+    const res = await collections.devices.findOne({ _id: deviceId });
     if (!res) throw new Error("No such device");
     device = flattenDevice(res);
   }
 
-  let connectionRequestUrl, udpConnectionRequestAddress, username, password;
+  let connectionRequestUrl,
+    udpConnectionRequestAddress,
+    stunEnable,
+    connReqJabberId,
+    username,
+    password;
 
   if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
     connectionRequestUrl = (device[
@@ -54,6 +66,12 @@ export async function connectionRequest(
         "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
       ] || {}
     ).value || [""])[0];
+    stunEnable = ((
+      device["InternetGatewayDevice.ManagementServer.STUNEnable"] || {}
+    ).value || [""])[0];
+    connReqJabberId = ((
+      device["InternetGatewayDevice.ManagementServer.ConnReqJabberID"] || {}
+    ).value || [""])[0];
     username = ((
       device[
         "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
@@ -71,6 +89,10 @@ export async function connectionRequest(
     udpConnectionRequestAddress = ((
       device["Device.ManagementServer.UDPConnectionRequestAddress"] || {}
     ).value || [""])[0];
+    stunEnable = ((device["Device.ManagementServer.STUNEnable"] || {})
+      .value || [""])[0];
+    connReqJabberId = ((device["Device.ManagementServer.ConnReqJabberID"] || {})
+      .value || [""])[0];
     username = ((
       device["Device.ManagementServer.ConnectionRequestUsername"] || {}
     ).value || [""])[0];
@@ -78,8 +100,12 @@ export async function connectionRequest(
       device["Device.ManagementServer.ConnectionRequestPassword"] || {}
     ).value || [""])[0];
   }
-
-  const remoteAddress = new URL(connectionRequestUrl).hostname;
+  let remoteAddress;
+  try {
+    remoteAddress = new URL(connectionRequestUrl).hostname;
+  } catch (err) {
+    return "Invalid connection request URL";
+  }
 
   const evalCallback = (exp): Expression => {
     if (!Array.isArray(exp)) return exp;
@@ -103,32 +129,32 @@ export async function connectionRequest(
     return exp;
   };
 
-  const snapshot = await getCurrentSnapshot();
+  const snapshot = await getRevision();
   const now = Date.now();
   const UDP_CONNECTION_REQUEST_PORT = +getConfig(
     snapshot,
     "cwmp.udpConnectionRequestPort",
     {},
     now,
-    evalCallback
+    evalCallback,
   );
   const CONNECTION_REQUEST_TIMEOUT = +getConfig(
     snapshot,
     "cwmp.connectionRequestTimeout",
     {},
     now,
-    evalCallback
+    evalCallback,
   );
   const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = !!getConfig(
     snapshot,
     "cwmp.connectionRequestAllowBasicAuth",
     {},
     now,
-    evalCallback
+    evalCallback,
   );
   let authExp: Expression = getConfigExpression(
     snapshot,
-    "cwmp.connectionRequestAuth"
+    "cwmp.connectionRequestAuth",
   );
 
   if (authExp === undefined) {
@@ -145,7 +171,7 @@ export async function connectionRequest(
   const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
 
   let udpProm = Promise.resolve(false);
-  if (udpConnectionRequestAddress) {
+  if (udpConnectionRequestAddress && +stunEnable) {
     try {
       const u = new URL("udp://" + udpConnectionRequestAddress);
       udpProm = udpConnectionRequest(
@@ -154,24 +180,36 @@ export async function connectionRequest(
         authExp,
         UDP_CONNECTION_REQUEST_PORT,
         debug,
-        deviceId
+        deviceId,
       ).then(
         () => true,
-        () => false
+        () => false,
       );
     } catch (err) {
       // Ignore invalid address
     }
   }
 
-  const status = await httpConnectionRequest(
-    connectionRequestUrl,
-    authExp,
-    CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
-    CONNECTION_REQUEST_TIMEOUT,
-    debug,
-    deviceId
-  );
+  let status;
+
+  if (connReqJabberId && XMPP_CONFIGURED) {
+    status = await xmppConnectionRequest(
+      connReqJabberId,
+      authExp,
+      CONNECTION_REQUEST_TIMEOUT,
+      debug,
+      deviceId,
+    );
+  } else {
+    status = await httpConnectionRequest(
+      connectionRequestUrl,
+      authExp,
+      CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
+      CONNECTION_REQUEST_TIMEOUT,
+      debug,
+      deviceId,
+    );
+  }
 
   if (await udpProm) return "";
 
@@ -181,17 +219,17 @@ export async function connectionRequest(
 export async function awaitSessionStart(
   deviceId: string,
   lastInform: number,
-  timeout: number
+  timeout: number,
 ): Promise<boolean> {
   const now = Date.now();
-  const device = await db.devicesCollection.findOne(
+  const device = await collections.devices.findOne(
     { _id: deviceId },
-    { projection: { _lastInform: 1 } }
+    { projection: { _lastInform: 1 } },
   );
   const li = (device["_lastInform"] as Date).getTime();
   if (li > lastInform) return true;
-  const token = await cache.get(`cwmp_session_${deviceId}`);
-  if (token) return true;
+  const token = await getToken(`cwmp_session_${deviceId}`);
+  if (token?.startsWith("cwmp_session_")) return true;
   if (timeout < 500) return false;
   await new Promise((resolve) => setTimeout(resolve, 500));
   timeout -= Date.now() - now;
@@ -200,11 +238,11 @@ export async function awaitSessionStart(
 
 export async function awaitSessionEnd(
   deviceId: string,
-  timeout: number
+  timeout: number,
 ): Promise<boolean> {
   const now = Date.now();
-  const token = await cache.get(`cwmp_session_${deviceId}`);
-  if (!token) return true;
+  const token = await getToken(`cwmp_session_${deviceId}`);
+  if (!token?.startsWith("cwmp_session_")) return true;
   if (timeout < 500) return false;
   await new Promise((resolve) => setTimeout(resolve, 500));
   timeout -= Date.now() - now;
@@ -295,8 +333,8 @@ function sanitizeTask(task): void {
         !task.provisions.every((arr) =>
           arr.every(
             (s) =>
-              s == null || ["boolean", "number", "string"].includes(typeof s)
-          )
+              s == null || ["boolean", "number", "string"].includes(typeof s),
+          ),
         )
       )
         throw new Error("Invalid 'provisions' property");
@@ -322,30 +360,140 @@ export async function insertTasks(tasks: any[]): Promise<Task[]> {
   for (const task of tasks) {
     sanitizeTask(task);
     if (task.uniqueKey) {
-      await db.tasksCollection.deleteOne({
+      await collections.tasks.deleteOne({
         device: task.device,
         uniqueKey: task.uniqueKey,
       });
     }
   }
-  await db.tasksCollection.insertMany(tasks);
+  await collections.tasks.insertMany(tasks);
+  for (const task of tasks) task._id = task._id.toString();
   return tasks;
 }
 
 export async function deleteDevice(deviceId: string): Promise<void> {
-  await Promise.all([
-    db.tasksCollection.deleteMany({ device: deviceId }),
-    db.devicesCollection.deleteOne({ _id: deviceId }),
-    db.faultsCollection.deleteMany({
-      _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
-      },
-    }),
-    db.operationsCollection.deleteMany({
-      _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
-      },
-    }),
-    cache.del(`${deviceId}_tasks_faults_operations`),
-  ]);
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+  if (!token) throw new ResourceLockedError("Device is in session");
+  try {
+    await Promise.all([
+      collections.tasks.deleteMany({ device: deviceId }),
+      collections.devices.deleteOne({ _id: deviceId }),
+      collections.faults.deleteMany({
+        _id: {
+          $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+        },
+      }),
+      collections.operations.deleteMany({
+        _id: {
+          $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+        },
+      }),
+    ]);
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
+}
+
+export async function deleteFault(id: string): Promise<void> {
+  const deviceId = id.split(":", 1)[0];
+  const channel = id.slice(deviceId.length + 1);
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+  if (!token) throw new ResourceLockedError("Device is in session");
+  try {
+    const proms = [dbDeleteFault(id)];
+    if (channel.startsWith("task_"))
+      proms.push(deleteTask(new ObjectId(channel.slice(5))));
+    await Promise.all(proms);
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
+}
+
+export async function deleteResource(
+  resource: string,
+  id: string,
+): Promise<void> {
+  if (resource === "devices") {
+    await deleteDevice(id);
+  } else if (resource === "files") {
+    await deleteFile(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "faults") {
+    await deleteFault(id);
+  } else if (resource === "provisions") {
+    await deleteProvision(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "presets") {
+    await deletePreset(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "virtualParameters") {
+    await deleteVirtualParameter(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "config") {
+    await deleteConfig(id);
+    await Promise.all([
+      cache.del("ui-local-cache-hash"),
+      cache.del("cwmp-local-cache-hash"),
+    ]);
+  } else if (resource === "permissions") {
+    await deletePermission(id);
+    await cache.del("ui-local-cache-hash");
+  } else if (resource === "users") {
+    await deleteUser(id);
+    await cache.del("ui-local-cache-hash");
+  } else {
+    throw new Error(`Unknown resource ${resource}`);
+  }
+}
+
+// TODO Implement validation
+export async function putResource(
+  resource: string,
+  id: string,
+  data: any,
+): Promise<void> {
+  if (resource === "presets") {
+    await putPreset(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "provisions") {
+    await putProvision(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "virtualParameters") {
+    await putVirtualParameter(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "config") {
+    await putConfig(id, data);
+    await Promise.all([
+      cache.del("ui-local-cache-hash"),
+      cache.del("cwmp-local-cache-hash"),
+    ]);
+  } else if (resource === "permissions") {
+    await putPermission(id, data);
+    await cache.del("ui-local-cache-hash");
+  } else if (resource === "users") {
+    delete data["password"];
+    delete data["salt"];
+    await putUser(id, data);
+    await cache.del("ui-local-cache-hash");
+  } else {
+    throw new Error(`Unknown resource ${resource}`);
+  }
+}
+
+export function authLocal(
+  snapshot: string,
+  username: string,
+  password: string,
+): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    const users = getUsers(snapshot);
+    const user = users[username];
+    if (!user?.password) return void resolve(null);
+    hashPassword(password, user.salt)
+      .then((hash) => {
+        if (hash === user.password) resolve(true);
+        else resolve(false);
+      })
+      .catch(reject);
+  });
 }

lib/auth.ts

@@ -1,23 +1,4 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { createHash, randomBytes, pbkdf2 } from "crypto";
+import { createHash, randomBytes, pbkdf2 } from "node:crypto";
 
 function parseHeaderFeilds(str: string): Record<string, string> {
   const res = {};
@@ -63,7 +44,7 @@ export function parseAuthorizationHeader(authHeader: string): {
     // Inspired by https://github.com/jshttp/basic-auth
     const USER_PASS_REGEX = /^([^:]*):(.*)$/;
     const creds = USER_PASS_REGEX.exec(
-      Buffer.from(authHeader.slice(method.length + 1), "base64").toString()
+      Buffer.from(authHeader.slice(method.length + 1), "base64").toString(),
     );
 
     if (!creds) throw new Error("Unable to parse auth header");
@@ -77,7 +58,7 @@ export function parseAuthorizationHeader(authHeader: string): {
 }
 
 export function parseWwwAuthenticateHeader(
-  authHeader: string
+  authHeader: string,
 ): Record<string, string> {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
@@ -100,7 +81,7 @@ export function digest(
   qop?: string | Buffer,
   body?: string | Buffer,
   cnonce?: string | Buffer,
-  nc?: string | Buffer
+  nc?: string | Buffer,
 ): string {
   const ha1 = createHash("md5");
   ha1.update(username).update(":").update(realm).update(":").update(password);
@@ -141,7 +122,7 @@ export function solveDigest(
   uri: string | Buffer,
   httpMethod: string | Buffer,
   body: string | Buffer,
-  authHeader: Record<string, string>
+  authHeader: Record<string, string>,
 ): string {
   const cnonce = randomBytes(8).toString("hex");
   const nc = "00000001";
@@ -163,7 +144,7 @@ export function solveDigest(
     qop,
     body,
     cnonce,
-    nc
+    nc,
   );
 
   let authString = `Digest username="${username}"`;

lib/cache.ts

@@ -1,118 +1,35 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { Collection } from "mongodb";
-import { onConnect } from "./db";
-import * as config from "./config";
+import { collections } from "./db/db.ts";
+import * as config from "./config.ts";
 
 const CLOCK_SKEW_TOLERANCE = 30000;
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let cacheCollection: Collection;
-
-onConnect(async (db) => {
-  cacheCollection = db.collection("cache");
-  await cacheCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
-});
-
 export async function get(key: string): Promise<string> {
-  const res = await cacheCollection.findOne({ _id: key });
-  if (res) return res["value"];
-  return null;
+  const res = await collections.cache.findOne({ _id: key });
+  return res?.value;
 }
 
 export async function del(key: string): Promise<void> {
-  await cacheCollection.deleteOne({ _id: key });
+  await collections.cache.deleteOne({ _id: key });
 }
 
 export async function set(
   key: string,
   value: string,
-  ttl: number = MAX_CACHE_TTL
+  ttl: number = MAX_CACHE_TTL,
 ): Promise<void> {
   const timestamp = new Date();
   const expire = new Date(
-    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000
+    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000,
   );
-  await cacheCollection.replaceOne(
+  await collections.cache.replaceOne(
     { _id: key },
-    { _id: key, value, expire, timestamp },
-    { upsert: true }
+    { value, expire, timestamp },
+    { upsert: true },
   );
 }
 
-export async function pop(key: string): Promise<any> {
-  const res = await cacheCollection.findOneAndDelete({ _id: key });
-  if (res?.["value"]) return res["value"]["value"];
-  return null;
-}
-
-export async function acquireLock(
-  lockName: string,
-  ttl: number,
-  timeout = 0,
-  token = Math.random().toString(36).slice(2)
-): Promise<string> {
-  try {
-    const now = Date.now();
-    const r = await cacheCollection.findOneAndUpdate(
-      { _id: lockName, token },
-      {
-        $set: {
-          value: token,
-          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
-        },
-        $currentDate: { timestamp: true as unknown as Date },
-      },
-      { upsert: true, returnDocument: "after" }
-    );
-    const v = r.value;
-    if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
-      throw new Error("Database clock skew too great");
-  } catch (err) {
-    if (err.code === 11000) {
-      if (timeout > 0) {
-        return new Promise((resolve, reject) => {
-          const w = 50 + Math.random() * 50;
-          setTimeout(() => {
-            acquireLock(lockName, ttl, timeout - w, token).then(
-              resolve,
-              reject
-            );
-          }, w);
-        });
-      }
-      throw new Error("Failed to acquire lock");
-    }
-    throw err;
-  }
-
-  return token;
-}
-
-export async function releaseLock(
-  lockName: string,
-  token: string
-): Promise<void> {
-  const res = await cacheCollection.deleteOne({
-    _id: lockName,
-    value: token,
-  });
-  if (res.deletedCount !== 1) throw new Error("Lock expired");
+export async function pop(key: string): Promise<string> {
+  const res = await collections.cache.findOneAndDelete({ _id: key });
+  return res.value?.value;
 }

lib/cluster.ts

@@ -1,25 +1,6 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import cluster, { Worker } from "cluster";
-import { cpus } from "os";
-import * as logger from "./logger";
+import cluster, { Worker } from "node:cluster";
+import { cpus } from "node:os";
+import * as logger from "./logger.ts";
 
 let respawnTimestamp = 0;
 let crashes: number[] = [];
@@ -92,7 +73,7 @@ function restartWorker(worker, code, signal): void {
 export function start(
   workerCount: number,
   servicePort: number,
-  serviceAddress: string
+  serviceAddress: string,
 ): void {
   cluster.on("listening", (worker, address) => {
     if (

lib/common/authorizer.ts

@@ -1,33 +1,14 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { PermissionSet, Expression } from "../types";
-import { evaluate, or } from "./expression";
+import { PermissionSet, Expression } from "../types.ts";
+import { evaluate, or } from "./expression/util.ts";
 
 export default class Authorizer {
-  private declare permissionSets: PermissionSet[];
-  private declare validatorCache: WeakMap<
+  declare private permissionSets: PermissionSet[];
+  declare private validatorCache: WeakMap<
     any,
     (mutationType, mutation, any) => boolean
   >;
-  private declare hasAccessCache: Map<string, boolean>;
-  private declare getFilterCache: Map<string, Expression>;
+  declare private hasAccessCache: Map<string, boolean>;
+  declare private getFilterCache: Map<string, Expression>;
 
   public constructor(permissionSets: PermissionSet[]) {
     this.permissionSets = permissionSets;
@@ -78,7 +59,7 @@ export default class Authorizer {
 
   public getValidator(
     resourceType: string,
-    resource: unknown
+    resource: unknown,
   ): (mutationType: string, mutation?: any, args?: any) => boolean {
     if (this.validatorCache.has(resource))
       return this.validatorCache.get(resource);
@@ -99,7 +80,7 @@ export default class Authorizer {
     const validator = (
       mutationType: string,
       mutation: any,
-      any: any
+      any: any,
     ): boolean => {
       if (!validators.length) return false;
 
@@ -118,10 +99,9 @@ export default class Authorizer {
         if (["mutation", "options"].includes(entry)) {
           value = object[entry];
           for (const seg of paramName.split(".")) {
-            // typeof null is "object"
-            if (value != null && typeof value !== "object") value = null;
-            else value = value[seg];
             if (value == null) break;
+            if (typeof value !== "object") value = null;
+            else value = value[seg];
           }
         } else if (object[entry]) {
           if (paramName) value = object[entry][paramName];
@@ -134,7 +114,7 @@ export default class Authorizer {
       const res = evaluate(
         validators.length > 1 ? ["OR", validators] : validators[0],
         valueFunction,
-        Date.now()
+        Date.now(),
       );
       return !Array.isArray(res) && !!res;
     };

lib/common/debounce.ts

@@ -0,0 +1,16 @@
+export default function debounce<T>(
+  func: (args: T[]) => void,
+  timeout: number,
+): (arg: T) => void {
+  let timer: ReturnType<typeof setTimeout>;
+  let args: T[] = [];
+  return (arg: T) => {
+    args.push(arg);
+    clearTimeout(timer);
+    timer = setTimeout(() => {
+      const argscopy = args;
+      args = [];
+      func(argscopy);
+    }, timeout);
+  };
+}

lib/common/errors.ts

@@ -0,0 +1,6 @@
+export class ResourceLockedError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ResourceLockedError";
+  }
+}

lib/common/expression/bigint.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2020  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 export function and(a: bigint, b: bigint): bigint {
   return a & b;
 }
@@ -103,4 +84,4 @@ export function asIntN(a: number, b: bigint): bigint {
 
 const _BigInt = BigInt;
 type _bigint = bigint;
-export { _BigInt as BigInt, _bigint as bigint };
+export { _BigInt as BigInt, type _bigint as bigint };

lib/common/expression/normalize.ts

@@ -0,0 +1,492 @@
+import * as BI from "./bigint.ts";
+import { Expression } from "../../types.ts";
+import { map } from "./parser.ts";
+import { and, evaluateCallback } from "./util.ts";
+
+const ZERO = BI.BigInt(0);
+const ONE = BI.BigInt(1);
+const TWO = BI.BigInt(2);
+const NEGATIVE_ONE = BI.BigInt(-1);
+
+class Indeterminates {
+  declare public map: Map<string, number>;
+  declare public sortedKeys: string[];
+
+  public constructor(str?: string) {
+    this.map = new Map();
+    if (str) {
+      this.map.set(str, 1);
+      this.sortedKeys = [str];
+    } else {
+      this.sortedKeys = [];
+    }
+  }
+
+  public reciprocal(): Indeterminates {
+    const res = new Indeterminates();
+    res.sortedKeys = this.sortedKeys;
+    res.map = new Map();
+    for (const [k, v] of this.map) res.map.set(k, 0 - v);
+    return res;
+  }
+
+  public static multiply(
+    indeterminates1: Indeterminates,
+    indeterminates2: Indeterminates,
+  ): Indeterminates {
+    const res = new Indeterminates();
+    res.sortedKeys = indeterminates1.sortedKeys.slice();
+    res.map = new Map(indeterminates1.map);
+
+    for (const [key, val] of indeterminates2.map) {
+      const v = res.map.get(key);
+      if (!v) {
+        res.map.set(key, val);
+        res.sortedKeys.push(key);
+      } else {
+        const v2 = val + v;
+        if (!v2) {
+          res.map.delete(key);
+          res.sortedKeys = res.sortedKeys.filter((s) => s !== key);
+        } else {
+          res.map.set(key, v2);
+        }
+      }
+    }
+
+    res.sortedKeys.sort((a, b) => {
+      if (a.length !== b.length) return b.length - a.length;
+      else if (a > b) return 1;
+      else if (a < b) return -1;
+      return 0;
+    });
+
+    return res;
+  }
+
+  public static compare(a: Indeterminates, b: Indeterminates): number {
+    if (a.sortedKeys.length !== b.sortedKeys.length)
+      return b.sortedKeys.length - a.sortedKeys.length;
+    for (let i = 0; i < a.sortedKeys.length; ++i) {
+      const k1 = a.sortedKeys[i];
+      const w1 = a.map.get(k1);
+      const k2 = b.sortedKeys[i];
+      const w2 = b.map.get(k2);
+      if (w1 !== w2) return w2 - w1;
+      if (k1.length > k2.length) return -1;
+      else if (k1.length < k2.length) return 1;
+      else if (k1 > k2) return 1;
+      else if (k1 < k2) return -1;
+    }
+    return 0;
+  }
+}
+
+interface Term {
+  indeterminates: Indeterminates;
+  coefficientNumerator: BI.bigint;
+  coefficientDenominator: BI.bigint;
+}
+
+function findGcd(a: BI.bigint, b: BI.bigint): BI.bigint {
+  while (BI.ne(b, ZERO)) {
+    const t = b;
+    b = BI.rem(a, b);
+    a = t;
+  }
+  return a;
+}
+
+class Polynomial {
+  declare public terms: Term[];
+
+  public constructor(terms: Term[]) {
+    this.terms = terms;
+  }
+
+  public static simplifyTerms(terms: Term[]): Term[] {
+    const ts = terms
+      .slice()
+      .sort((a: Term, b: Term) =>
+        Indeterminates.compare(a.indeterminates, b.indeterminates),
+      );
+
+    for (let i = 1; i < ts.length; ++i) {
+      const t1 = ts[i - 1];
+      const t2 = ts[i];
+      if (Indeterminates.compare(t1.indeterminates, t2.indeterminates) === 0) {
+        const numerator = BI.add(
+          BI.mul(t1.coefficientNumerator, t2.coefficientDenominator),
+          BI.mul(t2.coefficientNumerator, t1.coefficientDenominator),
+        );
+
+        const denominator = BI.mul(
+          t1.coefficientDenominator,
+          t2.coefficientDenominator,
+        );
+
+        const gcd = findGcd(numerator, denominator);
+
+        ts[i] = {
+          indeterminates: t2.indeterminates,
+          coefficientNumerator: BI.div(numerator, gcd),
+          coefficientDenominator: BI.div(denominator, gcd),
+        };
+        ts[i - 1] = {
+          indeterminates: t1.indeterminates,
+          coefficientNumerator: ZERO,
+          coefficientDenominator: t1.coefficientDenominator,
+        };
+      }
+    }
+    return ts.filter((v) => BI.ne(v.coefficientNumerator, ZERO));
+  }
+
+  public static fromIndeterminate(indeterminate: Expression): Polynomial {
+    const indeterminates = new Indeterminates(JSON.stringify(indeterminate));
+    const terms = [
+      {
+        indeterminates: indeterminates,
+        coefficientNumerator: ONE,
+        coefficientDenominator: ONE,
+      },
+    ];
+    return new Polynomial(terms);
+  }
+
+  public static fromConstant(constant: number): Polynomial {
+    const [int, frac] = Math.abs(constant).toString(2).split(".", 2);
+    let numerator = BI.BigInt("0b" + int);
+    if (constant < 0) numerator = BI.mul(numerator, NEGATIVE_ONE);
+    let denominator = ONE;
+    if (frac) {
+      denominator = BI.exp(TWO, BI.BigInt(frac.length));
+      numerator = BI.add(
+        BI.mul(numerator, denominator),
+        BI.BigInt("0b" + frac),
+      );
+    }
+
+    const terms = [
+      {
+        indeterminates: new Indeterminates(),
+        coefficientNumerator: numerator,
+        coefficientDenominator: denominator,
+      },
+    ];
+
+    return new Polynomial(terms);
+  }
+
+  public negation(): Polynomial {
+    const terms = this.terms.map((t) => ({
+      indeterminates: t.indeterminates,
+      coefficientNumerator: BI.mul(t.coefficientNumerator, NEGATIVE_ONE),
+      coefficientDenominator: t.coefficientDenominator,
+    }));
+
+    return new Polynomial(terms);
+  }
+
+  public reciprocal(): Polynomial {
+    const terms = this.terms.map((t) => ({
+      indeterminates: t.indeterminates.reciprocal(),
+      coefficientNumerator: t.coefficientDenominator,
+      coefficientDenominator: t.coefficientNumerator,
+    }));
+    return new Polynomial(terms);
+  }
+
+  public constant(): Polynomial {
+    const terms = this.terms.filter((t) => !t.indeterminates.sortedKeys.length);
+    return new Polynomial(terms);
+  }
+
+  public add(rhs: Polynomial): Polynomial {
+    return new Polynomial(
+      Polynomial.simplifyTerms(this.terms.concat(rhs.terms)),
+    );
+  }
+
+  public subtract(rhs: Polynomial): Polynomial {
+    return this.add(rhs.negation());
+  }
+
+  public multiply(rhs: Polynomial): Polynomial {
+    const terms: Term[] = [];
+
+    for (const t1 of this.terms) {
+      for (const t2 of rhs.terms) {
+        const numerator = BI.mul(
+          t1.coefficientNumerator,
+          t2.coefficientNumerator,
+        );
+        const denominator = BI.mul(
+          t1.coefficientDenominator,
+          t2.coefficientDenominator,
+        );
+        const gcd = findGcd(numerator, denominator);
+
+        terms.push({
+          indeterminates: Indeterminates.multiply(
+            t1.indeterminates,
+            t2.indeterminates,
+          ),
+          coefficientNumerator: BI.div(numerator, gcd),
+          coefficientDenominator: BI.div(denominator, gcd),
+        });
+      }
+    }
+
+    return new Polynomial(Polynomial.simplifyTerms(terms));
+  }
+
+  public divide(rhs: Polynomial): Polynomial {
+    return this.multiply(rhs.reciprocal());
+  }
+
+  public toString(): string {
+    const add: string[] = [];
+    for (const t of this.terms) {
+      const coefficient =
+        BI.toNumber(t.coefficientNumerator) /
+        BI.toNumber(t.coefficientDenominator);
+
+      const mul: string[] = [];
+      if (t.indeterminates.sortedKeys.length) {
+        for (const k of t.indeterminates.sortedKeys) {
+          const w = t.indeterminates.map.get(k);
+          for (let i = Math.abs(w); i > 0; --i) {
+            if (w > 0) mul.push(k);
+            else mul.push(`["/",1,${k}]`);
+          }
+        }
+
+        if (coefficient !== 1) mul.push(coefficient.toString());
+
+        if (mul.length > 1) add.push(`["*",${mul.join(",")}]`);
+        else add.push(mul["0"]);
+      } else {
+        add.push(coefficient.toString());
+      }
+    }
+
+    if (!add.length) return "0";
+    else if (add.length === 1) return add[0];
+    else return `["+",${add.join(",")}]`;
+  }
+}
+
+const ADDITIVE_IDENTITY = Polynomial.fromConstant(0);
+const MULTIPLICATIVE_IDENTITY = Polynomial.fromConstant(1);
+
+const SWAPPED_OPS = {
+  "=": "=",
+  "<>": "<>",
+  ">": "<",
+  ">=": "<=",
+  "<": ">",
+  "<=": ">=",
+};
+
+function normalizeCallback(exp: Expression): Expression {
+  if (!Array.isArray(exp)) return exp;
+  const op = exp[0];
+
+  if (op === "FUNC" && exp[1] === "COALESCE") {
+    const res: Expression[] = ["CASE"];
+    for (let i = 2; i < exp.length; ++i)
+      res.push(normalizeCallback(["IS NOT NULL", exp[i]]), exp[i]);
+    return normalizeCallback(res);
+  }
+
+  if (op === "CASE") {
+    const res = [] as [Expression, Expression][];
+    for (let i = 1; i < exp.length; i += 2) {
+      let w = exp[i];
+      if (w instanceof Polynomial) w = JSON.parse(w.toString());
+      if (!Array.isArray(w) && !w) continue;
+      const t = exp[i + 1];
+      if (!Array.isArray(t) || t[0] !== "CASE") {
+        res.push([w, t]);
+        continue;
+      }
+      for (let j = 1; j < t.length; j += 2) res.push([and(w, t[j]), t[j + 1]]);
+      res.push([w, null]);
+      if (!Array.isArray(w) && w) break;
+    }
+    while (res[res.length - 1][1] == null) res.pop();
+    return ["CASE", ...res.flat()];
+  }
+
+  const permutations: Map<number, [Expression, Expression][]> = new Map();
+  for (const [i, e] of exp.entries()) {
+    if (!Array.isArray(e)) continue;
+    if (e[0] !== "CASE") continue;
+    const perms: [Expression, Expression][] = [];
+    for (let j = 1; j < e.length; j += 2) perms.push([e[j], e[j + 1]]);
+    perms.push([true, null]);
+    permutations.set(i, perms);
+  }
+
+  if (permutations.size) {
+    let res: [Expression, Expression][] = [[true, exp]];
+    for (const [i, perms] of permutations) {
+      const res2: [Expression, Expression][] = [];
+      for (const [w, t] of perms) {
+        res2.push(
+          ...res.map((r) => {
+            const e = (r[1] as Expression[]).slice();
+            e[i] = t;
+            return [and(w, r[0]), e] as [Expression, Expression];
+          }),
+        );
+        if (!Array.isArray(w) && w) break;
+      }
+      res = res2;
+    }
+
+    for (const r of res) r[1] = normalizeCallback(r[1]);
+
+    if (res[0][0] === true) return res[0][1];
+    while (res[res.length - 1][1] == null) res.pop();
+
+    return ["CASE", ...res.flat()];
+  }
+
+  function toPolynomial(e: Expression): Polynomial {
+    if (e == null) return null;
+    if (e instanceof Polynomial) return e;
+    if (typeof e === "number") return Polynomial.fromConstant(e);
+    if (typeof e === "string")
+      return Polynomial.fromConstant(parseFloat(e) || 0);
+    if (typeof e === "boolean") return Polynomial.fromConstant(+e);
+    return Polynomial.fromIndeterminate(e);
+  }
+
+  if (op === "+") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce(
+      (previousValue, currentValue) => previousValue.add(currentValue),
+      ADDITIVE_IDENTITY,
+    ) as unknown as Expression;
+  } else if (op === "*") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce(
+      (previousValue, currentValue) => previousValue.multiply(currentValue),
+      MULTIPLICATIVE_IDENTITY,
+    ) as unknown as Expression;
+  } else if (op === "-") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce((previousValue, currentValue) =>
+      previousValue.subtract(currentValue),
+    ) as unknown as Expression;
+  } else if (op === "/") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce((previousValue, currentValue) =>
+      previousValue.divide(currentValue),
+    ) as unknown as Expression;
+  } else if (["=", "<>", ">", ">=", "<", "<="].includes(op)) {
+    if (exp[1] == null || exp[2] == null) return null;
+    let lhs: Polynomial, rhs: Polynomial;
+    if (exp[1] instanceof Polynomial) lhs = exp[1];
+    else if (typeof exp[1] === "number") lhs = Polynomial.fromConstant(exp[1]);
+
+    if (exp[2] instanceof Polynomial) rhs = exp[2];
+    else if (typeof exp[2] === "number") rhs = Polynomial.fromConstant(exp[2]);
+
+    if (lhs || rhs) {
+      if (!lhs) lhs = Polynomial.fromIndeterminate(exp[1]);
+      if (!rhs) rhs = Polynomial.fromIndeterminate(exp[2]);
+
+      lhs = lhs.subtract(rhs);
+      rhs = lhs.constant().negation();
+      lhs = lhs.add(rhs);
+
+      if (!lhs.terms.length) {
+        exp = [op, JSON.parse(lhs.toString()), JSON.parse(rhs.toString())];
+
+        if (op === "=") return exp[1] === exp[2];
+        else if (op === "<>") return exp[1] !== exp[2];
+        else if (op === ">") return exp[1] > exp[2];
+        else if (op === ">=") return exp[1] >= exp[2];
+        else if (op === "<") return exp[1] < exp[2];
+        else if (op === "<=") return exp[1] <= exp[2];
+      } else {
+        let flipOp = 1;
+
+        const n = lhs.terms[0].coefficientNumerator;
+        const d = lhs.terms[0].coefficientDenominator;
+
+        if (BI.lt(n, ZERO) || BI.lt(d, ZERO)) flipOp *= -1;
+
+        const reciprocal = new Polynomial([
+          {
+            indeterminates: new Indeterminates(),
+            coefficientNumerator: d,
+            coefficientDenominator: n,
+          },
+        ]);
+
+        lhs = lhs.multiply(reciprocal);
+        rhs = rhs.multiply(reciprocal);
+
+        const keys = lhs.terms[0].indeterminates.sortedKeys;
+        let invert = lhs.terms[0].indeterminates.map.get(keys[0]) < 0 ? -1 : 0;
+
+        for (const t of lhs.terms)
+          for (const v of t.indeterminates.map.values()) invert += v;
+
+        if (invert < 0) {
+          flipOp *= -1;
+          lhs = lhs.reciprocal();
+          rhs = rhs.reciprocal();
+        }
+
+        if (flipOp < 0) exp = [SWAPPED_OPS[op], lhs, rhs];
+        else exp = [op, lhs, rhs];
+      }
+    }
+  }
+
+  // Restore polynomial expressions
+  exp = exp.map((e) =>
+    e instanceof Polynomial ? JSON.parse(e.toString()) : e,
+  );
+
+  exp = evaluateCallback(exp);
+
+  return exp;
+}
+
+export default function normalize(expr: Expression): Expression {
+  expr = map(expr, normalizeCallback);
+  if (expr instanceof Polynomial) {
+    expr = JSON.parse(expr.toString());
+  } else if (Array.isArray(expr) && expr[0] === "CASE") {
+    expr = expr.map((e) =>
+      e instanceof Polynomial ? JSON.parse(e.toString()) : e,
+    );
+  }
+  return expr;
+}

lib/common/expression/pagination.ts

@@ -0,0 +1,148 @@
+import { complement } from "espresso-iisojs";
+import { Expression } from "../../types";
+import normalize from "./normalize";
+import { Clause, SynthContext } from "./synth";
+import { and, or } from "./util";
+
+type Bookmark = Record<string, null | boolean | number | string>;
+
+type Minterm = number[];
+
+export function toBookmark(
+  sort: Record<string, number>,
+  row: unknown,
+): Bookmark {
+  const bookmark: Bookmark = {};
+  for (const param of Object.keys(sort)) {
+    let v = row[param];
+    if (v != null && typeof v === "object") v = v.value?.[0];
+    bookmark[param] = v;
+  }
+  return bookmark;
+}
+
+export function bookmarkToExpression(
+  bookmark: Bookmark,
+  sort: Record<string, number>,
+): Expression {
+  return Object.entries(sort)
+    .reverse()
+    .reduce((cur, kv) => {
+      const [param, asc] = kv;
+      if (asc < 0) {
+        if (bookmark[param] == null) {
+          return or(
+            ["IS NOT NULL", ["PARAM", param]],
+            and(["IS NULL", ["PARAM", param]], cur),
+          );
+        }
+        let f = null;
+        f = or(f, [">", ["PARAM", param], bookmark[param]]);
+        return or(f, and(["=", ["PARAM", param], bookmark[param]], cur));
+      } else {
+        let f: Expression = ["IS NULL", ["PARAM", param]];
+        if (bookmark[param] == null) return and(f, cur);
+        f = or(f, ["<", ["PARAM", param], bookmark[param]]);
+        return or(f, and(["=", ["PARAM", param], bookmark[param]], cur));
+      }
+    }, true as Expression);
+}
+
+function getCover(
+  context: SynthContext,
+  minterm: Minterm,
+  allSort: [string, number][],
+): Minterm[] {
+  if (!allSort.length) return [[]];
+  const [param, sort] = allSort[0];
+
+  const cov: Minterm = [];
+  const nextCov: Minterm = [];
+
+  if (sort > 0) {
+    const lhs = new Clause.Exp(["PARAM", param]);
+    const isNull = context.getVar(new Clause.IsNull(lhs));
+    cov.push((isNull << 2) ^ 2);
+  }
+
+  for (const m of minterm) {
+    const clause = context.getClause(m >>> 2);
+    if (clause instanceof Clause.IsNull) {
+      const lhs = clause.operand.expression();
+      if (!Array.isArray(lhs) || lhs[0] !== "PARAM") continue;
+      if (lhs[1] !== param) continue;
+      nextCov.push(m);
+      if (sort < 0 && m & 1) cov.push(m, m ^ 1);
+    } else if (clause instanceof Clause.Compare) {
+      const lhs = clause.lhs.expression();
+      if (!Array.isArray(lhs) || lhs[0] !== "PARAM") continue;
+      if (lhs[1] !== param) continue;
+      if (!(m & 1) && sort > 0) continue;
+      nextCov.push(m);
+
+      const negate = (m ^ (m >> 1)) & 1;
+
+      if (sort > 0) {
+        if (
+          (clause.op === "=" && !negate) ||
+          (clause.op === ">" && !negate) ||
+          (clause.op === "<" && negate)
+        ) {
+          const c = new Clause.Compare(clause.lhs, ">", clause.rhs);
+          const v = context.getVar(c);
+          cov.push((v << 2) ^ 3);
+        }
+      } else if (sort < 0) {
+        if (
+          (clause.op === "=" && !negate) ||
+          (clause.op === ">" && negate) ||
+          (clause.op === "<" && !negate)
+        ) {
+          const c = new Clause.Compare(clause.lhs, "<", clause.rhs);
+          const v = context.getVar(c);
+          cov.push((v << 2) ^ 0);
+        }
+      }
+    }
+  }
+
+  const next = getCover(context, minterm, allSort.slice(1));
+
+  return [cov, ...next.map((n) => [...nextCov, ...n])];
+}
+
+export function paginate(
+  fetched: Expression,
+  toFetch: Expression,
+  sort: Record<string, number>,
+): [Expression, Expression] {
+  fetched = normalize(fetched);
+  if (!fetched) return [false, toFetch];
+
+  toFetch = normalize(toFetch);
+  if (!toFetch) return [false, toFetch];
+
+  const synth1 = Clause.fromExpression(fetched);
+  const synth2 = Clause.fromExpression(toFetch);
+
+  const context = new SynthContext();
+
+  const expr1Minterms = synth1.true(context);
+  const expr2MintermsC = complement(synth2.true(context));
+
+  const gaps = context.sanitizeMinterms(
+    complement([
+      ...expr1Minterms,
+      ...expr2MintermsC,
+      ...context.getDcSet([...expr1Minterms, ...expr2MintermsC]),
+    ]),
+  );
+
+  const cover = gaps.flatMap((m) => getCover(context, m, Object.entries(sort)));
+  const minterms1 = context.minimize(complement([...cover, ...expr2MintermsC]));
+  const minterms2 = context.minimize(
+    complement([...minterms1, ...expr2MintermsC]),
+  );
+
+  return [context.toExpression(minterms1), context.toExpression(minterms2)];
+}

lib/common/expression/parser.ts

@@ -1,24 +1,5 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 import parsimmon from "parsimmon";
-import { Expression } from "../types";
+import { Expression } from "../../types.ts";
 
 // Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
 function interpretEscapes(str): string {
@@ -40,10 +21,7 @@ function interpretEscapes(str): string {
   });
 }
 
-export function map(
-  exp: Expression,
-  callback: (e: Expression) => Expression
-): Expression {
+export function map<T>(exp: Expression, callback: (e: Expression) => T): T {
   if (!Array.isArray(exp)) return callback(exp);
 
   let clone;
@@ -60,7 +38,7 @@ export function map(
 
 export async function mapAsync(
   exp: Expression,
-  callback: (e: Expression) => Promise<Expression>
+  callback: (e: Expression) => Promise<Expression>,
 ): Promise<Expression> {
   if (!Array.isArray(exp)) return callback(exp);
 
@@ -78,7 +56,7 @@ export async function mapAsync(
 
 function binaryLeft(
   operatorsParser: parsimmon.Parser<string>,
-  nextParser: parsimmon.Parser<Expression>
+  nextParser: parsimmon.Parser<Expression>,
 ): parsimmon.Parser<Expression> {
   return parsimmon.seqMap(
     nextParser,
@@ -87,10 +65,8 @@ function binaryLeft(
       rest.reduce((acc, ch) => {
         const [op, another] = ch;
         if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
-        if (Array.isArray(another) && op === another[0])
-          return [op, acc].concat(another.slice(1));
         return [op, acc, another];
-      }, first)
+      }, first),
   );
 }
 
@@ -103,7 +79,7 @@ const lang = parsimmon.createLanguage({
         parsimmon.string("<="),
         parsimmon.string("="),
         parsimmon.string(">"),
-        parsimmon.string("<")
+        parsimmon.string("<"),
       )
       .skip(parsimmon.optWhitespace);
   },
@@ -114,7 +90,7 @@ const lang = parsimmon.createLanguage({
         parsimmon
           .regexp(/not\s+like/i)
           .result("NOT LIKE")
-          .desc("NOT LIKE")
+          .desc("NOT LIKE"),
       )
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
@@ -129,7 +105,7 @@ const lang = parsimmon.createLanguage({
         parsimmon
           .regexp(/is\s+not\s+null/i)
           .result("IS NOT NULL")
-          .desc("IS NOT NULL")
+          .desc("IS NOT NULL"),
       )
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
@@ -164,8 +140,8 @@ const lang = parsimmon.createLanguage({
         parsimmon.regexp(/[a-zA-Z0-9_.*-]+/),
         r.Expression.wrap(
           parsimmon.string("{").skip(parsimmon.optWhitespace),
-          parsimmon.string("}")
-        )
+          parsimmon.string("}"),
+        ),
       )
       .atLeast(1)
       .map((x) => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
@@ -199,7 +175,7 @@ const lang = parsimmon.createLanguage({
     return parsimmon
       .alt(
         parsimmon.regexp(/true/i).result(true).desc("TRUE"),
-        parsimmon.regexp(/false/i).result(false).desc("FALSE")
+        parsimmon.regexp(/false/i).result(false).desc("FALSE"),
       )
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
@@ -220,9 +196,9 @@ const lang = parsimmon.createLanguage({
         .desc("function"),
       r.ExpressionList.wrap(
         parsimmon.string("(").skip(parsimmon.optWhitespace),
-        parsimmon.string(")").skip(parsimmon.optWhitespace)
+        parsimmon.string(")").skip(parsimmon.optWhitespace),
       ),
-      (f, args) => ["FUNC", f.toUpperCase()].concat(args)
+      (f, args) => ["FUNC", f.toUpperCase()].concat(args),
     );
   },
   WhenPair: function (r) {
@@ -238,7 +214,7 @@ const lang = parsimmon.createLanguage({
         .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
         .skip(parsimmon.optWhitespace)
         .desc("THEN")
-        .then(r.Expression)
+        .then(r.Expression),
     );
   },
   CaseStatement: function (r) {
@@ -261,10 +237,10 @@ const lang = parsimmon.createLanguage({
         .skip(
           parsimmon
             .regex(/end/i)
-            .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+            .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/)),
         )
         .skip(parsimmon.optWhitespace),
-      (...arr) => arr.flat(2)
+      (...arr) => arr.flat(2),
     );
   },
   Value: function (r) {
@@ -275,7 +251,7 @@ const lang = parsimmon.createLanguage({
       r.StringValueSql,
       r.StringValueJs,
       r.FuncValue,
-      r.CaseStatement
+      r.CaseStatement,
     );
   },
   ValueExpression: function (r) {
@@ -287,18 +263,22 @@ const lang = parsimmon.createLanguage({
           .skip(parsimmon.optWhitespace),
         binaryLeft(
           parsimmon
-            .alt(parsimmon.string("*"), parsimmon.string("/"))
+            .alt(
+              parsimmon.string("*"),
+              parsimmon.string("/"),
+              parsimmon.string("%"),
+            )
             .skip(parsimmon.optWhitespace),
           parsimmon.alt(
             r.Value,
             r.Parameter,
             r.Expression.wrap(
               parsimmon.string("(").skip(parsimmon.optWhitespace),
-              parsimmon.string(")").skip(parsimmon.optWhitespace)
-            )
-          )
-        )
-      )
+              parsimmon.string(")").skip(parsimmon.optWhitespace),
+            ),
+          ),
+        ),
+      ),
     );
   },
   Comparison: function (r) {
@@ -308,7 +288,7 @@ const lang = parsimmon.createLanguage({
         r.ValueExpression,
         r.ComparisonOperator,
         r.ValueExpression,
-        (p, o, v) => [o, p, v]
+        (p, o, v) => [o, p, v],
       ),
       parsimmon.seqMap(
         r.ValueExpression,
@@ -318,28 +298,28 @@ const lang = parsimmon.createLanguage({
             .regexp(/escape/i)
             .result("ESCAPE")
             .skip(parsimmon.whitespace)
-            .desc("ESCAPE")
+            .desc("ESCAPE"),
         ),
         r.ValueExpression,
-        (a, b, c, d) => [b, a, c, d]
+        (a, b, c, d) => [b, a, c, d],
       ),
       parsimmon.seqMap(
         r.ValueExpression,
         r.LikeOperator,
         r.ValueExpression,
-        (a, b, c) => [b, a, c]
-      )
+        (a, b, c) => [b, a, c],
+      ),
     );
   },
   ExpressionList: function (r) {
     return r.Expression.sepBy(
-      parsimmon.string(",").skip(parsimmon.optWhitespace)
+      parsimmon.string(",").skip(parsimmon.optWhitespace),
     );
   },
   Expression: function (r) {
     function unary(
       operatorsParser: parsimmon.Parser<string>,
-      nextParser: parsimmon.Parser<Expression>
+      nextParser: parsimmon.Parser<Expression>,
     ): parsimmon.Parser<Expression> {
       return parsimmon.seq(operatorsParser, nextParser).or(nextParser);
     }
@@ -348,8 +328,8 @@ const lang = parsimmon.createLanguage({
       r.OrOperator,
       binaryLeft(
         r.AndOperator,
-        unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
-      )
+        unary(r.NotOperator, r.Comparison.or(r.ValueExpression)),
+      ),
     ).trim(parsimmon.optWhitespace);
   },
 });
@@ -386,6 +366,7 @@ export function stringify(exp: Expression, level = 0): string {
     "-": 31,
     "*": 32,
     "/": 32,
+    "%": 32,
   };
 
   const op = exp[0].toUpperCase();
@@ -400,7 +381,7 @@ export function stringify(exp: Expression, level = 0): string {
       `${exp[1]}(${exp
         .slice(2)
         .map((e) => stringify(e))
-        .join(", ")})`
+        .join(", ")})`,
     );
   } else if (op === "PARAM") {
     if (typeof exp[1] === "string") {
@@ -413,7 +394,7 @@ export function stringify(exp: Expression, level = 0): string {
             if (typeof p === "string") return p;
             else return `{${stringify(p)}}`;
           })
-          .join("")
+          .join(""),
       );
     } else {
       return wrap(`{${stringify(exp[1])}}`);
@@ -425,15 +406,15 @@ export function stringify(exp: Expression, level = 0): string {
       return wrap(
         `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
           exp[2],
-          opLevels[op]
-        )} ESCAPE ${stringify(exp[3], opLevels[op])}`
+          opLevels[op],
+        )} ESCAPE ${stringify(exp[3], opLevels[op])}`,
       );
     } else {
       return wrap(
         `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
           exp[2],
-          opLevels[op]
-        )}`
+          opLevels[op],
+        )}`,
       );
     }
   } else if (op === "CASE") {

lib/common/expression/synth.ts

@@ -0,0 +1,972 @@
+import { espresso, complement, tautology } from "espresso-iisojs";
+import { Expression } from "../../types.ts";
+import normalize from "./normalize.ts";
+import { map, parseLikePattern } from "./parser.ts";
+
+type Minterm = number[];
+
+export abstract class SynthContextBase<
+  T extends { toString: () => string } = unknown,
+  U = unknown,
+> {
+  public variables = new Map<string, number>();
+  protected clauses = new Map<number, U>();
+
+  public getVar(c: U): number {
+    const str = c.toString();
+    let idx = this.variables.get(str);
+    if (idx == null) {
+      idx = this.variables.size;
+      this.variables.set(str, idx);
+      this.clauses.set(idx, c);
+    }
+    return idx;
+  }
+
+  public getClause(v: number): U {
+    return this.clauses.get(v);
+  }
+
+  abstract getMinterms(exp: T, res: true | false | null): Minterm[];
+  abstract getDcSet(minterms: Minterm[]): Minterm[];
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  canRaise(idx: number, set: Set<number>): boolean {
+    return true;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  canLower(idx: number, set: Set<number>): boolean {
+    return true;
+  }
+
+  bias(a: number, b: number): number {
+    // Bias towards 1 then lower index
+    return (b ^ 1) - (a ^ 1);
+  }
+
+  sanitizeMinterms(minterms: Minterm[]): Minterm[] {
+    return minterms;
+  }
+
+  minimize(minterms: Minterm[], dcSet: Minterm[] = []): Minterm[] {
+    minterms = this.sanitizeMinterms(minterms);
+    const canRaise = this.canRaise.bind(this);
+    const canLower = this.canLower.bind(this);
+    const bias = this.bias.bind(this);
+
+    return espresso(
+      minterms,
+      [...this.getDcSet([...minterms, ...dcSet]), ...dcSet],
+      {
+        canRaise,
+        canLower,
+        bias,
+      },
+    );
+  }
+}
+
+function* findIsNullDeps(exp: Expression): IterableIterator<Expression> {
+  if (!Array.isArray(exp)) return;
+  const op = exp[0];
+  if (op === "IS NULL" || op === "IS NOT NULL") return;
+
+  if (op === "FUNC") {
+    if (exp[1] === "NOW") {
+      return;
+    } else if (exp[1] === "LOWER" || exp[1] === "UPPER") {
+      yield* findIsNullDeps(exp[2]);
+    } else if (exp[1] === "ROUND") {
+      for (const e of exp.slice(2, 4)) yield* findIsNullDeps(e);
+      return;
+    }
+  } else if (op !== "PARAM") {
+    for (const e of exp.slice(1)) yield* findIsNullDeps(e);
+    return;
+  }
+  yield exp;
+}
+
+export abstract class Clause {
+  private _isNullable: Set<string>;
+  protected _expression: Expression;
+  abstract true(context: SynthContextBase<Clause>): Minterm[];
+  abstract false(context: SynthContextBase<Clause>): Minterm[];
+  abstract null(context: SynthContextBase<Clause>): Minterm[];
+  expression(): Expression {
+    if (this._expression !== undefined) return this._expression;
+    const context = createSynthContext();
+    const minterms = this.true(context);
+    const minimized = context.minimize(minterms);
+    this._expression = context.toExpression(minimized) as Expression;
+    return this._expression;
+  }
+  isBoolean(): boolean {
+    return true;
+  }
+  isNullable(c: Clause.IsNull): boolean {
+    if (!this._isNullable) {
+      this._isNullable = new Set(
+        [...this.getNullables()].map((n) => n.toString()),
+      );
+    }
+    return this._isNullable.has(c.toString());
+  }
+  *getNullables(): IterableIterator<Clause.IsNull> {
+    const exp = this.expression();
+    for (const e of findIsNullDeps(exp)) {
+      if (e === exp) yield new Clause.IsNull(this);
+      else yield new Clause.IsNull(new Clause.Exp(e));
+    }
+  }
+  toString(): string {
+    return JSON.stringify(this.expression());
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/no-namespace
+export namespace Clause {
+  export class Not extends Clause {
+    constructor(public operand: Clause) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.false(context);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.true(context);
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.null(context);
+    }
+  }
+
+  export class Or extends Clause {
+    constructor(public operands: Clause[]) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      for (const s of this.operands) {
+        const m = s.true(context);
+        if (m.length === 1 && !m[0].length) return [[]];
+        minterms.push(...m);
+      }
+      return minterms;
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[][] = [];
+      const operands = [...this.operands];
+      for (const e of operands) {
+        if (e instanceof Or) {
+          operands.push(...e.operands);
+          continue;
+        }
+        const m = e.false(context);
+        if (!m.length) return [];
+        minterms.push(m);
+      }
+      if (!minterms.length) return [[]];
+      if (minterms.length === 1) return minterms[0];
+      return complement(minterms.map((m) => complement(m)).flat());
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      const trueMinterms: Minterm[] = [];
+      const operands = [...this.operands];
+      for (const s of operands) {
+        if (s instanceof Or) {
+          operands.push(...s.operands);
+          continue;
+        }
+        const t = s.true(context);
+        if (t.length === 1 && !t[0].length) return [];
+        trueMinterms.push(...t);
+      }
+
+      const nullMinterms: Minterm[] = [];
+      for (const s of operands) {
+        const n = s.null(context);
+        if (n.length === 1 && !n[0].length && !trueMinterms.length) return [[]];
+        nullMinterms.push(...n);
+      }
+
+      if (!trueMinterms.length) return nullMinterms;
+      return complement([...complement(nullMinterms), ...trueMinterms]);
+    }
+  }
+
+  export class And extends Clause {
+    constructor(public operands: Clause[]) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[][] = [];
+      const operands = [...this.operands];
+      for (const s of operands) {
+        if (s instanceof And) {
+          operands.push(...s.operands);
+          continue;
+        }
+        const m = s.true(context);
+        if (!m.length) return [];
+        if (m.length === 1 && !m[0].length) continue;
+        minterms.push(m);
+      }
+      if (!minterms.length) return [[]];
+      if (minterms.length === 1) return minterms[0];
+      return complement(minterms.map((m) => complement(m)).flat());
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      for (const s of this.operands) {
+        const m = s.false(context);
+        if (m.length === 1 && !m[0].length) return [[]];
+        minterms.push(...m);
+      }
+      return minterms;
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      const falseMinterms: Minterm[] = [];
+      const operands = [...this.operands];
+      for (const s of operands) {
+        if (s instanceof And) {
+          operands.push(...s.operands);
+          continue;
+        }
+        const f = s.false(context);
+        if (f.length === 1 && !f[0].length) return [];
+        falseMinterms.push(...f);
+      }
+
+      const nullMinterms: Minterm[] = [];
+      for (const s of operands) {
+        const n = s.null(context);
+        if (n.length === 1 && !n[0].length && !falseMinterms.length)
+          return [[]];
+        nullMinterms.push(...n);
+      }
+
+      if (!falseMinterms.length) return nullMinterms;
+      return complement([...complement(nullMinterms), ...falseMinterms]);
+    }
+  }
+
+  export class Case extends Clause {
+    constructor(public clauses: Clause[]) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      const cumulative: Minterm[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        const w = this.clauses[i].true(context);
+        const t = this.clauses[i + 1].true(context);
+        minterms.push(
+          ...complement([...cumulative, ...complement(w), ...complement(t)]),
+        );
+        if (i < this.clauses.length - 2) {
+          cumulative.push(
+            ...complement([
+              ...this.clauses[i].false(context),
+              ...this.clauses[i].null(context),
+            ]),
+          );
+        }
+      }
+      return minterms;
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      const cumulative: Minterm[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        const w = this.clauses[i].true(context);
+        const t = this.clauses[i + 1].false(context);
+        minterms.push(
+          ...complement([...cumulative, ...complement(w), ...complement(t)]),
+        );
+        if (i < this.clauses.length - 2) {
+          cumulative.push(
+            ...complement([
+              ...this.clauses[i].false(context),
+              ...this.clauses[i].null(context),
+            ]),
+          );
+        }
+      }
+      return minterms;
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      const cumulative: Minterm[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        const w = this.clauses[i].true(context);
+        const t = this.clauses[i + 1].null(context);
+        minterms.push(
+          ...complement([...cumulative, ...complement(w), ...complement(t)]),
+        );
+        cumulative.push(
+          ...complement([
+            ...this.clauses[i].false(context),
+            ...this.clauses[i].null(context),
+          ]),
+        );
+      }
+      minterms.push(...complement([...cumulative]));
+      return minterms;
+    }
+    expression(): Expression {
+      if (this._expression != null) return this._expression;
+      if (this.isBoolean()) {
+        this._expression = new Clause.Not(new Clause.Not(this)).expression();
+        return this._expression;
+      }
+      const context = createSynthContext();
+      const cases: { when: Minterm[]; then: Expression }[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        let minterms = this.clauses[i].true(context);
+        const then = this.clauses[i + 1].expression();
+        if (
+          cases.length &&
+          JSON.stringify(then) === JSON.stringify(cases[cases.length - 1].then)
+        )
+          minterms.push(...cases.pop().when);
+        minterms = context.minimize(
+          minterms,
+          cases.flatMap((c) => c.when),
+        );
+        if (!minterms.length) continue;
+        cases.push({ when: minterms, then });
+        if (minterms.length === 1 && !minterms[0].length) break;
+      }
+      while (cases[cases.length - 1].then == null) cases.pop();
+      if (!cases.length) return null;
+      this._expression = [
+        "CASE",
+        ...cases.flatMap((c) => [context.toExpression(c.when), c.then]),
+      ];
+      return this._expression;
+    }
+
+    isBoolean(): boolean {
+      for (let i = 1; i < this.clauses.length; i += 2)
+        if (!this.clauses[i].isBoolean()) return false;
+      return true;
+    }
+  }
+
+  export class IsNull extends Clause {
+    private _boolean: Clause;
+    constructor(public operand: Clause) {
+      super();
+      this._boolean = new Not(new Not(this));
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.null(context);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return complement(this.true(context));
+    }
+    null(): Minterm[] {
+      return [];
+    }
+    isBoolean(): boolean {
+      return true;
+    }
+    *getNullables(): IterableIterator<IsNull> {
+      // Never returns null
+    }
+    expression(): Expression {
+      const nullables = [...this.operand.getNullables()];
+      if (nullables.length === 1 && nullables[0].operand === this.operand)
+        return ["IS NULL", this.operand.expression()];
+      return super.expression();
+    }
+  }
+
+  export class Exp extends Clause {
+    constructor(public exp: Expression) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      if (Array.isArray(this.exp)) return context.getMinterms(this, true);
+      if (this.exp) return [[]];
+      return [];
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      if (Array.isArray(this.exp)) return context.getMinterms(this, false);
+      if (!this.exp && this.exp != null) return [[]];
+      return [];
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      if (Array.isArray(this.exp)) return context.getMinterms(this, null);
+      if (this.exp == null) return [[]];
+      return [];
+    }
+    expression(): Expression {
+      return this.exp;
+    }
+    isBoolean(): boolean {
+      return this.exp === true || this.exp === false || this.exp == null;
+    }
+  }
+
+  export class Compare extends Clause {
+    constructor(
+      public lhs: Clause,
+      public op: ">" | "<" | "=",
+      public rhs: boolean | number | string,
+    ) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, true);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, false);
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      return this.lhs.null(context);
+    }
+    *getNullables(): IterableIterator<IsNull> {
+      yield* this.lhs.getNullables();
+    }
+    isBoolean(): boolean {
+      return true;
+    }
+    expression(): Expression {
+      return [this.op, this.lhs.expression(), this.rhs];
+    }
+  }
+
+  export class Like extends Clause {
+    public readonly caseSensitive: boolean;
+    public readonly contradiction: boolean;
+    public readonly pattern: string[];
+    public readonly lhs: Clause;
+
+    constructor(
+      lhs: Clause,
+      public rhs: string,
+      public esc?: string,
+    ) {
+      super();
+      const exp = lhs.expression();
+      let caseSensitive = true;
+      let contradiction = false;
+      if (Array.isArray(exp)) {
+        if (exp[0] === "FUNC") {
+          const func = exp[1];
+          if (func === "UPPER" || func === "LOWER") {
+            const p = func === "UPPER" ? rhs.toUpperCase() : rhs.toLowerCase();
+            if (p === rhs) caseSensitive = false;
+            else contradiction = true;
+            lhs = new Exp(exp[2]);
+          }
+        }
+      }
+      this.lhs = lhs;
+      this.pattern = parseLikePattern(rhs, esc);
+      this.caseSensitive = caseSensitive;
+      this.contradiction = contradiction;
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, true);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, false);
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      return this.lhs.null(context);
+    }
+    isBoolean(): boolean {
+      return true;
+    }
+    isNullable(c: IsNull): boolean {
+      return this.lhs.isNullable(c);
+    }
+    getNullables(): IterableIterator<IsNull> {
+      return this.lhs.getNullables();
+    }
+    expression(): Expression {
+      let lhs = this.lhs.expression();
+      if (this.contradiction) {
+        if (this.rhs === this.rhs.toLocaleUpperCase())
+          lhs = ["FUNC", "LOWER", lhs];
+        else lhs = ["FUNC", "UPPER", lhs];
+      } else if (!this.caseSensitive) {
+        if (this.rhs === this.rhs.toLocaleUpperCase())
+          lhs = ["FUNC", "UPPER", lhs];
+        else lhs = ["FUNC", "LOWER", lhs];
+      }
+      return ["LIKE", lhs, this.rhs, ...(this.esc ? [this.esc] : [])];
+    }
+  }
+
+  export function fromExpression(exp: Expression): Clause {
+    const res = map(exp, (e) => {
+      if (!Array.isArray(e)) return new Clause.Exp(e);
+      let op = e[0];
+      let negate = true;
+      if (op === "NOT LIKE") op = "LIKE";
+      else if (op === "IS NOT NULL") op = "IS NULL";
+      else if (op === "<>") op = "=";
+      else if (op === ">=") op = "<";
+      else if (op === "<=") op = ">";
+      else negate = false;
+
+      let clause: Clause;
+      if (op === "IS NULL") {
+        clause = new Clause.IsNull(e[1]);
+      } else if (op === "NOT") {
+        clause = new Clause.Not(e[1]);
+      } else if (op === "OR") {
+        clause = new Clause.Or(e.slice(1));
+      } else if (op === "AND") {
+        clause = new Clause.And(e.slice(1));
+      } else if (op === "CASE") {
+        clause = new Clause.Case(e.slice(1));
+      } else if (op === "LIKE") {
+        const rhs = e[2] instanceof Exp ? e[2].expression() : null;
+        const esc = e[3] instanceof Exp ? e[3].expression() : null;
+        if (typeof rhs === "string" && typeof esc === "string")
+          clause = new Clause.Like(e[1], rhs, esc);
+        else if (typeof rhs === "string" && esc == null)
+          clause = new Clause.Like(e[1], rhs);
+      } else if (op === ">" || op === "<" || op === "=") {
+        const rhs = e[2] instanceof Exp ? e[2].expression() : null;
+        if (["boolean", "number", "string"].includes(typeof rhs)) {
+          clause = new Clause.Compare(
+            e[1],
+            op,
+            rhs as boolean | number | string,
+          );
+        }
+      }
+
+      if (!clause) {
+        const args = e.slice(1).map((a) => (a as Clause).expression());
+        clause = new Clause.Exp([op, ...args]);
+      }
+
+      if (negate) clause = new Clause.Not(clause);
+      return clause;
+    });
+
+    return res;
+  }
+}
+
+function groupBy<T, K>(
+  input: T[],
+  callback: (item: T) => K,
+): Iterable<[K, T[]]> {
+  const groups = new Map<K, T[]>();
+  for (const item of input) {
+    const key = callback(item);
+    let arr = groups.get(key);
+    if (!arr) groups.set(key, (arr = []));
+    arr.push(item);
+  }
+
+  return groups.entries();
+}
+
+export class SynthContext extends SynthContextBase<Clause, Clause> {
+  constructor() {
+    super();
+  }
+
+  getMinterms(clause: Clause, res: true | false | null): number[][] {
+    const v = this.getVar(clause);
+    if (res === true) return [[(v << 2) ^ 3]];
+    else if (res === false) return [[(v << 2) ^ 1]];
+    else return [[v << 2, (v << 2) ^ 2]];
+  }
+
+  getDcSet(minterms: Minterm[]): number[][] {
+    const dcSet: number[][] = [];
+
+    const whitelist = new Set([...minterms.flat()].map((v) => v >> 2));
+
+    const allClauses = [...whitelist].map((v) => this.getClause(v));
+    const comparisons = allClauses.filter(
+      (c) => c instanceof Clause.Compare,
+    ) as Clause.Compare[];
+
+    // Comparisons
+    for (const [, clauses] of groupBy(comparisons, (c) =>
+      JSON.stringify(c.lhs),
+    )) {
+      const lhs = clauses[0].lhs;
+      const values = new Set(clauses.map((c) => c.rhs));
+      const valuesSorted = [...values].sort((a, b) => {
+        const ta = typeof a;
+        const tb = typeof b;
+        if (ta === tb) return a > b ? 1 : -1;
+        else if (ta === "string") return 1;
+        else if (tb === "string") return -1;
+        return +a - +b;
+      });
+
+      for (const [i, v] of valuesSorted.entries()) {
+        const eq = this.getVar(new Clause.Compare(lhs, "=", v));
+        const gt = this.getVar(new Clause.Compare(lhs, ">", v));
+        const lt = this.getVar(new Clause.Compare(lhs, "<", v));
+
+        dcSet.push([(eq << 2) ^ 3, (gt << 2) ^ 3]);
+        dcSet.push([(lt << 2) ^ 3, (gt << 2) ^ 3]);
+        dcSet.push([(lt << 2) ^ 3, (eq << 2) ^ 3]);
+        dcSet.push([(lt << 2) ^ 1, (eq << 2) ^ 1, (gt << 2) ^ 1]);
+
+        const negEquivOp = [lt, eq, gt].filter((o) => !whitelist.has(o));
+        if (negEquivOp.length === 1) whitelist.add(negEquivOp[0]);
+
+        for (let j = 0; j < i; j++) {
+          const eq2 = this.getVar(
+            new Clause.Compare(lhs, "=", valuesSorted[j]),
+          );
+          const gt2 = this.getVar(
+            new Clause.Compare(lhs, ">", valuesSorted[j]),
+          );
+          const lt2 = this.getVar(
+            new Clause.Compare(lhs, "<", valuesSorted[j]),
+          );
+
+          // This is the minimum clauses required if all relavent vars
+          // were included in the DC set.
+          // dcSet.push([(eq << 2) ^ 3, (eq2 << 2) ^ 3]);
+          // dcSet.push([(lt << 2) ^ 1, (gt2 << 2) ^ 1]);
+
+          // But we use non-minimal set because intermediate vars
+          // between any two may not be present in the DC set.
+          dcSet.push([(gt2 << 2) ^ 1, (lt << 2) ^ 1]);
+          dcSet.push([(eq2 << 2) ^ 3, (lt << 2) ^ 1]);
+          dcSet.push([(lt2 << 2) ^ 3, (lt << 2) ^ 1]);
+          dcSet.push([(gt2 << 2) ^ 1, (gt << 2) ^ 3]);
+          dcSet.push([(gt2 << 2) ^ 1, (eq << 2) ^ 3]);
+          dcSet.push([(eq2 << 2) ^ 3, (gt << 2) ^ 3]);
+          dcSet.push([(eq2 << 2) ^ 3, (eq << 2) ^ 3]);
+          dcSet.push([(lt2 << 2) ^ 3, (gt << 2) ^ 3]);
+          dcSet.push([(lt2 << 2) ^ 3, (eq << 2) ^ 3]);
+        }
+      }
+    }
+
+    // LIKE
+    const likes = allClauses.filter(
+      (c) => c instanceof Clause.Like,
+    ) as Clause.Like[];
+
+    for (const [, clauses] of groupBy(likes, (c) => JSON.stringify(c.lhs))) {
+      for (let i1 = 0; i1 < clauses.length; ++i1) {
+        const l1 = clauses[i1];
+        if (l1.contradiction) {
+          dcSet.push([(this.getVar(l1) << 2) ^ 3]);
+          continue;
+        }
+        for (let i2 = i1 + 1; i2 < clauses.length; ++i2) {
+          const l2 = clauses[i2];
+          if (l2.contradiction) continue;
+          let p1 = l1.pattern;
+          let p2 = l2.pattern;
+          if (!l1.caseSensitive || !l2.caseSensitive) {
+            p1 = p1.map((c) => c.toLowerCase());
+            p2 = p2.map((c) => c.toLowerCase());
+          }
+          if (likeDisjoint(p1, p2)) {
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 3,
+              (this.getVar(l2) << 2) ^ 3,
+            ]);
+          } else if (
+            (!l1.caseSensitive || l2.caseSensitive) &&
+            likeImplies(p1, p2)
+          ) {
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 2,
+              (this.getVar(l2) << 2) ^ 3,
+            ]);
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 1,
+              (this.getVar(l2) << 2) ^ 0,
+            ]);
+          } else if (
+            (!l2.caseSensitive || l1.caseSensitive) &&
+            likeImplies(p2, p1)
+          ) {
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 3,
+              (this.getVar(l2) << 2) ^ 2,
+            ]);
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 0,
+              (this.getVar(l2) << 2) ^ 1,
+            ]);
+          }
+        }
+      }
+    }
+
+    for (const v of whitelist) {
+      const clause = this.getClause(v);
+      const nullables = [...clause.getNullables()].map((c) => this.getVar(c));
+      if (nullables.length) {
+        dcSet.push([
+          ...nullables.map((n) => (n << 2) ^ 2),
+          (v << 2) ^ 0,
+          (v << 2) ^ 2,
+        ]);
+        for (const n of nullables) {
+          dcSet.push([(n << 2) ^ 3, (v << 2) ^ 1]);
+          dcSet.push([(n << 2) ^ 3, (v << 2) ^ 3]);
+          whitelist.add(n);
+        }
+      }
+
+      if (clause instanceof Clause.IsNull) {
+        const str = clause.operand.toString();
+        if (str === '["PARAM","DeviceID.ID"]' || str === '["PARAM","_id"]')
+          dcSet.push([(v << 2) ^ 3]);
+      }
+    }
+    return dcSet.filter((m) => m.every((v) => whitelist.has(v >> 2)));
+  }
+
+  canRaise(idx: number, set: Set<number>): boolean {
+    const clause = this.getClause(idx >> 2);
+    if (clause instanceof Clause.IsNull) {
+      for (const i of set) {
+        if (i === idx || i & 1) continue;
+        const c = this.getClause(i >> 2);
+        if (c.isNullable(clause)) return false;
+      }
+      return true;
+    }
+    return !(idx & 1) || !set.has(idx ^ 3);
+  }
+
+  canLower(idx: number, set: Set<number>): boolean {
+    if (idx & 1) return true;
+    const clause = this.getClause(idx >> 2);
+    if (clause instanceof Clause.IsNull) return true;
+    return set.has(idx ^ 3) || set.has(idx ^ 1);
+  }
+
+  bias(a: number, b: number): number {
+    // Bias towards 1 then true
+    return ((b & 3) ^ 3) - ((a & 3) ^ 3);
+  }
+
+  sanitizeMinterms(minterms: Minterm[]): Minterm[] {
+    const res = [] as number[][];
+
+    loop: for (const m of minterms) {
+      const merged: Map<number, number> = new Map();
+      for (const i of m)
+        merged.set(i >> 2, (merged.get(i >> 2) || 0) | (1 << (i & 3)));
+      const minterm: number[] = [];
+      const perms: number[][] = [];
+      for (const [k, v] of merged) {
+        if ((v & 0b0011) === 0b0011) continue loop;
+        if ((v & 0b1100) === 0b1100) continue loop;
+        const clause = this.clauses.get(k);
+        if (!clause) throw new Error("Invalid literal");
+        if (clause instanceof Clause.IsNull) {
+          if (v === 0b0100) minterm.push((k << 2) ^ 2);
+          else if (v === 0b1000) minterm.push((k << 2) ^ 3);
+          else throw new Error("Invalid literal");
+          continue;
+        }
+        if ((v & 0b1010) === 0b1010) continue loop;
+        const isNullVars = [...clause.getNullables()].map((c) =>
+          this.getVar(c),
+        );
+        const t = k << 2;
+        if (v === 0b0101) {
+          if (isNullVars.length === 1) minterm.push((isNullVars[0] << 2) ^ 3);
+          else perms.push(isNullVars.map((n) => (n << 2) ^ 3));
+        } else if (v === 0b0001) {
+          perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 3]);
+        } else if (v === 0b0100) {
+          perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 1]);
+        } else if (v & 0b1000) {
+          minterm.push(t ^ 3);
+        } else if (v & 0b0010) {
+          minterm.push(t ^ 1);
+        }
+      }
+      let ms = [minterm];
+      while (perms.length) {
+        const newMs: number[][] = [];
+        const perm = perms.pop();
+        for (const p of perm) newMs.push(...ms.map((mm) => [...mm, p]));
+        ms = newMs;
+      }
+      res.push(...ms);
+    }
+    return res;
+  }
+
+  toExpression(sop: number[][]): Expression {
+    if (!sop.length) return false;
+    const res: Expression[] = [];
+    for (const s of sop) {
+      if (!s.length) return true;
+      const conjs: Expression[] = [];
+      for (const i of s) {
+        const clause = this.getClause(i >>> 2);
+        if (!clause) throw new Error("Invalid literal");
+        if (clause instanceof Clause.IsNull) {
+          if (!(i & 2)) throw new Error("Invalid literal");
+        } else if (!(i & 1)) {
+          // Should never be reached if minimized correctly
+          const isNullVars = [...clause.getNullables()].map((c) =>
+            this.getVar(c),
+          );
+          conjs.push(
+            this.toExpression([
+              [i ^ 3],
+              ...isNullVars.map((n) => [(n << 2) ^ 3]),
+            ]),
+          );
+          continue;
+        }
+
+        let expr = clause.expression() as Expression;
+        if (!(i & 1) !== !(i & 2)) expr = ["NOT", expr];
+        if (
+          Array.isArray(expr) &&
+          expr[0] === "NOT" &&
+          Array.isArray(expr[1])
+        ) {
+          const e: Expression[] = expr[1];
+          if (e[0] === "IS NULL") expr = ["IS NOT NULL", ...e.slice(1)];
+          else if (e[0] === "LIKE") expr = ["NOT LIKE", ...e.slice(1)];
+          else if (e[0] === "=") expr = ["<>", ...e.slice(1)];
+          else if (e[0] === "<>") expr = ["=", ...e.slice(1)];
+          else if (e[0] === ">") expr = ["<=", ...e.slice(1)];
+          else if (e[0] === ">=") expr = ["<", ...e.slice(1)];
+          else if (e[0] === "<") expr = [">=", ...e.slice(1)];
+          else if (e[0] === "<=") expr = [">", ...e.slice(1)];
+          else if (e[0] === "NOT") expr = e[1];
+        }
+        conjs.push(expr);
+      }
+      if (conjs.length > 1) res.push(["AND", ...conjs]);
+      else res.push(conjs[0]);
+    }
+    if (res.length > 1) return ["OR", ...res];
+    return res[0];
+  }
+}
+
+// Classes aren't hoisted in JS but functions are. This function is used to
+// create a new SynthContext instance from inside the Clause class.
+export function createSynthContext(): SynthContext {
+  return new SynthContext();
+}
+
+export function likeImplies(pat1: string[], pat2: string[]): boolean {
+  let backtrack: [number, number] = null;
+
+  for (let i1 = 0, i2 = 0; ; ++i1, ++i2) {
+    while (i1 < pat1.length && pat1[i1] === "\\%") backtrack = [i1++, i2];
+
+    if (i2 >= pat2.length) return i1 >= pat1.length;
+
+    const c = i1 < pat1.length ? pat1[i1] : null;
+
+    if (c !== pat2[i2] && c !== "\\_") {
+      if (!backtrack) return false;
+      [i1, i2] = backtrack;
+      ++backtrack[1];
+    }
+  }
+}
+
+export function likeDisjoint(pat1: string[], pat2: string[]): boolean {
+  const left1Idx = pat1.indexOf("\\%");
+  const left2Idx = pat2.indexOf("\\%");
+  const right1Idx = pat1.lastIndexOf("\\%");
+  const right2Idx = pat2.lastIndexOf("\\%");
+
+  const left1 = pat1.slice(0, left1Idx !== -1 ? left1Idx : pat1.length);
+  const left2 = pat2.slice(0, left2Idx !== -1 ? left2Idx : pat2.length);
+  const right1 = pat1.slice(right1Idx !== -1 ? right1Idx + 1 : 0).reverse();
+  const right2 = pat2.slice(right2Idx !== -1 ? right2Idx + 1 : 0).reverse();
+
+  for (let i = 0; i < Math.min(left1.length, left2.length); ++i) {
+    if (left1[i] !== left2[i] && left1[i] !== "\\_" && left2[i] !== "\\_")
+      return true;
+  }
+
+  for (let i = 0; i < Math.min(right1.length, right2.length); ++i) {
+    if (right1[i] !== right2[i] && right1[i] !== "\\_" && right2[i] !== "\\_")
+      return true;
+  }
+
+  if (pat1.length === left1.length)
+    return pat2.filter((c) => c !== "\\%").length > pat1.length;
+  else if (pat2.length === left2.length)
+    return pat1.filter((c) => c !== "\\%").length > pat2.length;
+
+  return false;
+}
+
+export function minimize(expr: Expression, boolean = false): Expression {
+  let synth = Clause.fromExpression(normalize(expr));
+  if (boolean) synth = new Clause.Not(new Clause.Not(synth));
+  return synth.expression();
+}
+
+export function unionDiff(
+  expr1: Expression,
+  expr2: Expression,
+): [Expression, Expression] {
+  expr2 = normalize(expr2);
+
+  if (!expr2) return [expr1, false];
+
+  const synth2 = Clause.fromExpression(expr2);
+
+  if (!expr1) {
+    const e = synth2.expression();
+    return [e, e];
+  }
+
+  expr1 = normalize(expr1);
+  const synth1 = Clause.fromExpression(expr1);
+
+  const context = new SynthContext();
+
+  const expr2Minterms = synth2.true(context);
+  const expr1Minterms = synth1.true(context);
+
+  const union = context.minimize([...expr1Minterms, ...expr2Minterms]);
+
+  const diff = context.minimize(
+    complement([...expr1Minterms, ...complement(expr2Minterms)]),
+  );
+
+  return [context.toExpression(union), context.toExpression(diff)];
+}
+
+export function covers(expr1: Expression, expr2: Expression): boolean {
+  expr2 = normalize(expr2);
+  if (!expr2) return true;
+  expr1 = normalize(expr1);
+  if (!Array.isArray(expr1) && expr1) return true;
+
+  const synt1 = Clause.fromExpression(expr1);
+  const synt2 = Clause.fromExpression(expr2);
+
+  const context = new SynthContext();
+  const expr1Minterms = synt1.true(context);
+  const expr2Minterms = synt2.true(context);
+
+  return tautology([
+    ...context.sanitizeMinterms(complement(expr2Minterms)),
+    ...context.getDcSet([...expr2Minterms, ...expr1Minterms]),
+    ...context.sanitizeMinterms(expr1Minterms),
+  ]);
+}

lib/common/expression/util.ts

@@ -1,24 +1,5 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { map, mapAsync, parseLikePattern } from "./expression-parser";
-import { Expression } from "../types";
+import { map, mapAsync, parseLikePattern } from "./parser.ts";
+import { Expression } from "../../types.ts";
 
 const isArray = Array.isArray;
 
@@ -74,7 +55,7 @@ export function likePatternToRegExp(pat: string, esc = "", flags = ""): RegExp {
 
 function compare(
   a: boolean | number | string,
-  b: boolean | number | string
+  b: boolean | number | string,
 ): number {
   if (typeof a === "boolean") a = +a;
   if (typeof b === "boolean") b = +b;
@@ -244,9 +225,12 @@ export function evaluateCallback(exp: Expression): Expression {
   } else if (exp[0] === "/") {
     return reduce(exp, (a, b, i) => {
       if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b))
-        return i === 0 ? toNumber(a) / toNumber(b) : toNumber(a) * toNumber(b);
-      return REDUCE_SKIP;
+      if (isArray(a) || isArray(b)) return REDUCE_SKIP;
+      const n1 = toNumber(a);
+      const n2 = toNumber(b);
+      if (i !== 0) return n1 * n2;
+      if (n2 === 0) return null;
+      return n1 / n2;
     });
   } else if (exp[0] === "+") {
     return reduce(exp, (a, b) => {
@@ -261,6 +245,15 @@ export function evaluateCallback(exp: Expression): Expression {
         return i === 0 ? toNumber(a) - toNumber(b) : toNumber(a) + toNumber(b);
       return REDUCE_SKIP;
     });
+  } else if (exp[0] === "%") {
+    return reduce(exp, (a, b, i) => {
+      if (a == null || b == null) return null;
+      if (isArray(a) || isArray(b) || i !== 0) return REDUCE_SKIP;
+      const n1 = toNumber(a);
+      const n2 = Math.trunc(toNumber(b));
+      if (n2 === 0) return null;
+      return n1 % n2;
+    });
   } else if (exp[0] === "||") {
     return reduce(exp, (a, b) => {
       if (a == null || b == null) return null;
@@ -275,19 +268,19 @@ export function evaluate(
   exp: Expression,
   obj: Record<string, unknown> | ((e: string) => Expression),
   now: number,
-  cb?: (e: Expression) => Expression
+  cb?: (e: Expression) => Expression,
 ): string | number | boolean | null;
 export function evaluate(
   exp: Expression,
   obj?: Record<string, unknown> | ((e: string) => Expression),
   now?: number,
-  cb?: (e: Expression) => Expression
+  cb?: (e: Expression) => Expression,
 ): Expression;
 export function evaluate(
   exp: Expression,
   obj?: Record<string, unknown> | ((e: string) => Expression),
   now?: number,
-  cb?: (e: Expression) => Expression
+  cb?: (e: Expression) => Expression,
 ): Expression {
   return map(exp, (e) => {
     if (cb) e = cb(e);
@@ -315,19 +308,19 @@ export async function evaluateAsync(
   exp: Expression,
   obj: Record<string, unknown>,
   now: number,
-  cb?: (e: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>,
 ): Promise<string | number | boolean | null>;
 export async function evaluateAsync(
   exp: Expression,
   obj?: Record<string, unknown>,
   now?: number,
-  cb?: (e: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>,
 ): Promise<Expression>;
 export async function evaluateAsync(
   exp: Expression,
   obj?: Record<string, unknown>,
   now?: number,
-  cb?: (e: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>,
 ): Promise<Expression> {
   return mapAsync(exp, async (e) => {
     if (cb) e = await cb(e);

lib/common/memoize.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 let cache1 = new Map();
 let cache2 = new Map();
 const keys = new WeakMap();

lib/common/path-set.ts

@@ -1,28 +1,9 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./path";
+import Path from "./path.ts";
 
 export default class PathSet {
-  private declare lengthIndex: Set<Path>[];
-  private declare fragmentIndex: Map<string, Set<Path>>[];
-  private declare stringIndex: Map<string, Path>;
+  declare private lengthIndex: Set<Path>[];
+  declare private fragmentIndex: Map<string, Set<Path>>[];
+  declare private stringIndex: Map<string, Path>;
 
   public constructor() {
     this.lengthIndex = [];
@@ -75,7 +56,7 @@ export default class PathSet {
     path: Path,
     superset = false,
     subset = false,
-    depth: number = path.length
+    depth: number = path.length,
   ): Path[] {
     if (path.alias) throw new Error("PathSet does not support aliased paths");
 
@@ -115,7 +96,7 @@ export default class PathSet {
           res = [...fragmentIndexSet1, ...fragmentIndexSet2];
         } else {
           res = res.filter(
-            (r) => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
+            (r) => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r),
           );
         }
       }

lib/common/path.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 type Segments = (string | Alias)[];
 type Alias = [Path, string][];
 
@@ -49,15 +30,15 @@ function legalChar(c: number): boolean {
 }
 
 export default class Path {
-  public declare readonly segments: Segments;
-  public declare readonly wildcard: number;
-  public declare readonly alias: number;
-  protected declare _string: string;
-  protected declare _stringIndex: number[];
+  declare public readonly segments: Segments;
+  declare public readonly wildcard: number;
+  declare public readonly alias: number;
+  declare protected _string: string;
+  declare protected _stringIndex: number[];
 
   protected static parseAliasValue(
     pattern: string,
-    index: number
+    index: number,
   ): { index: number; value: string } {
     let i = index;
     while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
@@ -89,7 +70,7 @@ export default class Path {
 
   protected static parseAliasPath(
     pattern: string,
-    index: number
+    index: number,
   ): { index: number; path: Path } {
     let i = index;
     while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
@@ -103,7 +84,7 @@ export default class Path {
 
   protected static parseAlias(
     pattern: string,
-    index: number
+    index: number,
   ): { index: number; alias: Alias } {
     const alias: Alias = [];
     let i = index;
@@ -136,7 +117,7 @@ export default class Path {
 
   protected static parsePath(
     pattern: string,
-    index: number
+    index: number,
   ): { index: number; segments: Segments } {
     const segments = [];
 
@@ -173,7 +154,7 @@ export default class Path {
       if (Array.isArray(s)) {
         alias |= 1 << i;
         const parts = s.map(
-          (al) => `${al[0].toString()}:${JSON.stringify(al[1])}`
+          (al) => `${al[0].toString()}:${JSON.stringify(al[1])}`,
         );
         return `[${parts.join(",")}]`;
       } else if (s === "*") {

lib/common/yaml.ts

@@ -15,7 +15,7 @@ const STRING_RESERVED = new Set([
 
 function isPrintable(str: string): boolean {
   return !/[^\t\n\x20-\x7e\x85\u{a0}-\u{d7ff}\u{e000}-\u{fffd}\u{10000}-\u{10ffff}]/u.test(
-    str
+    str,
   );
 }
 
@@ -99,7 +99,7 @@ function stringifyString(str: string, res: string[], prefix1, prefix2): void {
     if (!isFolded) {
       res.push(
         `${prefix1}|${idt}${chmp}`,
-        ...lines.map((l) => (l ? prefix2 + l : l))
+        ...lines.map((l) => (l ? prefix2 + l : l)),
       );
       return;
     }
@@ -129,7 +129,7 @@ function stringifyAny(
   obj: unknown,
   res: string[],
   prefix1 = "",
-  prefix2 = ""
+  prefix2 = "",
 ): void {
   if (obj == null) {
     res.push(`${prefix1}null`);
@@ -183,7 +183,7 @@ function stringifyAny(
       entries[0][1],
       res,
       prefix1 + `${stringifyKey(entries[0][0])}: `,
-      prefix2 + INDENTATION
+      prefix2 + INDENTATION,
     );
     prefix1 = prefix2;
     prefix2 = prefix2 + INDENTATION;
@@ -192,7 +192,7 @@ function stringifyAny(
         entries[i][1],
         res,
         prefix1 + `${stringifyKey(entries[i][0])}: `,
-        prefix2
+        prefix2,
       );
     }
   } else {
@@ -204,7 +204,7 @@ function stringifyAny(
         entries[i][1],
         res,
         prefix1 + `${stringifyKey(entries[i][0])}: `,
-        prefix2
+        prefix2,
       );
     }
   }

lib/config.ts

@@ -1,24 +1,5 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { resolve } from "path";
-import { readFileSync, existsSync } from "fs";
+import { resolve } from "node:path";
+import { readFileSync, existsSync } from "node:fs";
 
 // Find project root directory
 export let ROOT_DIR = resolve(__dirname, "..");
@@ -101,6 +82,9 @@ const options = {
 
   // Should probably never be changed
   DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
+
+  XMPP_JID: { type: "string", default: "" },
+  XMPP_PASSWORD: { type: "string", default: "" },
 };
 
 const allConfig: { [name: string]: string | number } = {};
@@ -245,7 +229,7 @@ if (fsHostname) {
   const FS_SSL = !!allConfig["FS_SSL_CERT"];
   setConfig(
     "FS_URL_PREFIX",
-    (FS_SSL ? "https" : "http") + `://${fsHostname}:${FS_PORT}/`
+    (FS_SSL ? "https" : "http") + `://${fsHostname}:${FS_PORT}/`,
   );
 }
 
@@ -255,7 +239,7 @@ for (const [k, v] of Object.entries(options))
 
 export function get(
   optionName: string,
-  deviceId?: string
+  deviceId?: string,
 ): string | number | boolean {
   if (!deviceId) return allConfig[optionName];
 

lib/connection-request.ts

@@ -1,35 +1,19 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as crypto from "crypto";
-import * as dgram from "dgram";
-import { URL } from "url";
-import * as http from "http";
-import { evaluateAsync } from "./common/expression";
-import { Expression } from "./types";
-import * as auth from "./auth";
-import * as extensions from "./extensions";
-import * as debug from "./debug";
+import * as crypto from "node:crypto";
+import * as dgram from "node:dgram";
+import * as http from "node:http";
+import { evaluateAsync } from "./common/expression/util.ts";
+import { Expression } from "./types.ts";
+import * as auth from "./auth.ts";
+import * as extensions from "./extensions.ts";
+import * as debug from "./debug.ts";
+import XmppClient from "./xmpp-client.ts";
+import * as config from "../lib/config.ts";
+import { encodeEntities, parseAttrs, Element } from "./xml-parser.ts";
+import * as logger from "../lib/logger.ts";
 
 async function extractAuth(
   exp: Expression,
-  dflt: any
+  dflt: any,
 ): Promise<[string, string, Expression]> {
   let username, password;
   const _exp = await evaluateAsync(
@@ -55,7 +39,7 @@ async function extractAuth(
         }
       }
       return e;
-    }
+    },
   );
   return [username, password, _exp];
 }
@@ -64,7 +48,7 @@ function httpGet(
   url: URL,
   options: http.RequestOptions,
   _debug: boolean,
-  deviceId: string
+  deviceId: string,
 ): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
   return new Promise((resolve, reject) => {
     const req = http
@@ -94,7 +78,7 @@ export async function httpConnectionRequest(
   allowBasicAuth: boolean,
   timeout: number,
   _debug: boolean,
-  deviceId: string
+  deviceId: string,
 ): Promise<string> {
   const url = new URL(address);
   if (url.protocol !== "http:")
@@ -120,7 +104,7 @@ export async function httpConnectionRequest(
               Authorization: auth.basic(username || "", password || ""),
             },
           },
-          options
+          options,
         );
       } else if (authHeader["method"] === "Digest") {
         opts = Object.assign(
@@ -132,11 +116,11 @@ export async function httpConnectionRequest(
                 url.pathname + url.search,
                 "GET",
                 null,
-                authHeader
+                authHeader,
               ),
             },
           },
-          options
+          options,
         );
       } else {
         return "Unrecognized auth method";
@@ -171,7 +155,7 @@ export async function httpConnectionRequest(
     if (res.statusCode === 401 && res.headers["www-authenticate"]) {
       try {
         authHeader = auth.parseWwwAuthenticateHeader(
-          res.headers["www-authenticate"]
+          res.headers["www-authenticate"],
         );
       } catch (err) {
         return "Connection request error: Error parsing www-authenticate header";
@@ -191,7 +175,7 @@ export async function udpConnectionRequest(
   authExp: Expression,
   sourcePort = 0,
   _debug: boolean,
-  deviceId: string
+  deviceId: string,
 ): Promise<void> {
   const now = Date.now();
 
@@ -237,3 +221,84 @@ export async function udpConnectionRequest(
   }
   client.close();
 }
+
+const XMPP_JID = config.get("XMPP_JID") as string;
+const XMPP_PASSWORD = config.get("XMPP_PASSWORD") as string;
+const XMPP_RESOURCE = crypto.randomBytes(8).toString("hex");
+
+let xmppClient: XmppClient;
+
+function xmppClientOnError(err: Error): void {
+  xmppClient = null;
+  logger.error({
+    message: "XMPP exception",
+    exception: err,
+    pid: process.pid,
+  });
+}
+
+function xmppClientOnClose(): void {
+  xmppClient = null;
+}
+
+export async function xmppConnectionRequest(
+  jid: string,
+  authExp: Expression,
+  timeout: number,
+  _debug: boolean,
+  deviceId: string,
+): Promise<string> {
+  if (!xmppClient) {
+    const [host, username] = XMPP_JID.split("@").reverse();
+    xmppClient = await XmppClient.connect({
+      host,
+      username,
+      resource: XMPP_RESOURCE,
+      password: XMPP_PASSWORD,
+      timeout: 120000,
+    });
+    xmppClient.on("error", xmppClientOnError);
+    xmppClient.on("close", xmppClientOnClose);
+    xmppClient.unref();
+  }
+
+  let username: string;
+  let password: string;
+
+  [username, password, authExp] = await extractAuth(authExp, null);
+  while (username != null && password != null) {
+    const msg = `<connectionRequest xmlns="urn:broadband-forum-org:cwmp:xmppConnReq-1-0"><username>${encodeEntities(
+      username,
+    )}</username><password>${encodeEntities(
+      password,
+    )}</password></connectionRequest>`;
+    let res: Element, rawRes: string, rawReq: string;
+    try {
+      ({ res, rawRes, rawReq } = await xmppClient.sendIqStanza(
+        XMPP_JID,
+        jid,
+        "get",
+        msg,
+        timeout,
+      ));
+    } catch (err) {
+      return err.message;
+    }
+    if (_debug) {
+      debug.outgoingXmppStanza(deviceId, rawReq);
+      debug.incomingXmppStanza(deviceId, rawRes);
+    }
+    const attrs = parseAttrs(res.attrs);
+    const type = attrs.find((a) => a.name === "type");
+    if (type && type.value === "result") return "";
+    const error = res.children.find((c) => c.name === "error");
+    if (!error || !error.children[0])
+      return "Unexpected XMPP connection request response";
+    if (error.children[0].name === "service-unavailable")
+      return "Device is offline";
+    if (error.children[0].name !== "not-authorized")
+      return "Unexpected XMPP connection request response";
+    [username, password, authExp] = await extractAuth(authExp, null);
+  }
+  return "Incorrect connection request credentials";
+}

lib/cwmp/db.ts

@@ -1,50 +1,16 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { MongoClient, ObjectId, Collection, Db } from "mongodb";
-import { get } from "./config";
-import { decodeTag, encodeTag, escapeRegExp } from "./common";
-import { parse } from "./common/expression-parser";
+import { ObjectId } from "mongodb";
+import { decodeTag, encodeTag, escapeRegExp } from "../util.ts";
 import {
   DeviceData,
   Attributes,
   SessionFault,
   Task,
   Operation,
-  Expression,
-} from "./types";
-import Path from "./common/path";
-
-export let tasksCollection: Collection,
-  devicesCollection: Collection,
-  presetsCollection: Collection,
-  objectsCollection: Collection,
-  provisionsCollection: Collection,
-  virtualParametersCollection: Collection,
-  faultsCollection: Collection,
-  filesCollection: Collection,
-  operationsCollection: Collection,
-  permissionsCollection: Collection,
-  usersCollection: Collection,
-  configCollection: Collection;
-
-let clientPromise: Promise<MongoClient>;
+} from "../types.ts";
+import Path from "../common/path.ts";
+import { collections } from "../db/db.ts";
+import { optimizeProjection } from "../db/util.ts";
+import * as MongoTypes from "../db/types.ts";
 
 const INVALID_PATH_SUFFIX = "__invalid";
 
@@ -54,65 +20,9 @@ function compareAccessLists(list1: string[], list2: string[]): boolean {
   return true;
 }
 
-const onConnectCallbacks: ((db: Db) => Promise<void>)[] = [];
-
-export function onConnect(callback: (db: Db) => Promise<void>): void {
-  onConnectCallbacks.push(callback);
-}
-
-export async function connect(): Promise<void> {
-  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"));
-
-  const client = await clientPromise;
-  const db = client.db();
-  await Promise.all(onConnectCallbacks.map((c) => c(db)));
-}
-
-onConnect(async (db) => {
-  tasksCollection = db.collection("tasks");
-  await tasksCollection.createIndex({ device: 1, timestamp: 1 });
-
-  devicesCollection = db.collection("devices");
-  presetsCollection = db.collection("presets");
-  objectsCollection = db.collection("objects");
-  filesCollection = db.collection("fs.files");
-  provisionsCollection = db.collection("provisions");
-  virtualParametersCollection = db.collection("virtualParameters");
-  faultsCollection = db.collection("faults");
-  operationsCollection = db.collection("operations");
-  permissionsCollection = db.collection("permissions");
-  usersCollection = db.collection("users");
-  configCollection = db.collection("config");
-});
-
-export async function disconnect(): Promise<void> {
-  if (clientPromise) await (await clientPromise).close();
-}
-
-// Optimize projection by removing overlaps
-// This can modify the object
-function optimizeProjection(obj: { [path: string]: 1 }): { [path: string]: 1 } {
-  if (obj[""]) return { "": obj[""] };
-
-  const keys = Object.keys(obj).sort();
-  if (keys.length <= 1) return obj;
-
-  for (let i = 1; i < keys.length; ++i) {
-    const a = keys[i - 1];
-    const b = keys[i];
-    if (b.startsWith(a)) {
-      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
-        delete obj[b];
-        keys.splice(i--, 1);
-      }
-    }
-  }
-  return obj;
-}
-
 export async function fetchDevice(
   id: string,
-  timestamp: number
+  timestamp: number,
 ): Promise<[Path, number, Attributes?][]> {
   const res: [Path, number, Attributes?][] = [
     [
@@ -127,14 +37,14 @@ export async function fetchDevice(
     ],
   ];
 
-  const device = await devicesCollection.findOne({ _id: id });
+  const device = await collections.devices.findOne({ _id: id });
   if (!device) return null;
 
   function storeParams(
     obj,
     path: string,
     pathLength: number,
-    ts: number
+    ts: number,
   ): void {
     if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
     if (obj["_attributesTimestamp"])
@@ -146,7 +56,7 @@ export async function fetchDevice(
 
     if (obj["_value"] != null) {
       attrs.value = [obj["_timestamp"] || 1, [obj["_value"], obj["_type"]]];
-      if (obj["_type"] === "xsd:dateTime")
+      if (obj["_type"] === "xsd:dateTime" && obj["_value"] instanceof Date)
         attrs.value[1][0] = +attrs.value[1][0];
 
       obj["_object"] = false;
@@ -191,7 +101,7 @@ export async function fetchDevice(
       res.push([Path.parse(path + ".*"), obj["_timestamp"]]);
   }
 
-  const ts: number = device["_timestamp"] || 0;
+  const ts: number = +device["_timestamp"] || 0;
   if (ts) res.push([Path.parse("*"), ts]);
 
   for (const [k, v] of Object.entries(device)) {
@@ -333,7 +243,7 @@ export async function saveDevice(
   deviceId: string,
   deviceData: DeviceData,
   isNew: boolean,
-  sessionTimestamp: number
+  sessionTimestamp: number,
 ): Promise<void> {
   const update = { $set: {}, $unset: {}, $addToSet: {}, $pull: {} };
 
@@ -446,14 +356,18 @@ export async function saveDevice(
           if (value2 != null) {
             if (!update["$addToSet"]["_tags"])
               update["$addToSet"]["_tags"] = { $each: [] };
-            update["$addToSet"]["_tags"]["$each"].push(path.segments[1]);
+            update["$addToSet"]["_tags"]["$each"].push(
+              decodeTag(path.segments[1] as string),
+            );
           } else {
             if (!update["$pull"]["_tags"]) {
               update["$pull"]["_tags"] = {
                 $in: [],
               };
             }
-            update["$pull"]["_tags"]["$in"].push(path.segments[1]);
+            update["$pull"]["_tags"]["$in"].push(
+              decodeTag(path.segments[1] as string),
+            );
           }
         }
 
@@ -465,7 +379,10 @@ export async function saveDevice(
           if (pathStr.endsWith(INVALID_PATH_SUFFIX)) {
             const splits = pathStr.split(".");
             splits[splits.length - 1] = decodeTag(
-              splits[splits.length - 1].slice(0, 0 - INVALID_PATH_SUFFIX.length)
+              splits[splits.length - 1].slice(
+                0,
+                0 - INVALID_PATH_SUFFIX.length,
+              ),
             );
             pathStr = splits.join(".");
           }
@@ -487,7 +404,7 @@ export async function saveDevice(
                     Number.isInteger(value2 as number)
                   ) {
                     update["$set"][path.toString() + "._value"] = new Date(
-                      value2 as number
+                      value2 as number,
                     );
                   } else {
                     update["$set"][path.toString() + "._value"] = value2;
@@ -499,7 +416,7 @@ export async function saveDevice(
 
                 if (valueTimestamp2 !== valueTimestamp1) {
                   update["$set"][path.toString() + "._timestamp"] = new Date(
-                    valueTimestamp2
+                    valueTimestamp2,
                   );
                 }
 
@@ -614,35 +531,43 @@ export async function saveDevice(
     delete update["$pull"];
   }
 
-  const result = await devicesCollection.updateOne({ _id: deviceId }, update, {
-    upsert: isNew,
-  });
+  const result = await collections.devices.updateOne(
+    { _id: deviceId },
+    update,
+    {
+      upsert: isNew,
+    },
+  );
 
   if (!result.matchedCount && !result.upsertedCount)
     throw new Error(`Device ${deviceId} not found in database`);
 
   if (update2) {
-    await devicesCollection.updateOne({ _id: deviceId }, update2);
+    await collections.devices.updateOne({ _id: deviceId }, update2);
     return;
   }
 }
 
 export async function getFaults(
-  deviceId: string
+  deviceId: string,
 ): Promise<{ [channel: string]: SessionFault }> {
-  const res = await faultsCollection
+  const res = await collections.faults
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
     .toArray();
 
   const faults: { [channel: string]: SessionFault } = {};
   for (const r of res) {
     const channel = r._id.slice(deviceId.length + 1);
-    delete r._id;
-    delete r.channel;
-    delete r.device;
-    r.timestamp = +r.timestamp;
-    r.provisions = JSON.parse(r.provisions);
-    faults[channel] = r as SessionFault;
+    const fault: SessionFault = {
+      code: r.code,
+      message: r.message,
+      ...(r.detail && { detail: r.detail }),
+      timestamp: +r.timestamp,
+      provisions: JSON.parse(r.provisions),
+      retries: r.retries,
+      ...(r.expiry && { expiry: +r.expiry }),
+    };
+    faults[channel] = fault;
   }
 
   return faults;
@@ -651,48 +576,83 @@ export async function getFaults(
 export async function saveFault(
   deviceId: string,
   channel: string,
-  fault: SessionFault
+  fault: SessionFault,
 ): Promise<void> {
   const id = `${deviceId}:${channel}`;
-  const f = Object.assign({}, fault) as Record<string, any>;
-  f["_id"] = id;
-  f["device"] = deviceId;
-  f["channel"] = channel;
-  f["timestamp"] = new Date(fault.timestamp);
-  f["provisions"] = JSON.stringify(fault.provisions);
-  await faultsCollection.replaceOne({ _id: id }, f, { upsert: true });
+  const f: MongoTypes.Fault = {
+    _id: id,
+    device: deviceId,
+    channel: channel,
+    timestamp: new Date(fault.timestamp),
+    code: fault.code,
+    message: fault.message,
+    ...(fault.detail && { detail: fault.detail }),
+    retries: fault.retries,
+    ...(fault.expiry && { expiry: new Date(fault.expiry) }),
+    provisions: JSON.stringify(fault.provisions),
+  };
+  await collections.faults.replaceOne({ _id: id }, f, { upsert: true });
 }
 
 export async function deleteFault(
   deviceId: string,
-  channel: string
+  channel: string,
 ): Promise<void> {
-  await faultsCollection.deleteOne({ _id: `${deviceId}:${channel}` });
+  await collections.faults.deleteOne({ _id: `${deviceId}:${channel}` });
 }
 
 export async function getDueTasks(
   deviceId: string,
-  timestamp: number
+  timestamp: number,
 ): Promise<[Task[], number]> {
-  const cur = tasksCollection.find({ device: deviceId }).sort({ timestamp: 1 });
+  const cur = collections.tasks
+    .find({ device: deviceId })
+    .sort({ timestamp: 1 });
   const tasks = [] as Task[];
 
-  for await (const task of cur) {
-    if (task.timestamp) task.timestamp = +task.timestamp;
-    if (task.expiry) task.expiry = +task.expiry;
-    if (task.timestamp >= timestamp) return [tasks, +task.timestamp];
-    task._id = String(task._id);
+  for await (const t of cur) {
+    if (+t.timestamp >= timestamp) return [tasks, +t.timestamp];
+    const task: Task = {
+      _id: t._id.toString(),
+      name: t.name,
+      ...(t.timestamp && { timestamp: +t.timestamp }),
+      ...(t.expiry && { expiry: +t.expiry }),
+      ...(t.name === "getParameterValues" && {
+        parameterNames: t.parameterNames,
+      }),
+      ...(t.name === "setParameterValues" && {
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "refreshObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "download" && {
+        fileType: t.fileType,
+        fileName: t.fileName,
+        targetFileName: t.targetFileName,
+      }),
+      ...(t.name === "addObject" && {
+        objectName: t.objectName,
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "deleteObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "provisions" && {
+        provisions: t.provisions,
+      }),
+    };
 
-    tasks.push(task as Task);
+    tasks.push(task);
 
     // For API compatibility
-    if (task.name === "download" && task.file) {
+    if (task.name === "download" && t["file"]) {
       let q;
-      if (ObjectId.isValid(task.file))
-        q = { _id: { $in: [task.file, new ObjectId(task.file)] } };
-      else q = { _id: task.file };
+      if (ObjectId.isValid(t["file"]))
+        q = { _id: { $in: [t["file"], new ObjectId(t["file"])] } };
+      else q = { _id: t["file"] };
 
-      const res = await filesCollection.find(q).toArray();
+      const res = await collections.files.find(q).toArray();
 
       if (res[0]) {
         if (!task.fileType) task.fileType = res[0].metadata.fileType;
@@ -707,34 +667,39 @@ export async function getDueTasks(
 
 export async function clearTasks(
   deviceId: string,
-  taskIds: string[]
+  taskIds: string[],
 ): Promise<void> {
-  await tasksCollection.deleteMany({
+  await collections.tasks.deleteMany({
     _id: { $in: taskIds.map((id) => new ObjectId(id)) },
   });
 }
 
 export async function getOperations(
-  deviceId: string
+  deviceId: string,
 ): Promise<{ [commandKey: string]: Operation }> {
-  const res = await operationsCollection
+  const res = await collections.operations
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
     .toArray();
 
   const operations: { [commandKey: string]: Operation } = {};
   for (const r of res) {
     const commandKey = r._id.slice(deviceId.length + 1);
-    delete r._id;
     // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
     if (typeof r.provisions !== "string") {
-      operations[commandKey] = r as Operation;
+      delete r._id;
+      operations[commandKey] = r as unknown as Operation;
       continue;
     }
-    r.timestamp = +r.timestamp;
-    if (r.args) r.args = JSON.parse(r.args);
-    r.provisions = JSON.parse(r.provisions);
-    r.retries = JSON.parse(r.retries);
-    operations[commandKey] = r as Operation;
+    const operation: Operation = {
+      name: r.name,
+      timestamp: +r.timestamp,
+      channels:
+        typeof r.channels === "string" ? JSON.parse(r.channels) : r.channels,
+      retries: JSON.parse(r.retries),
+      provisions: JSON.parse(r.provisions),
+      ...(r.args && { args: JSON.parse(r.args) }),
+    };
+    operations[commandKey] = operation;
   }
   return operations;
 }
@@ -742,76 +707,26 @@ export async function getOperations(
 export async function saveOperation(
   deviceId: string,
   commandKey: string,
-  operation: Operation
+  operation: Operation,
 ): Promise<void> {
   const id = `${deviceId}:${commandKey}`;
-  const o = Object.assign({}, operation) as Record<string, any>;
-  o["_id"] = id;
-  o["timestamp"] = new Date(operation.timestamp);
-  o["provisions"] = JSON.stringify(operation.provisions);
-  o["retries"] = JSON.stringify(operation.retries);
-  o["args"] = JSON.stringify(operation.args);
-  await operationsCollection.replaceOne({ _id: id }, o, {
+  const o: MongoTypes.Operation = {
+    _id: id,
+    name: operation.name,
+    timestamp: new Date(operation.timestamp),
+    channels: JSON.stringify(operation.channels),
+    provisions: JSON.stringify(operation.provisions),
+    retries: JSON.stringify(operation.retries),
+    args: JSON.stringify(operation.args),
+  };
+  await collections.operations.replaceOne({ _id: id }, o, {
     upsert: true,
   });
 }
 
 export async function deleteOperation(
   deviceId: string,
-  commandKey: string
+  commandKey: string,
 ): Promise<void> {
-  await operationsCollection.deleteOne({ _id: `${deviceId}:${commandKey}` });
-}
-
-export async function getPresets(): Promise<Record<string, any>[]> {
-  return presetsCollection.find().toArray();
-}
-
-export async function getObjects(): Promise<Record<string, any>[]> {
-  return objectsCollection.find().toArray();
-}
-
-export async function getProvisions(): Promise<Record<string, any>[]> {
-  return provisionsCollection.find().toArray();
-}
-
-export async function getVirtualParameters(): Promise<Record<string, any>[]> {
-  return virtualParametersCollection.find().toArray();
-}
-
-export function getFiles(): Promise<Record<string, any>[]> {
-  return filesCollection.find().toArray();
-}
-
-export async function getConfig(): Promise<
-  { id: string; value: Expression }[]
-> {
-  const res = await configCollection.find().toArray();
-  return res.map((c) => ({
-    id: c["_id"],
-    value: parse(c["value"]),
-  }));
-}
-
-interface Permission {
-  role: string;
-  resource: string;
-  access: number;
-  filter: string;
-  validate: string;
-}
-
-export async function getPermissions(): Promise<Permission[]> {
-  return permissionsCollection.find().toArray() as unknown as Permission[];
-}
-
-interface User {
-  _id: string;
-  password: string;
-  salt: string;
-  roles: string;
-}
-
-export async function getUsers(): Promise<User[]> {
-  return usersCollection.find().toArray() as unknown as User[];
+  await collections.operations.deleteOne({ _id: `${deviceId}:${commandKey}` });
 }

lib/cwmp/local-cache.ts

@@ -0,0 +1,357 @@
+import * as vm from "node:vm";
+import * as crypto from "node:crypto";
+import * as config from "../config.ts";
+import { collections } from "../db/db.ts";
+import { convertOldPrecondition } from "../db/util.ts";
+import * as logger from "../logger.ts";
+import * as scheduling from "../scheduling.ts";
+import * as expression from "../common/expression/util.ts";
+import { parse } from "../common/expression/parser.ts";
+import {
+  Preset,
+  Expression,
+  Provisions,
+  VirtualParameters,
+  Files,
+  Config,
+} from "../types.ts";
+import { LocalCache } from "../local-cache.ts";
+
+interface Snapshot {
+  presets: Preset[];
+  provisions: Provisions;
+  virtualParameters: VirtualParameters;
+  files: Files;
+  config: Config;
+}
+
+function flattenObject<T extends Record<keyof T, unknown>>(
+  src: T,
+  prefix = "",
+  dst = {} as T,
+): T {
+  for (const k of Object.keys(src)) {
+    const v = src[k];
+    if (typeof v === "object" && !Array.isArray(v))
+      flattenObject(v as T, `${prefix}${k}.`, dst);
+    else dst[`${prefix}${k}`] = v;
+  }
+  return dst;
+}
+
+async function fetchPresets(): Promise<[string, Preset[]]> {
+  const res = await collections.presets.find().toArray();
+  let objects = await collections.objects.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  objects.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(res))
+    .update(JSON.stringify(objects))
+    .digest("hex");
+
+  objects = objects.map((obj) => {
+    // Flatten object
+    obj = flattenObject(obj);
+
+    // If no keys are defined, consider all parameters as keys to keep the
+    // same behavior from v1.0
+    if (!obj["_keys"]?.length)
+      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
+
+    return obj;
+  });
+
+  res.sort((a, b) => {
+    if (a["weight"] === b["weight"])
+      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
+    else return a["weight"] - b["weight"];
+  });
+
+  const presets = [] as Preset[];
+  for (const preset of res) {
+    let schedule: { md5: string; duration: number; schedule: any } = null;
+    if (preset["schedule"]) {
+      const parts = preset["schedule"].trim().split(/\s+/);
+      schedule = {
+        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
+        duration: null,
+        schedule: null,
+      };
+
+      try {
+        schedule.duration = +parts.shift() * 1000;
+        schedule.schedule = scheduling.parseCron(parts.join(" "));
+      } catch (err) {
+        logger.warn({
+          message: "Invalid preset schedule",
+          preset: preset["_id"],
+          schedule: preset["schedule"],
+        });
+        schedule.schedule = false;
+      }
+    }
+
+    const events = preset["events"] || {};
+    let precondition = true as Expression;
+    if (preset["precondition"]) {
+      try {
+        precondition = parse(preset["precondition"]);
+      } catch (error) {
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"]),
+        );
+      }
+
+      // Simplify expression
+      precondition = expression.evaluate(precondition);
+    }
+
+    const _provisions: Preset["provisions"] = [];
+
+    // Generate provisions from the old configuration format
+    for (const c of preset.configurations) {
+      switch (c.type) {
+        case "age":
+          _provisions.push(["refresh", c.name, +c.age]);
+          break;
+
+        case "value":
+          _provisions.push(["value", c.name, c.value]);
+          break;
+
+        case "add_tag":
+          _provisions.push(["tag", c.tag, true]);
+          break;
+
+        case "delete_tag":
+          _provisions.push(["tag", c.tag, false]);
+          break;
+
+        case "provision":
+          _provisions.push([c.name, ...(c.args || [])]);
+          break;
+
+        case "add_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 1]);
+
+              for (const k in obj) {
+                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
+                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
+              }
+            }
+          }
+
+          break;
+
+        case "delete_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 0]);
+            }
+          }
+
+          break;
+
+        default: {
+          const exhaustiveCheck: never = c;
+          throw new Error(
+            `Unknown configuration type ${exhaustiveCheck["type"]}`,
+          );
+        }
+      }
+    }
+
+    presets.push({
+      name: preset["_id"],
+      channel: (preset["channel"] as string) || "default",
+      schedule: schedule,
+      events: events,
+      precondition: precondition,
+      provisions: _provisions,
+    });
+  }
+
+  return [h, presets];
+}
+
+async function fetchProvisions(): Promise<[string, Provisions]> {
+  const res = await collections.provisions.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+
+  const provisions = {};
+  for (const r of res) {
+    provisions[r._id] = {};
+    provisions[r._id].md5 = crypto
+      .createHash("md5")
+      .update(r.script)
+      .digest("hex");
+    provisions[r._id].script = new vm.Script(
+      `"use strict";(function(){\n${r.script}\n})();`,
+      { filename: r._id, lineOffset: -1 },
+    );
+  }
+
+  return [h, provisions];
+}
+
+async function fetchVirtualParameters(): Promise<[string, VirtualParameters]> {
+  const res = await collections.virtualParameters.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+
+  const virtualParameters = {};
+  for (const r of res) {
+    virtualParameters[r._id] = {};
+    virtualParameters[r._id].md5 = crypto
+      .createHash("md5")
+      .update(r.script)
+      .digest("hex");
+    virtualParameters[r._id].script = new vm.Script(
+      `"use strict";(function(){\n${r.script}\n})();`,
+      { filename: r._id, lineOffset: -1 },
+    );
+  }
+
+  return [h, virtualParameters];
+}
+
+async function fetchFiles(): Promise<[string, Files]> {
+  const res = await collections.files.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+  const files = {};
+
+  for (const r of res) {
+    const id = r.filename || r._id.toString();
+    files[id] = {};
+    files[id].length = r.length;
+  }
+
+  return [h, files];
+}
+
+async function fetchConfig(): Promise<[string, Config]> {
+  const conf = await collections.config.find().toArray();
+  conf.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(conf)).digest("hex");
+
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    _config[c._id] = expression.evaluate(parse(c.value));
+  }
+
+  return [h, _config];
+}
+
+const localCache = new LocalCache<Snapshot>("cwmp-local-cache-hash", refresh);
+
+async function refresh(): Promise<[string, Snapshot]> {
+  const res = await Promise.all([
+    fetchPresets(),
+    fetchProvisions(),
+    fetchVirtualParameters(),
+    fetchFiles(),
+    fetchConfig(),
+  ]);
+
+  const h = crypto.createHash("md5");
+  for (const r of res) h.update(r[0]);
+
+  const snapshot = {
+    presets: res[0][1],
+    provisions: res[1][1],
+    virtualParameters: res[2][1],
+    files: res[3][1],
+    config: res[4][1],
+  };
+
+  return [h.digest("hex"), snapshot];
+}
+
+export async function getRevision(): Promise<string> {
+  return await localCache.getRevision();
+}
+
+export function getPresets(revision: string): Preset[] {
+  return localCache.get(revision).presets;
+}
+
+export function getProvisions(revision: string): Provisions {
+  return localCache.get(revision).provisions;
+}
+
+export function getVirtualParameters(revision: string): VirtualParameters {
+  return localCache.get(revision).virtualParameters;
+}
+
+export function getFiles(revision: string): Files {
+  return localCache.get(revision).files;
+}
+
+export function getConfig(
+  snapshotKey: string,
+  key: string,
+  context: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Expression,
+): string | number | boolean | null {
+  const snapshot = localCache.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const oldOpts = {
+    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
+    "cwmp.debug": "DEBUG",
+    "cwmp.retryDelay": "RETRY_DELAY",
+    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
+    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
+    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
+    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
+    "cwmp.cookiesPath": "COOKIES_PATH",
+    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
+    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
+    "cwmp.connectionRequestAllowBasicAuth":
+      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
+    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
+    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
+  };
+
+  if (!(key in snapshot.config)) {
+    if (key in oldOpts) {
+      let id;
+      if (context?.["id"]) {
+        id = context["id"];
+      } else if (cb) {
+        id = cb(["PARAM", "DeviceID.ID"]);
+        if (Array.isArray(id)) id = null;
+      }
+      return config.get(oldOpts[key], id);
+    }
+    return null;
+  }
+
+  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+  return Array.isArray(v) ? null : v;
+}
+
+export function getConfigExpression(
+  snapshotKey: string,
+  key: string,
+): Expression {
+  const snapshot = localCache.get(snapshotKey);
+  return snapshot.config[key];
+}

lib/db/db.ts

@@ -0,0 +1,57 @@
+import { MongoClient, Collection, GridFSBucket } from "mongodb";
+import { get } from "../config.ts";
+import * as MongoTypes from "./types.ts";
+
+export let filesBucket: GridFSBucket;
+
+export const collections = {
+  devices: null as Collection<MongoTypes.Device>,
+  presets: null as Collection<MongoTypes.Preset>,
+  objects: null as Collection<MongoTypes.Object>,
+  provisions: null as Collection<MongoTypes.Provision>,
+  virtualParameters: null as Collection<MongoTypes.VirtualParameter>,
+  faults: null as Collection<MongoTypes.Fault>,
+  tasks: null as Collection<MongoTypes.Task>,
+  files: null as Collection<MongoTypes.File>,
+  operations: null as Collection<MongoTypes.Operation>,
+  permissions: null as Collection<MongoTypes.Permission>,
+  users: null as Collection<MongoTypes.User>,
+  config: null as Collection<MongoTypes.Config>,
+  cache: null as Collection<MongoTypes.Cache>,
+  locks: null as Collection<MongoTypes.Lock>,
+};
+
+let clientPromise: Promise<MongoClient>;
+
+export async function connect(): Promise<void> {
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"));
+
+  const client = await clientPromise;
+  const db = client.db();
+
+  collections.tasks = db.collection("tasks");
+  collections.devices = db.collection("devices");
+  collections.presets = db.collection("presets");
+  collections.objects = db.collection("objects");
+  collections.files = db.collection("fs.files");
+  collections.provisions = db.collection("provisions");
+  collections.virtualParameters = db.collection("virtualParameters");
+  collections.faults = db.collection("faults");
+  collections.operations = db.collection("operations");
+  collections.permissions = db.collection("permissions");
+  collections.users = db.collection("users");
+  collections.config = db.collection("config");
+  collections.cache = db.collection("cache");
+  collections.locks = db.collection("locks");
+  filesBucket = new GridFSBucket(db);
+
+  await Promise.all([
+    collections.tasks.createIndex({ device: 1, timestamp: 1 }),
+    collections.cache.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }),
+    collections.locks.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }),
+  ]);
+}
+
+export async function disconnect(): Promise<void> {
+  if (clientPromise != null) await (await clientPromise).close();
+}

lib/db/synth.ts

@@ -0,0 +1,654 @@
+import { Filter } from "mongodb";
+import { EJSON } from "bson";
+import { parseLikePattern } from "../common/expression/parser.ts";
+import { Expression } from "../types.ts";
+import { decodeTag } from "../util.ts";
+import {
+  SynthContextBase,
+  likeDisjoint,
+  likeImplies,
+  Clause,
+} from "../common/expression/synth.ts";
+import normalize from "../common/expression/normalize.ts";
+import * as BI from "../common/expression/bigint.ts";
+
+type Minterm = number[];
+
+function getParam(exp: Expression, collection: string): string {
+  if (!Array.isArray(exp))
+    throw new Error("Left-hand operand must be a parameter");
+
+  if (exp[0] !== "PARAM")
+    throw new Error("Left-hand operand must be a parameter");
+
+  if (typeof exp[1] !== "string")
+    throw new Error("Left-hand operand must be a parameter");
+
+  const p = exp[1];
+  if (collection === "devices") {
+    if (p === "DeviceID.ID") return "_id";
+    else if (p === "DeviceID") return "_deviceId";
+    else if (p.startsWith("DeviceID.")) return "_deviceId._" + p.slice(9);
+    else if (p === "Events.Inform") return "_lastInform";
+    else if (p === "Events.Registered") return "_registered";
+    else if (p === "Events.0_BOOTSTRAP") return "_lastBootstrap";
+    else if (p === "Events.1_BOOT") return "_lastBoot";
+    else if (!p.endsWith("._value") && !p.startsWith("Tags."))
+      return `${p}._value`;
+  }
+
+  return p;
+}
+
+function getTypes(parameter: string, collection: string): string[] {
+  if (collection === "devices") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "_lastInform") return ["date"];
+    if (parameter === "_registered") return ["date"];
+    if (parameter === "_lastBootstrap") return ["date"];
+    if (parameter === "_lastBoot") return ["date"];
+    if (parameter.startsWith("_deviceId.")) return ["string"];
+    if (parameter === "Reboot._value") return ["date"];
+    if (parameter === "FactoryReset._value") return ["date"];
+    if (parameter.endsWith("_timestamp")) return ["date"];
+    if (parameter.startsWith("Downloads.")) {
+      if (parameter.endsWith("Download._value")) return ["date"];
+      if (parameter.endsWith("Time._value")) return ["date"];
+      if (parameter.endsWith("Name._value")) return ["string"];
+      if (parameter.endsWith("Type._value")) return ["string"];
+    }
+    if (parameter.endsWith("_value"))
+      return ["bool", "number", "date", "string"];
+  } else if (collection === "tasks") {
+    if (parameter === "_id") return ["oid"];
+    if (parameter === "timestamp") return ["date"];
+    if (parameter === "expiry") return ["date"];
+    if (parameter === "name") return ["string"];
+    if (parameter === "device") return ["string"];
+  } else if (collection === "faults") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "timestamp") return ["date"];
+    if (parameter === "expiry") return ["date"];
+    if (parameter === "code") return ["string"];
+    if (parameter === "retries") return ["number"];
+    if (parameter === "channel") return ["string"];
+    if (parameter === "device") return ["string"];
+    if (parameter === "message") return ["string"];
+  } else if (collection === "users") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "roles") return ["string"];
+    else if (parameter === "password")
+      throw new Error("Cannot query restricted parameters");
+    else if (parameter === "salt")
+      throw new Error("Cannot query restricted parameters");
+  } else if (collection === "config") {
+    if (parameter === "_id") return ["string"];
+    else if (parameter === "value") return ["string"];
+  } else if (collection === "files") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "metadata.fileType") return ["string"];
+    if (parameter === "metadata.oui") return ["string"];
+    if (parameter === "metadata.productClass") return ["string"];
+    if (parameter === "metadata.version") return ["string"];
+  } else if (collection === "permissions") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "role") return ["string"];
+    if (parameter === "resource") return ["devices"];
+    if (parameter === "access") return ["number"];
+    if (parameter === "filter") return ["string"];
+    if (parameter === "validate") return ["string"];
+  } else if (collection === "presets") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "weight") return ["number"];
+    if (parameter === "channel") return ["string"];
+    if (parameter === "precondition") return ["string"];
+    if (parameter.startsWith("events.")) return ["bool"];
+  } else if (collection === "provisions") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "script") return ["string"];
+  } else if (collection === "virtualParameters") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "script") return ["string"];
+  }
+
+  if (parameter === "_id") return ["oid", "string"];
+  return ["bool", "number", "date", "string"];
+}
+
+function roundOid(oid: string, roundUp: boolean): string {
+  const match = (oid.match(/^[0-9a-f]*/)?.[0] ?? "").slice(0, 24);
+  let num = BI.BigInt("0x0" + match);
+  let lastChar = 0;
+  if (oid.length > match.length) lastChar += oid.charCodeAt(match.length);
+  if (match.length < 24) lastChar -= 48;
+  if (lastChar > 0 && roundUp) num++;
+  num <<= BI.BigInt(4 * (24 - match.length));
+  if (lastChar < 0 && !roundUp) --num;
+  const str = num.toString(16);
+  if (str.length > 24 || str.startsWith("-")) return null;
+  return str.padStart(24, "0");
+}
+
+function groupBy<T, K>(
+  input: T[],
+  callback: (item: T) => K,
+): Iterable<[K, T[]]> {
+  const groups = new Map<K, T[]>();
+  for (const item of input) {
+    const key = callback(item);
+    let arr = groups.get(key);
+    if (!arr) groups.set(key, (arr = []));
+    arr.push(item);
+  }
+
+  return groups.entries();
+}
+
+abstract class MongoClause {
+  abstract readonly parameter: string;
+  abstract toQuery(truthy: boolean): Filter<unknown>;
+  toString(): string {
+    return JSON.stringify(this.toQuery(true));
+  }
+}
+
+class MongoClauseArray extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly value: string,
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    if (truthy) return { [this.parameter]: { $eq: this.value } };
+    else return { [this.parameter]: { $ne: this.value } };
+  }
+}
+
+class MongoClauseCompare<T> extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly op: "$eq" | "$gt" | "$lt" | "$gte" | "$lte",
+    public readonly value: T,
+    public readonly type: "" | "bool" | "number" | "string" | "date" | "oid",
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    let v: any = this.value;
+    if (this.type === "date" && typeof v === "number")
+      v = { $date: new Date(v).toISOString() };
+    if (this.type === "oid" && typeof v === "string") v = { $oid: v };
+
+    if (truthy) return { [this.parameter]: { [this.op]: v } };
+    else if (this.op === "$eq") return { [this.parameter]: { $ne: v } };
+    else return { [this.parameter]: { $not: { [this.op]: v } } };
+  }
+}
+
+class MongoClauseType extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly type: string,
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    if (truthy) return { [this.parameter]: { $type: this.type } };
+    else return { [this.parameter]: { $not: { $type: this.type } } };
+  }
+}
+
+class MongoClauseLike extends MongoClause {
+  readonly pattern: string[];
+  constructor(
+    public readonly parameter: string,
+    pat: string,
+    esc: string,
+    public readonly caseSensitive: boolean,
+  ) {
+    super();
+    this.pattern = parseLikePattern(pat, esc);
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    const convChars = {
+      "-": "\\-",
+      "/": "\\/",
+      "\\": "\\/",
+      "^": "\\^",
+      $: "\\$",
+      "*": "\\*",
+      "+": "\\+",
+      "?": "\\?",
+      ".": "\\.",
+      "(": "\\(",
+      ")": "\\)",
+      "|": "\\|",
+      "[": "\\[",
+      "]": "\\]",
+      "{": "\\{",
+      "}": "\\}",
+      "\\%": ".*",
+      "\\_": ".",
+    };
+
+    const chars = this.pattern.map((c) => convChars[c] ?? c);
+    chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
+    const l = chars.length - 1;
+    chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
+    const pattern = chars.join("");
+    const options = this.caseSensitive ? "s" : "is";
+    if (truthy) {
+      return { [this.parameter]: { $regularExpression: { options, pattern } } };
+    } else {
+      return {
+        [this.parameter]: {
+          $not: { $regularExpression: { options, pattern } },
+        },
+      };
+    }
+  }
+}
+
+class MongoSynthContext extends SynthContextBase<Clause, MongoClause> {
+  constructor(private readonly collection: string) {
+    super();
+  }
+
+  getMinterms(clause: Clause, res: true | false | null): number[][] {
+    const exp = clause.expression();
+    if (!Array.isArray(exp)) throw new Error("Invalid query expression");
+
+    if (res === null) {
+      const minterms: number[][] = [];
+      for (const dep of clause.getNullables()) {
+        const e = dep.operand.expression();
+        const param = getParam(e, this.collection);
+        if (param.startsWith("Tags.") && this.collection === "devices") {
+          const t = decodeTag(param.slice(5));
+          const c = new MongoClauseArray("_tags", t);
+          minterms.push([this.getVar(c) << 1]);
+          continue;
+        }
+        const c = new MongoClauseCompare(param, "$eq", null, "");
+        minterms.push([(this.getVar(c) << 1) ^ 1]);
+      }
+      return minterms;
+    }
+
+    if ([">", "<", "="].includes(exp[0])) {
+      const param = getParam(exp[1], this.collection);
+
+      let rhs = exp[2];
+      if (typeof rhs === "boolean") rhs = +rhs;
+      else if (typeof rhs !== "string" && typeof rhs !== "number")
+        throw new Error(`Right-hand operand must be a literal value`);
+
+      if (
+        param.startsWith("Tags.") &&
+        this.collection === "devices" &&
+        exp[0] === "="
+      ) {
+        const t = decodeTag(param.slice(5));
+        const c = new MongoClauseArray("_tags", t);
+        if (typeof rhs === "string") rhs = 2;
+        if (exp[0] === "=") {
+          if ((rhs === 1) !== res) return [];
+        } else if (exp[0] === ">") {
+          if ((res && rhs >= 1) || (!res && rhs < 1)) return [];
+        } else if (exp[0] === "<") {
+          if ((res && rhs <= 1) || (!res && rhs > 1)) return [];
+        }
+        return [[(this.getVar(c) << 1) ^ 1]];
+      }
+
+      let op: "$gt" | "$lt" | "$eq" | "$gte" | "$lte";
+      if (exp[0] === "=") op = "$eq";
+      else if (exp[0] === ">") op = res ? "$gt" : "$lte";
+      else if (exp[0] === "<") op = res ? "$lt" : "$gte";
+
+      const possibleTypes = new Set(getTypes(param, this.collection));
+      const clauses: MongoClause[] = [];
+
+      if (typeof rhs === "number") {
+        if (possibleTypes.has("number"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "number"));
+
+        if (possibleTypes.has("date"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "date"));
+        if (possibleTypes.has("bool") && (rhs === 0 || rhs === 1))
+          clauses.push(new MongoClauseCompare(param, op, !!rhs, "bool"));
+      } else if (typeof rhs === "string") {
+        if (possibleTypes.has("string"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "string"));
+
+        if (possibleTypes.has("oid")) {
+          const oid = roundOid(rhs, op === "$lt" || op === "$lte");
+          if (oid && (op !== "$eq" || oid === rhs))
+            clauses.push(new MongoClauseCompare(param, op, oid, "oid"));
+        }
+      }
+
+      if (op === "$eq" && !res) {
+        clauses.push(new MongoClauseCompare(param, "$eq", null, ""));
+        return [clauses.map((c) => this.getVar(c) << 1)];
+      }
+
+      // In the following clauses we could use $type operator, but we want the
+      // final query to use comparison operators to check for type because the
+      // $type operator in MongoDB doesn't use indexes
+      if (typeof rhs === "number") {
+        if (possibleTypes.has("bool")) {
+          if (
+            (rhs > 1 && (op === "$lt" || op === "$lte")) ||
+            (rhs < 0 && (op === "$gt" || op === "$gte"))
+          )
+            clauses.push(new MongoClauseCompare(param, "$gte", false, "bool"));
+        }
+
+        if (op === "$gt" || op === "$gte") {
+          if (possibleTypes.has("string"))
+            clauses.push(new MongoClauseCompare(param, "$gte", "", "string"));
+
+          if (possibleTypes.has("oid")) {
+            clauses.push(
+              new MongoClauseCompare(
+                param,
+                "$gte",
+                "000000000000000000000000",
+                "oid",
+              ),
+            );
+          }
+        }
+      } else if (typeof rhs === "string") {
+        if (op === "$lt" || op === "$lte") {
+          if (possibleTypes.has("bool"))
+            clauses.push(new MongoClauseCompare(param, "$gte", false, "bool"));
+
+          if (possibleTypes.has("number")) {
+            clauses.push(new MongoClauseCompare(param, "$gte", 0, "number"));
+            clauses.push(new MongoClauseCompare(param, "$lt", 0, "number"));
+          }
+          if (possibleTypes.has("date")) {
+            clauses.push(new MongoClauseCompare(param, "$gte", 0, "date"));
+            clauses.push(new MongoClauseCompare(param, "$lt", 0, "date"));
+          }
+        }
+      }
+
+      return clauses.map((c) => [(this.getVar(c) << 1) ^ 1]);
+    } else if (exp[0] === "LIKE") {
+      const pat = exp[2];
+      if (typeof pat !== "string")
+        throw new Error("Right-hand operand of 'LIKE' must be a string");
+      let p = exp[1];
+      let caseSensitive = true;
+      if (
+        Array.isArray(p) &&
+        p[0] === "FUNC" &&
+        ["UPPER", "LOWER"].includes(p[1])
+      ) {
+        if (p[1] === "UPPER" && pat !== pat.toUpperCase())
+          return res ? [] : [[]];
+        if (p[1] === "LOWER" && pat !== pat.toLowerCase())
+          return res ? [] : [[]];
+        caseSensitive = false;
+        p = p[2];
+      }
+
+      const param = getParam(p, this.collection);
+      const c = new MongoClauseLike(param, pat, exp[3], caseSensitive);
+      if (res) return [[(this.getVar(c) << 1) ^ 1]];
+      const typeClase = new MongoClauseType(param, "string");
+      const r = [[this.getVar(c) << 1, (this.getVar(typeClase) << 1) ^ 1]];
+      return r;
+    } else {
+      throw new Error("Invalid query expression");
+    }
+  }
+
+  getDcSet(minterms: Minterm[]): number[][] {
+    const dcSet: number[][] = [];
+
+    const vars = new Set(minterms.flat().map((l) => l >> 1));
+    const clauses = Array.from(vars).map((v) => this.getClause(v));
+
+    for (const [parameter, clauses2] of groupBy(clauses, (c) => c.parameter)) {
+      const comparisons = clauses2.filter(
+        (c) => c instanceof MongoClauseCompare && c.value !== null,
+      ) as MongoClauseCompare<unknown>[];
+
+      for (const [type, clauses3] of groupBy(comparisons, (c) => c.type)) {
+        const isType = this.getVar(new MongoClauseType(parameter, type));
+        const values = new Set(clauses3.map((c) => c.value));
+        const valuesSorted = Array.from(values).sort((a, b) =>
+          a > b ? 1 : -1,
+        );
+        for (const [i, v] of valuesSorted.entries()) {
+          const eq = this.getVar(
+            new MongoClauseCompare(parameter, "$eq", v, type),
+          );
+          const gt = this.getVar(
+            new MongoClauseCompare(parameter, "$gt", v, type),
+          );
+          const lt = this.getVar(
+            new MongoClauseCompare(parameter, "$lt", v, type),
+          );
+          const gte = this.getVar(
+            new MongoClauseCompare(parameter, "$gte", v, type),
+          );
+          const lte = this.getVar(
+            new MongoClauseCompare(parameter, "$lte", v, type),
+          );
+
+          if (type === "bool") {
+            if (v === false) dcSet.push([(lt << 1) ^ 1]);
+            else if (v === true) dcSet.push([(gt << 1) ^ 1]);
+          } else if (type === "string") {
+            if (v === "") dcSet.push([(lt << 1) ^ 1]);
+          } else if (type === "oid") {
+            if (v < "000000000000000000000000") dcSet.push([(lte << 1) ^ 1]);
+            else if (v === "000000000000000000000000")
+              dcSet.push([(lt << 1) ^ 1]);
+            else if (v > "ffffffffffffffffffffffff")
+              dcSet.push([(gte << 1) ^ 1]);
+            else if (v === "ffffffffffffffffffffffff")
+              dcSet.push([(gt << 1) ^ 1]);
+          }
+          dcSet.push([(lt << 1) ^ 1, (gte << 1) ^ 1]);
+          dcSet.push([(gt << 1) ^ 1, (gte << 1) ^ 0]);
+          dcSet.push([(eq << 1) ^ 0, (lt << 1) ^ 0, (lte << 1) ^ 1]);
+          dcSet.push([(eq << 1) ^ 1, (gte << 1) ^ 0]);
+          dcSet.push([(eq << 1) ^ 1, (gt << 1) ^ 1]);
+
+          if (i === 0) {
+            dcSet.push([(isType << 1) ^ 0, (gte << 1) ^ 1]);
+            dcSet.push([(isType << 1) ^ 0, (lt << 1) ^ 1]);
+          } else {
+            const gt2 = this.getVar(
+              new MongoClauseCompare(
+                parameter,
+                "$gt",
+                valuesSorted[i - 1],
+                type,
+              ),
+            );
+            const lte2 = this.getVar(
+              new MongoClauseCompare(
+                parameter,
+                "$lte",
+                valuesSorted[i - 1],
+                type,
+              ),
+            );
+            dcSet.push([(gt2 << 1) ^ 0, (gte << 1) ^ 1]);
+            dcSet.push([(gt2 << 1) ^ 0, (lte2 << 1) ^ 0, (lt << 1) ^ 1]);
+          }
+
+          if (i === valuesSorted.length - 1)
+            dcSet.push([(isType << 1) ^ 1, (gt << 1) ^ 0, (lte << 1) ^ 0]);
+        }
+      }
+
+      const likes = clauses2.filter(
+        (c) => c instanceof MongoClauseLike,
+      ) as MongoClauseLike[];
+      if (likes.length) {
+        const isType = this.getVar(new MongoClauseType(parameter, "string"));
+        for (let i1 = 0; i1 < likes.length; ++i1) {
+          const l1 = likes[i1];
+          let p1 = l1.pattern;
+          dcSet.push([(this.getVar(l1) << 1) ^ 1, (isType << 1) ^ 0]);
+          for (let i2 = i1 + 1; i2 < likes.length; ++i2) {
+            const l2 = likes[i2];
+            let p2 = l2.pattern;
+            if (!l1.caseSensitive || !l2.caseSensitive) {
+              p1 = p1.map((c) => c.toLowerCase());
+              p2 = p2.map((c) => c.toLowerCase());
+            }
+            if (likeDisjoint(p1, p2)) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 1,
+                (this.getVar(l2) << 1) ^ 1,
+              ]);
+            } else if (
+              (!l1.caseSensitive || l2.caseSensitive) &&
+              likeImplies(p1, p2)
+            ) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 0,
+                (this.getVar(l2) << 1) ^ 1,
+              ]);
+            } else if (
+              (!l2.caseSensitive || l1.caseSensitive) &&
+              likeImplies(p2, p1)
+            ) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 1,
+                (this.getVar(l2) << 1) ^ 0,
+              ]);
+            }
+          }
+        }
+      }
+
+      const isNull = this.getVar(
+        new MongoClauseCompare(parameter, "$eq", null, ""),
+      );
+
+      const types = getTypes(parameter, this.collection).map((t) =>
+        this.getVar(new MongoClauseType(parameter, t)),
+      );
+
+      dcSet.push([(isNull << 1) ^ 0, ...types.map((t) => (t << 1) ^ 0)]);
+      for (let i = 0; i < types.length; ++i) {
+        const t1 = types[i];
+        dcSet.push([(t1 << 1) ^ 1, (isNull << 1) ^ 1]);
+        for (let j = i + 1; j < types.length; ++j) {
+          const t2 = types[j];
+          dcSet.push([(t1 << 1) ^ 1, (t2 << 1) ^ 1]);
+        }
+      }
+
+      if (parameter === "_id") dcSet.push([(isNull << 1) ^ 1]);
+    }
+    return dcSet;
+  }
+
+  canRaise(i: number, s: Set<number>): boolean {
+    if (!(i & 1)) return true;
+    const c = this.getClause(i >> 1);
+    if (c instanceof MongoClauseCompare) {
+      for (const j of s) {
+        if (j === i) continue;
+        const c2 = this.getClause(j >> 1);
+        if (c2.parameter !== c.parameter) continue;
+        if (!(j & 1)) return false;
+        if (c2 instanceof MongoClauseType) return false;
+      }
+    }
+    return true;
+  }
+
+  toQuery(minterms: Minterm[]): Filter<unknown> {
+    const or = [];
+
+    const ejsonOps = ["$oid", "$date", "$regularExpression"];
+
+    for (const minterm of minterms) {
+      const query = {};
+      loop: for (const clause of minterm) {
+        if (!minterm.length) return {};
+        const negate = !!(clause & 1);
+        const c = this.getClause(clause >> 1);
+        const q = c.toQuery(negate);
+        if (Object.keys(q).length !== 1)
+          throw new Error("Invalid query expression");
+        const [param, value] = Object.entries(q)[0];
+        const dests = [query, ...(query["$and"] ?? [])];
+        for (const dest of dests) {
+          if (!(param in dest)) {
+            dest[param] = value;
+            continue loop;
+          }
+
+          let src = value;
+          let dst = dest[param];
+          if (Object.getPrototypeOf(src).constructor !== Object) continue;
+          if (Object.getPrototypeOf(dst).constructor !== Object) continue;
+          let srcKeys = Object.keys(src);
+          let dstKeys = Object.keys(dst);
+
+          if ([...srcKeys, ...dstKeys].every((k) => k === "$not")) {
+            src = src["$not"];
+            dst = dst["$not"];
+            if (Object.getPrototypeOf(src).constructor !== Object) continue;
+            if (Object.getPrototypeOf(dst).constructor !== Object) continue;
+            srcKeys = Object.keys(src);
+            dstKeys = Object.keys(dst);
+          }
+
+          if (srcKeys.some((k) => dstKeys.includes(k))) continue;
+
+          // Don't mix regular operators with EJSON special operators
+          if (srcKeys.some((k) => ejsonOps.includes(k))) continue;
+          if (dstKeys.some((k) => ejsonOps.includes(k))) continue;
+
+          Object.assign(dst, src);
+          continue loop;
+        }
+
+        query["$and"] ??= [];
+        query["$and"].push({ [param]: value });
+      }
+      or.push(query);
+    }
+
+    if (or.length === 1) return or[0];
+    return { $or: or };
+  }
+}
+
+export function toMongoQuery(
+  exp: Expression,
+  resource: string,
+): Filter<unknown> | false {
+  exp = normalize(exp);
+  const clause = Clause.fromExpression(normalize(exp));
+  const context = new MongoSynthContext(resource);
+  const minterms = clause.true(context);
+  const minimized = context.minimize(minterms);
+  if (!minimized.length) return false;
+  return EJSON.deserialize(context.toQuery(minimized));
+}
+
+export function validQuery(exp: Expression, resource: string): void {
+  const clause = Clause.fromExpression(normalize(exp));
+  const context = new MongoSynthContext(resource);
+  clause.true(context);
+}

lib/db/types.ts

@@ -0,0 +1,183 @@
+import { ObjectId } from "mongodb";
+import { Expression, FaultStruct } from "../types.ts";
+
+export interface Fault {
+  _id: string;
+  device: string;
+  channel: string;
+  timestamp: Date;
+  provisions: string;
+  retries: number;
+  code: string;
+  message: string;
+  detail?:
+    | FaultStruct
+    | {
+        name: string;
+        message: string;
+        stack?: string;
+      };
+  expiry?: Date;
+}
+
+interface TaskBase {
+  _id: ObjectId;
+  timestamp?: Date;
+  expiry?: Date;
+  name: string;
+  device: string;
+}
+
+interface TaskGetParameterValues extends TaskBase {
+  name: "getParameterValues";
+  parameterNames: string[];
+}
+
+interface TaskSetParameterValues extends TaskBase {
+  name: "setParameterValues";
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskRefreshObject extends TaskBase {
+  name: "refreshObject";
+  objectName: string;
+}
+
+interface TaskReboot extends TaskBase {
+  name: "reboot";
+}
+
+interface TaskFactoryReset extends TaskBase {
+  name: "factoryReset";
+}
+
+interface TaskDownload extends TaskBase {
+  name: "download";
+  fileType: string;
+  fileName: string;
+  targetFileName?: string;
+}
+
+interface TaskAddObject extends TaskBase {
+  name: "addObject";
+  objectName: string;
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskDeleteObject extends TaskBase {
+  name: "deleteObject";
+  objectName: string;
+}
+
+interface TaskProvisions extends TaskBase {
+  name: "provisions";
+  provisions?: [string, ...Expression[]][];
+}
+
+export type Task =
+  | TaskGetParameterValues
+  | TaskSetParameterValues
+  | TaskRefreshObject
+  | TaskReboot
+  | TaskFactoryReset
+  | TaskDownload
+  | TaskAddObject
+  | TaskDeleteObject
+  | TaskProvisions;
+
+export interface Operation {
+  _id: string;
+  name: string;
+  timestamp: Date;
+  channels: string;
+  retries: string;
+  provisions: string;
+  args: string;
+}
+
+export interface Config {
+  _id: string;
+  value: string;
+}
+
+export interface Cache {
+  _id: string;
+  value: string;
+  timestamp: Date;
+  expire: Date;
+}
+
+export interface Device {
+  _id: string;
+  _lastInform: Date;
+  _registered: Date;
+  _tags?: string[];
+  _timestamp?: Date;
+}
+
+type Configuration =
+  | { type: "age"; name: string; age: number }
+  | { type: "value"; name: string; value: boolean | number | string }
+  | { type: "add_tag"; tag: string }
+  | { type: "delete_tag"; tag: string }
+  | { type: "add_object"; name: string; object: string }
+  | { type: "delete_object"; name: string; object: string }
+  | { type: "provision"; name: string; args?: Expression[] };
+
+export interface Preset {
+  _id: string;
+  weight: number;
+  channel: string;
+  events: Record<string, boolean>;
+  configurations: Configuration[];
+}
+
+export interface Object {
+  _id: string;
+}
+
+export interface Provision {
+  _id: string;
+  script: string;
+}
+
+export interface VirtualParameter {
+  _id: string;
+  script: string;
+}
+
+export interface File {
+  _id: string;
+  length: number;
+  filename: string;
+  uploadDate: Date;
+  metadata?: {
+    fileType?: string;
+    oui?: string;
+    productClass?: string;
+    version?: string;
+  };
+}
+
+export interface Permission {
+  _id: string;
+  role: string;
+  resource: string;
+  access: 1 | 2 | 3;
+  filter?: string;
+  validate?: string;
+}
+
+export interface User {
+  _id: string;
+  password: string;
+  roles: string;
+  salt: string;
+}
+
+export interface Lock {
+  _id: string;
+  value: string;
+  timestamp: Date;
+  expire: Date;
+}

lib/db/util.ts

@@ -0,0 +1,123 @@
+import { Expression } from "../types.ts";
+import { encodeTag } from "../util.ts";
+
+// Optimize projection by removing overlaps
+// This can modify the object
+export function optimizeProjection(obj: { [path: string]: 1 }): {
+  [path: string]: 1;
+} {
+  if (obj[""]) return { "": obj[""] };
+
+  const keys = Object.keys(obj).sort();
+  if (keys.length <= 1) return obj;
+
+  for (let i = 1; i < keys.length; ++i) {
+    const a = keys[i - 1];
+    const b = keys[i];
+    if (b.startsWith(a)) {
+      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
+        delete obj[b];
+        keys.splice(i--, 1);
+      }
+    }
+  }
+  return obj;
+}
+
+export function convertOldPrecondition(q: Record<string, unknown>): Expression {
+  function recursive(_query): Expression {
+    const expressions: Expression[] = [];
+    for (const [k, v] of Object.entries(_query)) {
+      if (k[0] === "$") {
+        if (k === "$and") {
+          const and: Expression = ["AND"];
+          for (const vv of Object.values(v)) and.push(recursive(vv));
+          expressions.push(and);
+        } else if (k === "$or") {
+          const or: Expression = ["OR"];
+          for (const vv of Object.values(v)) or.push(recursive(vv));
+          expressions.push(or);
+        } else {
+          throw new Error(`Operator ${k} not supported`);
+        }
+      } else if (k === "_tags") {
+        if (typeof v === "object") {
+          if (Array.isArray(v)) throw new Error(`Invalid type`);
+          const conjs: Expression[] = [];
+          for (const [op, val] of Object.entries(v)) {
+            if (op === "$ne") {
+              if (typeof v["$ne"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else {
+              throw new Error(`Invalid tag query`);
+            }
+          }
+          if (conjs.length === 1) expressions.push(conjs[0]);
+          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
+        } else {
+          expressions.push([
+            "IS NOT NULL",
+            ["PARAM", `Tags.${encodeTag(v as string)}`],
+          ]);
+        }
+      } else if (k.startsWith("Tags.")) {
+        let exists: boolean;
+        if (typeof v === "boolean") exists = v;
+        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
+        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
+        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
+        else throw new Error(`Invalid tag query`);
+
+        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
+      } else if (typeof v === "object") {
+        if (Array.isArray(v)) throw new Error(`Invalid type`);
+
+        const exps: Expression[] = [];
+        for (const [kk, vv] of Object.entries(v)) {
+          if (kk === "$eq") {
+            exps.push(["=", ["PARAM", k], vv]);
+          } else if (kk === "$ne") {
+            exps.push([
+              "OR",
+              ["<>", ["PARAM", k], vv],
+              ["IS NULL", ["PARAM", k]],
+            ]);
+          } else if (kk === "$lt") {
+            exps.push(["<", ["PARAM", k], vv]);
+          } else if (kk === "$lte") {
+            exps.push(["<=", ["PARAM", k], vv]);
+          } else if (kk === "$gt") {
+            exps.push([">", ["PARAM", k], vv]);
+          } else if (kk === "$gte") {
+            exps.push([">=", ["PARAM", k], vv]);
+          } else {
+            throw new Error(`Operator ${kk} not supported`);
+          }
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
+        }
+        if (exps.length === 1) {
+          expressions.push(exps[0]);
+        } else if (exps.length > 1) {
+          const and: Expression = ["AND"];
+          expressions.push(and.concat(exps));
+        }
+      } else {
+        expressions.push(["=", ["PARAM", k], v]);
+      }
+    }
+    if (expressions.length === 1) return expressions[0];
+    if (expressions.length === 0) return true;
+    return ["AND", ...expressions];
+  }
+
+  // empty filter
+  if (!Object.keys(q).length) return true;
+
+  return recursive(q);
+}

lib/debug.ts

@@ -1,27 +1,9 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { IncomingMessage, ServerResponse, ClientRequest } from "http";
-import { Socket } from "net";
-import { appendFileSync } from "fs";
-import { stringify } from "./common/yaml";
-import * as config from "./config";
+import { IncomingMessage, ServerResponse, ClientRequest } from "node:http";
+import { Socket } from "node:net";
+import { appendFileSync } from "node:fs";
+import { stringify } from "./common/yaml.ts";
+import * as config from "./config.ts";
+import { getSocketEndpoints } from "./server.ts";
 
 const DEBUG_FILE = "" + config.get("DEBUG_FILE");
 const DEBUG_FORMAT = "" + config.get("DEBUG_FORMAT");
@@ -40,18 +22,19 @@ function getConnectionTimestamp(connection: Socket): Date {
 export function incomingHttpRequest(
   httpRequest: IncomingMessage,
   deviceId: string,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
   const con = httpRequest.socket;
+  const socketEndpoints = getSocketEndpoints(con);
   const msg = {
     event: "incoming HTTP request",
     timestamp: now,
-    remoteAddress: con.remoteAddress,
+    remoteAddress: socketEndpoints.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
-    localPort: con.localPort,
+    localPort: socketEndpoints.localPort,
     method: httpRequest.method,
     url: httpRequest.url,
     headers: httpRequest.headers,
@@ -68,15 +51,16 @@ export function incomingHttpRequest(
 export function outgoingHttpResponse(
   httpResponse: ServerResponse,
   deviceId: string,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
   const con = httpResponse.socket;
+  const socketEndpoints = getSocketEndpoints(con);
   const msg = {
     event: "outgoing HTTP response",
     timestamp: now,
-    remoteAddress: con.remoteAddress,
+    remoteAddress: socketEndpoints.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
     statusCode: httpResponse.statusCode,
@@ -96,7 +80,7 @@ export function outgoingHttpRequest(
   deviceId: string,
   method: "GET" | "PUT" | "POST" | "DELETE",
   url: URL,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
@@ -126,7 +110,7 @@ export function outgoingHttpRequestError(
   deviceId: string,
   method: "GET" | "PUT" | "POST" | "DELETE",
   url: URL,
-  err: Error
+  err: Error,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
@@ -153,7 +137,7 @@ export function outgoingHttpRequestError(
 export function incomingHttpResponse(
   httpResponse: IncomingMessage,
   deviceId: string,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
@@ -180,7 +164,7 @@ export function outgoingUdpMessage(
   remoteAddress: string,
   deviceId: string,
   remotePort: number,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
@@ -199,3 +183,54 @@ export function outgoingUdpMessage(
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
 }
+
+export function clientError(remoteAddress: string, err: Error): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "client error",
+    timestamp: now,
+    remoteAddress: remoteAddress,
+    error: err.message,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function incomingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "incoming XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}

lib/default-provisions.ts

@@ -1,34 +1,15 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
-import * as config from "./config";
-import * as device from "./device";
-import * as scheduling from "./scheduling";
-import { SessionContext, Declaration } from "./types";
+import Path from "./common/path.ts";
+import * as config from "./config.ts";
+import * as device from "./device.ts";
+import * as scheduling from "./scheduling.ts";
+import { SessionContext, Declaration } from "./types.ts";
 
 const MAX_DEPTH = +config.get("MAX_DEPTH");
 
 export function refresh(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (
     (provision.length !== 2 || typeof provision[1] !== "string") &&
@@ -84,7 +65,7 @@ export function refresh(
 export function value(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (
     provision.length < 3 ||
@@ -127,7 +108,7 @@ export function value(
 export function tag(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (
     provision.length !== 3 ||
@@ -151,7 +132,7 @@ export function tag(
 export function reboot(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (provision.length !== 1) throw new Error("Invalid arguments");
 
@@ -170,7 +151,7 @@ export function reboot(
 export function reset(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (provision.length !== 1) throw new Error("Invalid arguments");
 
@@ -189,7 +170,7 @@ export function reset(
 export function download(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (
     (provision.length !== 3 ||
@@ -234,7 +215,7 @@ export function instances(
   provision: (string | number | boolean)[],
   declarations: Declaration[],
   startRevision: number,
-  endRevision: number
+  endRevision: number,
 ): boolean {
   if (provision.length !== 3 || typeof provision[1] !== "string")
     throw new Error("Invalid arguments");
@@ -260,7 +241,7 @@ export function instances(
     const unpacked = device.unpack(
       sessionContext.deviceData,
       path,
-      startRevision + 1
+      startRevision + 1,
     );
     count = Math.max(0, unpacked.length + count);
   }

lib/device.ts

@@ -1,30 +1,11 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
+import Path from "./common/path.ts";
 import {
   DeviceData,
   Declaration,
   Clear,
   AttributeTimestamps,
   Attributes,
-} from "./types";
+} from "./types.ts";
 
 const CHANGE_FLAGS = {
   object: 2,
@@ -43,7 +24,7 @@ function parseBool(v): boolean {
 }
 
 export function sanitizeParameterValue(
-  parameterValue: [string | number | boolean, string]
+  parameterValue: [string | number | boolean, string],
 ): [string | number | boolean, string] {
   if (parameterValue[0] != null) {
     switch (parameterValue[1]) {
@@ -87,7 +68,7 @@ export function sanitizeParameterValue(
 export function getAliasDeclarations(
   path: Path,
   timestamp: number,
-  attrGet = null
+  attrGet = null,
 ): Declaration[] {
   const stripped = path.stripAlias();
   let decs: Declaration[] = [
@@ -109,7 +90,7 @@ export function getAliasDeclarations(
           decs = decs.concat(
             getAliasDeclarations(parent.concat(p), timestamp, {
               value: timestamp,
-            })
+            }),
           );
         }
       }
@@ -122,7 +103,7 @@ export function getAliasDeclarations(
 export function unpack(
   deviceData: DeviceData,
   path: Path,
-  revision?: number
+  revision?: number,
 ): Path[] {
   let allMatches = [] as Path[];
   if (!path.alias) {
@@ -194,7 +175,7 @@ export function clear(
   path: Path,
   timestamp: number,
   attributes: AttributeTimestamps,
-  changeFlags = 0
+  changeFlags = 0,
 ): void {
   const changeTrackers = {};
 
@@ -212,7 +193,7 @@ export function clear(
     path,
     true,
     true,
-    descendantsTimestamp ? 99 : path.length
+    descendantsTimestamp ? 99 : path.length,
   )) {
     const tracker = deviceData.trackers.get(p);
     for (const k in tracker) changeTrackers[k] |= tracker[k];
@@ -274,7 +255,7 @@ export function set(
   path: Path,
   timestamp: number,
   attributes: Attributes,
-  toClear?: Clear[]
+  toClear?: Clear[],
 ): Clear[] {
   path = deviceData.paths.add(path);
 
@@ -315,7 +296,7 @@ export function set(
         else if (
           !compareEquality(
             attributes[attrName][1],
-            currentAttributes[attrName][1]
+            currentAttributes[attrName][1],
           )
         )
           changeFlags |= CHANGE_FLAGS[attrName];
@@ -334,7 +315,7 @@ export function set(
           path.slice(0, path.length - 1),
           timestamp,
           { object: [timestamp, 1] },
-          toClear
+          toClear,
         );
       }
     }
@@ -377,7 +358,7 @@ export function track(
   deviceData: DeviceData,
   path: Path,
   marker: string,
-  attributes?: string[]
+  attributes?: string[],
 ): void {
   path = deviceData.paths.add(path);
   let f = 1;
@@ -396,7 +377,7 @@ export function track(
 
 export function clearTrackers(
   deviceData: DeviceData,
-  tracker: string | string[]
+  tracker: string | string[],
 ): void {
   if (Array.isArray(tracker)) {
     for (const v of deviceData.trackers.values())

lib/extensions.ts

@@ -1,29 +1,10 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { spawn, ChildProcess } from "child_process";
-import * as crypto from "crypto";
-import * as config from "./config";
-import { Fault } from "./types";
-import { ROOT_DIR } from "./config";
-import * as logger from "./logger";
-import readline from "readline";
+import { spawn, ChildProcess } from "node:child_process";
+import * as crypto from "node:crypto";
+import readline from "node:readline";
+import * as config from "./config.ts";
+import { Fault } from "./types.ts";
+import { ROOT_DIR } from "./config.ts";
+import * as logger from "./logger.ts";
 
 const TIMEOUT = +config.get("EXT_TIMEOUT");
 
@@ -123,6 +104,6 @@ export async function killAll(): Promise<void> {
     Object.entries(processes).map(([k, p]) => {
       delete processes[k];
       return kill(p);
-    })
+    }),
   );
 }

lib/forwarded.ts

@@ -1,26 +1,8 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { IncomingMessage } from "http";
-import { TLSSocket } from "tls";
+import { IncomingMessage } from "node:http";
+import { TLSSocket } from "node:tls";
 import { parseCIDR, parse, IPv6, IPv4 } from "ipaddr.js";
-import * as config from "./config";
+import * as config from "./config.ts";
+import { getSocketEndpoints } from "./server.ts";
 
 interface RequestOrigin {
   localAddress: string;
@@ -98,19 +80,22 @@ export function getRequestOrigin(request: IncomingMessage): RequestOrigin {
   let origin = cache.get(request);
   if (!origin) {
     const soc = request.socket;
+    const socketEndpoints = getSocketEndpoints(soc);
     origin = {
-      localAddress: soc.localAddress,
-      localPort: soc.localPort,
-      remoteAddress: soc.remoteAddress,
-      remotePort: soc.remotePort,
+      localAddress: socketEndpoints.localAddress,
+      localPort: socketEndpoints.localPort,
+      remoteAddress: socketEndpoints.remoteAddress,
+      remotePort: socketEndpoints.remotePort,
       host: request.headers["host"],
       encrypted: !!(request.socket as TLSSocket).encrypted,
     };
 
     const header = request.headers["forwarded"];
     if (header) {
-      const ip = parse(soc.remoteAddress) as IPv4;
-      if (cidrs.some((cidr) => ip.match(cidr as [IPv4, number]))) {
+      const ip = parse(socketEndpoints.remoteAddress);
+      if (
+        cidrs.some((cidr) => ip.kind() === cidr[0].kind() && ip.match(cidr))
+      ) {
         const parsed = parseForwardedHeader(header);
 
         if (parsed["proto"] === "https") {

lib/fs.ts

@@ -1,63 +1,90 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as url from "url";
-import { IncomingMessage, ServerResponse } from "http";
-import { Collection, GridFSBucket } from "mongodb";
-import { onConnect } from "./db";
-import * as logger from "./logger";
-import { getRequestOrigin } from "./forwarded";
-import memoize from "./common/memoize";
-
-let filesCollection: Collection;
-let filesBucket: GridFSBucket;
-
-onConnect(async (db) => {
-  filesCollection = db.collection("fs.files");
-  filesBucket = new GridFSBucket(db);
-});
+import * as url from "node:url";
+import { IncomingMessage, ServerResponse } from "node:http";
+import { PassThrough, pipeline, Readable } from "node:stream";
+import { createHash } from "node:crypto";
+import { filesBucket, collections } from "./db/db.ts";
+import * as logger from "./logger.ts";
+import { getRequestOrigin } from "./forwarded.ts";
+import memoize from "./common/memoize.ts";
 
 const getFile = memoize(
-  (uploadDate: number, size: number, filename: string): Promise<Buffer> => {
-    return new Promise((resolve, reject) => {
-      const buffer = Buffer.allocUnsafe(size);
-      let i = 0;
-      const downloadStream = filesBucket.openDownloadStreamByName(filename);
-      downloadStream.on("error", reject);
-      downloadStream.on("data", (data: Buffer) => {
-        data.copy(buffer, i);
-        i += data.length;
-      });
-      downloadStream.on("end", () => {
-        if (i !== size) reject(new Error("File size mismatch"));
-        else resolve(buffer);
-      });
-    });
-  }
+  async (
+    etag: string,
+    size: number,
+    filename: string,
+  ): Promise<Iterable<Buffer>> => {
+    const chunks: Buffer[] = [];
+    // Using for-await over the download stream can throw ERR_STREAM_PREMATURE_CLOSE
+    // for very small files. Possibly a bug in MongoDB driver or Nodejs itself.
+    // Using a PassThrough stream to avoid this.
+    const downloadStream = pipeline(
+      filesBucket.openDownloadStreamByName(filename),
+      new PassThrough(),
+      (err) => {
+        if (err) throw err;
+      },
+    );
+    for await (const chunk of downloadStream) chunks.push(chunk);
+    // Node 12-14 don't throw error when stream is closed prematurely.
+    // However, we don't need to check for that since we're checking file size.
+    if (size !== chunks.reduce((a, b) => a + b.length, 0))
+      throw new Error("File size mismatch");
+    return chunks;
+  },
 );
 
+async function* partialContent(
+  chunks: Iterable<Buffer>,
+  start: number,
+  end: number,
+): AsyncIterable<Buffer> {
+  let bytesToSkip = start;
+  let bytesToRead = end - start;
+
+  for (let chunk of chunks) {
+    if (bytesToRead <= 0) return;
+    if (bytesToSkip >= chunk.length) {
+      bytesToSkip -= chunk.length;
+      continue;
+    }
+    chunk = chunk.subarray(bytesToSkip, bytesToSkip + bytesToRead);
+    bytesToRead -= chunk.length;
+    bytesToSkip = 0;
+    yield chunk;
+  }
+}
+
+function generateETag(file: {
+  _id: string;
+  uploadDate: Date;
+  length: number;
+}): string {
+  const hash = createHash("md5");
+  hash.update(`${file._id}-${file.uploadDate.getTime()}-${file.length}`);
+  return hash.digest("hex");
+}
+
+function matchEtag(etag: string, header: string): boolean {
+  for (let t of header.split(",")) {
+    t = t.trim();
+    if (t.startsWith("W/")) t = t.substring(2);
+    try {
+      t = JSON.parse(t);
+    } catch (e) {
+      // Ignore
+    }
+    if (t === "*") return true;
+    if (etag === t) return true;
+  }
+  return false;
+}
+
 export async function listener(
   request: IncomingMessage,
-  response: ServerResponse
+  response: ServerResponse,
 ): Promise<void> {
-  if (request.method !== "GET") {
-    response.writeHead(405, { Allow: "GET" });
+  if (request.method !== "GET" && request.method !== "HEAD") {
+    response.writeHead(405, { Allow: "GET, HEAD" });
     response.end("405 Method Not Allowed");
     return;
   }
@@ -69,9 +96,10 @@ export async function listener(
     message: "Fetch file",
     filename: filename,
     remoteAddress: getRequestOrigin(request).remoteAddress,
+    method: request.method,
   };
 
-  const file = await filesCollection.findOne({ _id: filename });
+  const file = await collections.files.findOne({ _id: filename });
 
   if (!file) {
     response.writeHead(404);
@@ -81,17 +109,95 @@ export async function listener(
     return;
   }
 
-  const buffer = await getFile(
-    file["uploadDate"].getTime(),
-    file.length,
-    filename
-  );
+  logger.accessInfo(log);
 
-  response.writeHead(200, {
+  const etag = generateETag(file);
+  const lastModified = file["uploadDate"];
+  lastModified.setMilliseconds(0);
+
+  let status = 200;
+  let start = 0;
+  let end = file.length;
+
+  if (request.headers["if-match"])
+    if (!matchEtag(etag, request.headers["if-match"])) status = 412;
+
+  if (request.headers["if-unmodified-since"]) {
+    const d = new Date(request.headers["if-unmodified-since"]);
+    if (lastModified > d) status = 412;
+  }
+
+  if (request.headers["if-none-match"]) {
+    if (matchEtag(etag, request.headers["if-none-match"])) status = 304;
+  } else if (request.headers["if-modified-since"]) {
+    const d = new Date(request.headers["if-modified-since"]);
+    if (lastModified <= d) status = 304;
+  }
+
+  if (request.headers.range && status === 200) {
+    const match = request.headers.range.match(/^bytes=(\d*)-(\d*)$/);
+    status = 416;
+    if (match && (match[1] || match[2])) {
+      if (match[2]) end = parseInt(match[2]) + 1;
+      if (match[1]) start = parseInt(match[1]);
+      else start = file.length - parseInt(match[2]);
+      if (start < end && end <= file.length) status = 206;
+    }
+
+    if (request.headers["if-range"]) {
+      const h = request.headers["if-range"] as string;
+      const d = new Date(h);
+      if (!matchEtag(etag, h) && !(lastModified <= d)) {
+        status = 200;
+        start = 0;
+        end = file.length;
+      }
+    }
+  }
+
+  if (status === 412) {
+    response.writeHead(412);
+    response.end();
+    return;
+  }
+
+  if (status === 304) {
+    response.writeHead(304, {
+      ETag: etag,
+      "Last-Modified": lastModified.toUTCString(),
+    });
+    response.end();
+    return;
+  }
+
+  if (status === 416) {
+    response.writeHead(416, {
+      "Content-Range": `bytes */${file.length}`,
+      "Content-Length": "0",
+    });
+    response.end();
+    return;
+  }
+
+  response.writeHead(status, {
     "Content-Type": "application/octet-stream",
-    "Content-Length": file.length,
+    "Content-Length": end - start,
+    "Accept-Ranges": "bytes",
+    ETag: etag,
+    "Last-Modified": lastModified.toUTCString(),
+    ...(status === 206 && {
+      "Content-Range": `bytes ${start}-${end - 1}/${file.length}`,
+    }),
   });
 
-  response.end(buffer);
-  logger.accessInfo(log);
+  if (request.method === "HEAD") {
+    response.end();
+    return;
+  }
+
+  const chunks = await getFile(etag, file.length, filename);
+
+  pipeline(Readable.from(partialContent(chunks, start, end)), response, () => {
+    // Ignore errors resulting from client disconnecting
+  });
 }

lib/gpn-heuristic.ts

@@ -1,23 +1,4 @@
-import Path from "./common/path";
-
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
+import Path from "./common/path.ts";
 
 const WILDCARD_MULTIPLIER = 2;
 const UNDISCOVERED_DEPTH = 7;
@@ -31,7 +12,7 @@ const UNDISCOVERED_DEPTH = 7;
 // point has been discovered.
 export function estimateGpnCount(
   gpnPatterns: [Path, number][],
-  depth = 0
+  depth = 0,
 ): number {
   const children: { [segment: string]: [Path, number][] } = {};
   const wildcardChildren: [Path, number][] = [];

lib/init.ts

@@ -1,26 +1,14 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as localCache from "./local-cache";
-import { generateSalt, hashPassword } from "./auth";
-import * as db from "./ui/db";
-import { del } from "./cache";
+import { getRevision, getUiConfig, getUsers } from "./ui/local-cache.ts";
+import { generateSalt, hashPassword } from "./auth.ts";
+import { collections } from "./db/db.ts";
+import {
+  putConfig,
+  putPermission,
+  putPreset,
+  putProvision,
+  putUser,
+} from "./ui/db.ts";
+import { del } from "./cache.ts";
 
 interface Status {
   users: boolean;
@@ -84,14 +72,16 @@ declare("Device.ManagementServer.PeriodicInformTime", {value: daily}, {value: in
 `.trim();
 
 export async function getStatus(): Promise<Status> {
-  const configSnapshot = await localCache.getCurrentSnapshot();
-  const users = localCache.getUsers(configSnapshot);
-  const presets = localCache.getPresets(configSnapshot);
-  const ui = localCache.getUiConfig(configSnapshot);
+  const [configSnapshot, presetCount] = await Promise.all([
+    getRevision(),
+    collections.presets.countDocuments(),
+  ]);
+  const users = getUsers(configSnapshot);
+  const ui = getUiConfig(configSnapshot);
 
   return {
     users: !Object.keys(users).length,
-    presets: !presets.length,
+    presets: !presetCount,
     filters: !Object.keys(ui["filters"]).length,
     device: !Object.keys(ui["device"]).length,
     index: !Object.keys(ui["index"]).length,
@@ -395,7 +385,7 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
   if (resources["permissions"]) {
     for (const p of resources["permissions"]) {
       p["_id"] = `${p["role"]}:${p["resource"]}:${p["access"]}`;
-      proms.push(db.putPermission(p["_id"], p));
+      proms.push(putPermission(p["_id"], p));
     }
   }
 
@@ -406,21 +396,21 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
       u["roles"] = (u["roles"] || []).join(",");
       u["_id"] = u["username"];
       delete u["username"];
-      proms.push(db.putUser(u["_id"], u));
+      proms.push(putUser(u["_id"], u));
     }
   }
 
   if (resources["provisions"]) {
     for (const p of resources["provisions"])
-      proms.push(db.putProvision(p["_id"], p));
+      proms.push(putProvision(p["_id"], p));
   }
 
   if (resources["presets"])
-    for (const p of resources["presets"]) proms.push(db.putPreset(p["_id"], p));
+    for (const p of resources["presets"]) proms.push(putPreset(p["_id"], p));
 
   if (resources["config"])
-    for (const c of resources["config"]) proms.push(db.putConfig(c["_id"], c));
+    for (const c of resources["config"]) proms.push(putConfig(c["_id"], c));
 
   await proms;
-  return del("presets_hash");
+  await Promise.all([del("ui-local-cache-hash"), del("cwmp-local-cache-hash")]);
 }

lib/instance-set.ts

@@ -1,28 +1,9 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 interface Instance {
   [name: string]: string;
 }
 
 export default class InstanceSet {
-  private declare set: Set<Instance>;
+  declare private set: Set<Instance>;
 
   public constructor() {
     this.set = new Set();

lib/local-cache.ts

@@ -1,543 +1,73 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as vm from "vm";
-import * as crypto from "crypto";
-import * as config from "./config";
-import * as db from "./db";
-import * as cache from "./cache";
-import { convertOldPrecondition } from "./mongodb-functions";
-import * as logger from "./logger";
-import * as scheduling from "./scheduling";
-import * as expression from "./common/expression";
-import { parse } from "./common/expression-parser";
-import {
-  Preset,
-  Expression,
-  Provisions,
-  VirtualParameters,
-  Files,
-  Users,
-  Permissions,
-  Config,
-  UiConfig,
-} from "./types";
-
-interface Snapshot {
-  presets: Preset[];
-  provisions: Provisions;
-  virtualParameters: VirtualParameters;
-  files: Files;
-  permissions: Permissions;
-  users: Users;
-  config: Config;
-  ui: UiConfig;
-}
+import { setTimeoutPromise } from "./util.ts";
+import { get, set } from "./cache.ts";
+import { acquireLock, releaseLock } from "./lock.ts";
 
 const REFRESH = 5000;
 const EVICT_TIMEOUT = 120000;
 
-const snapshots = new Map<string, Snapshot>();
-let currentSnapshot: string = null;
-let nextRefresh = 1;
+export class LocalCache<T> {
+  private nextRefresh = 1;
+  private currentRevision: string = null;
+  private snapshots: Map<string, T> = new Map();
 
-function computeHash(snapshot): string {
-  // MD5 hash for presets, provisions, virtual parameters for detecting changes
-  const h = crypto.createHash("md5");
-  for (const p of snapshot.presets) {
-    h.update(JSON.stringify(p.name));
-    h.update(JSON.stringify(p.channel));
-    h.update(JSON.stringify(p.schedule));
-    h.update(JSON.stringify(p.events));
-    h.update(JSON.stringify(p.precondition));
-    h.update(JSON.stringify(p.provisions));
+  constructor(
+    private cacheKey: string,
+    private callback: () => Promise<[string, T]>,
+  ) {}
+
+  async getRevision(): Promise<string> {
+    if (Date.now() > this.nextRefresh) await this.refresh();
+    return this.currentRevision;
   }
 
-  let keys;
-
-  keys = Object.keys(snapshot.provisions).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(snapshot.provisions[k].md5);
-
-  keys = Object.keys(snapshot.virtualParameters).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(snapshot.virtualParameters[k].md5);
-
-  keys = Object.keys(snapshot.config).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.config[k]));
-
-  keys = Object.keys(snapshot.files).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.files[k]));
-
-  keys = Object.keys(snapshot.users).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.users[k]));
-
-  const roles = Object.keys(snapshot.permissions).sort();
-  h.update(JSON.stringify(roles));
-  for (const r of roles) {
-    const levels = Object.keys(snapshot.permissions[r]).sort();
-    h.update(JSON.stringify(levels));
-    for (const l of levels) {
-      keys = Object.keys(snapshot.permissions[r][l]).sort();
-      h.update(JSON.stringify(keys));
-      for (const k of keys)
-        h.update(JSON.stringify(snapshot.permissions[r][l][k]));
-    }
+  hasRevision(revision: string): boolean {
+    return this.snapshots.has(revision);
   }
 
-  return h.digest("hex");
-}
-
-function flattenObject(
-  src: Record<string, unknown>,
-  prefix = "",
-  dst = {}
-): Record<string, unknown> {
-  for (const k of Object.keys(src)) {
-    const v = src[k];
-    if (typeof v === "object" && !Array.isArray(v))
-      flattenObject(v as Record<string, unknown>, `${prefix}${k}.`, dst);
-    else dst[`${prefix}${k}`] = v;
+  get(revision: string): T {
+    const snapshot = this.snapshots.get(revision);
+    if (!snapshot) throw new Error("Cache snapshot does not exist");
+    return snapshot;
   }
-  return dst;
-}
 
-async function fetchPresets(): Promise<Preset[]> {
-  const res = await db.getPresets();
-  let objects = await db.getObjects();
-
-  objects = objects.map((obj) => {
-    // Flatten object
-    obj = flattenObject(obj);
-
-    // If no keys are defined, consider all parameters as keys to keep the
-    // same behavior from v1.0
-    if (!obj["_keys"]?.length)
-      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
-
-    return obj;
-  });
-
-  res.sort((a, b) => {
-    if (a["weight"] === b["weight"])
-      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
-    else return a["weight"] - b["weight"];
-  });
-
-  const presets = [] as Preset[];
-  for (const preset of res) {
-    let schedule: { md5: string; duration: number; schedule: any } = null;
-    if (preset["schedule"]) {
-      const parts = preset["schedule"].trim().split(/\s+/);
-      schedule = {
-        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
-        duration: null,
-        schedule: null,
-      };
-
-      try {
-        schedule.duration = +parts.shift() * 1000;
-        schedule.schedule = scheduling.parseCron(parts.join(" "));
-      } catch (err) {
-        logger.warn({
-          message: "Invalid preset schedule",
-          preset: preset["_id"],
-          schedule: preset["schedule"],
-        });
-        schedule.schedule = false;
-      }
+  async refresh(): Promise<void> {
+    if (!this.nextRefresh) {
+      await setTimeoutPromise(20);
+      await this.refresh();
+      return;
     }
 
-    const events = preset["events"] || {};
-    let precondition = true as Expression;
-    if (preset["precondition"]) {
-      try {
-        precondition = parse(preset["precondition"]);
-      } catch (error) {
-        precondition = convertOldPrecondition(
-          JSON.parse(preset["precondition"])
-        );
-      }
+    const now = Date.now();
+    if (now < this.nextRefresh) return;
+    this.nextRefresh = 0;
 
-      // Simplify expression
-      precondition = expression.evaluate(precondition);
+    const dbHash = await get(this.cacheKey);
+
+    if (this.currentRevision && dbHash === this.currentRevision) {
+      this.nextRefresh = now + (REFRESH - (now % REFRESH));
+      return;
     }
 
-    const _provisions = preset["provisions"] || [];
+    const lockToken = await acquireLock(this.cacheKey, 5000);
 
-    // Generate provisions from the old configuration format
-    for (const c of preset["configurations"]) {
-      switch (c.type) {
-        case "age":
-          _provisions.push(["refresh", c.name, +c.age]);
-          break;
+    const [hash, snapshot] = await this.callback();
 
-        case "value":
-          _provisions.push(["value", c.name, c.value]);
-          break;
-
-        case "add_tag":
-          _provisions.push(["tag", c.tag, true]);
-          break;
-
-        case "delete_tag":
-          _provisions.push(["tag", c.tag, false]);
-          break;
-
-        case "provision":
-          _provisions.push([c.name].concat(c.args || []));
-          break;
-
-        case "add_object":
-          for (const obj of objects) {
-            if (obj["_id"] === c.object) {
-              const alias = obj["_keys"]
-                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
-                .join(",");
-              const p = `${c.name}.[${alias}]`;
-              _provisions.push(["instances", p, 1]);
-
-              for (const k in obj) {
-                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
-                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
-              }
-            }
-          }
-
-          break;
-
-        case "delete_object":
-          for (const obj of objects) {
-            if (obj["_id"] === c.object) {
-              const alias = obj["_keys"]
-                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
-                .join(",");
-              const p = `${c.name}.[${alias}]`;
-              _provisions.push(["instances", p, 0]);
-            }
-          }
-
-          break;
-
-        default:
-          throw new Error(`Unknown configuration type ${c.type}`);
-      }
+    if (this.currentRevision) {
+      const r = this.currentRevision;
+      const s = this.snapshots.get(r);
+      setTimeout(() => {
+        if (this.snapshots.get(r) === s) this.snapshots.delete(r);
+      }, EVICT_TIMEOUT).unref();
     }
 
-    presets.push({
-      name: preset["_id"],
-      channel: (preset["channel"] as string) || "default",
-      schedule: schedule,
-      events: events,
-      precondition: precondition,
-      provisions: _provisions,
-    });
-  }
+    this.currentRevision = hash;
+    this.snapshots.set(hash, snapshot);
 
-  return presets;
-}
-
-async function fetchProvisions(): Promise<Provisions> {
-  const res = await db.getProvisions();
-
-  const provisions = {};
-  for (const r of res) {
-    provisions[r["_id"]] = {};
-    provisions[r["_id"]].md5 = crypto
-      .createHash("md5")
-      .update(r["script"])
-      .digest("hex");
-    provisions[r["_id"]].script = new vm.Script(
-      `"use strict";(function(){\n${r["script"]}\n})();`,
-      { filename: r["_id"], lineOffset: -1, timeout: 50 }
-    );
-  }
-
-  return provisions;
-}
-
-async function fetchVirtualParameters(): Promise<VirtualParameters> {
-  const res = await db.getVirtualParameters();
-
-  const virtualParameters = {};
-  for (const r of res) {
-    virtualParameters[r["_id"]] = {};
-    virtualParameters[r["_id"]].md5 = crypto
-      .createHash("md5")
-      .update(r["script"])
-      .digest("hex");
-    virtualParameters[r["_id"]].script = new vm.Script(
-      `"use strict";(function(){\n${r["script"]}\n})();`,
-      { filename: r["_id"], lineOffset: -1, timeout: 50 }
-    );
-  }
-
-  return virtualParameters;
-}
-
-async function fetchPermissions(): Promise<Permissions> {
-  const perms = await db.getPermissions();
-  const permissions: Permissions = {};
-
-  for (const p of perms) {
-    if (!permissions[p.role]) permissions[p.role] = {};
-    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
-
-    permissions[p.role][p.access][p.resource] = {
-      access: p.access,
-      filter: parse(p.filter || "true"),
-    };
-    if (p.validate)
-      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
-  }
-
-  return permissions;
-}
-
-async function fetchFiles(): Promise<Files> {
-  const res = await db.getFiles();
-  const files = {};
-
-  for (const r of res) {
-    const id = r["filename"] || r["_id"].toString();
-    files[id] = {};
-    files[id].length = r["length"];
-  }
-
-  return files;
-}
-
-async function fetchUsers(): Promise<Users> {
-  const _users = await db.getUsers();
-  const users = {};
-
-  for (const user of _users) {
-    users[user._id] = {
-      password: user.password,
-      salt: user.salt,
-      roles: user.roles.split(",").map((s) => s.trim()),
-    };
-  }
-
-  return users;
-}
-
-async function fetchConfig(): Promise<[Config, UiConfig]> {
-  const conf = await db.getConfig();
-
-  conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
-
-  const ui = {
-    filters: {},
-    device: {},
-    index: {},
-    overview: {},
-  };
-  const _config = {};
-
-  for (const c of conf) {
-    // Evaluate expressions to simplify them
-    const val = expression.evaluate(c.value);
-    _config[c.id] = val;
-    if (c.id.startsWith("ui.")) {
-      const keys = c.id.split(".");
-      if (!(keys[1] in ui)) continue;
-      // remove the first key(ui)
-      keys.shift();
-      let ref = ui;
-      while (keys.length > 1) {
-        const k = keys.shift();
-        if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
-        ref = ref[k];
-      }
-      ref[keys[0]] = val;
+    if (lockToken) {
+      if (hash !== dbHash) await set(this.cacheKey, hash, 300);
+      await releaseLock(this.cacheKey, lockToken);
     }
+
+    this.nextRefresh = now + (REFRESH - (now % REFRESH));
   }
-
-  return [_config, ui];
-}
-
-async function refresh(): Promise<void> {
-  if (!nextRefresh) {
-    await new Promise((resolve) => setTimeout(resolve, 20));
-    await refresh();
-    return;
-  }
-
-  nextRefresh = 0;
-  const now = Date.now();
-
-  const dbHash = await cache.get("presets_hash");
-
-  if (currentSnapshot && dbHash === currentSnapshot) {
-    nextRefresh = now + (REFRESH - (now % REFRESH));
-    return;
-  }
-
-  const lockToken = await cache.acquireLock("presets_hash_lock", 5000, 5000);
-
-  const res = await Promise.all([
-    fetchPresets(),
-    fetchProvisions(),
-    fetchVirtualParameters(),
-    fetchFiles(),
-    fetchPermissions(),
-    fetchUsers(),
-    fetchConfig(),
-  ]);
-
-  const snapshot = {
-    presets: res[0],
-    provisions: res[1],
-    virtualParameters: res[2],
-    files: res[3],
-    permissions: res[4],
-    users: res[5],
-    config: res[6][0],
-    ui: res[6][1],
-  };
-
-  if (currentSnapshot) {
-    const h = currentSnapshot;
-    const s = snapshots.get(h);
-    setTimeout(() => {
-      if (snapshots.get(h) === s) snapshots.delete(h);
-    }, EVICT_TIMEOUT).unref();
-  }
-
-  currentSnapshot = computeHash(snapshot);
-  snapshots.set(currentSnapshot, snapshot);
-  await cache.set("presets_hash", currentSnapshot, 300);
-  await cache.releaseLock("presets_hash_lock", lockToken);
-
-  nextRefresh = now + (REFRESH - (now % REFRESH));
-}
-
-export async function getCurrentSnapshot(): Promise<string> {
-  if (Date.now() > nextRefresh) await refresh();
-  return currentSnapshot;
-}
-
-export function hasSnapshot(hash: string): boolean {
-  return snapshots.has(hash);
-}
-
-export function getPresets(snapshotKey: string): Preset[] {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.presets;
-}
-
-export function getProvisions(snapshotKey: string): Provisions {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.provisions;
-}
-
-export function getVirtualParameters(snapshotKey: string): VirtualParameters {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.virtualParameters;
-}
-
-export function getFiles(snapshotKey: string): Files {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.files;
-}
-
-export function getConfig(
-  snapshotKey: string,
-  key: string,
-  context: Record<string, unknown>,
-  now: number,
-  cb?: (e: Expression) => Expression
-): string | number | boolean | null {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  const oldOpts = {
-    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
-    "cwmp.debug": "DEBUG",
-    "cwmp.retryDelay": "RETRY_DELAY",
-    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
-    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
-    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
-    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
-    "cwmp.cookiesPath": "COOKIES_PATH",
-    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
-    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
-    "cwmp.connectionRequestAllowBasicAuth":
-      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
-    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
-    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
-    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
-  };
-
-  if (!(key in snapshot.config)) {
-    if (key in oldOpts) {
-      let id;
-      if (context?.["id"]) {
-        id = context["id"];
-      } else if (cb) {
-        id = cb(["PARAM", "DeviceID.ID"]);
-        if (Array.isArray(id)) id = null;
-      }
-      return config.get(oldOpts[key], id);
-    }
-    return null;
-  }
-
-  const v = expression.evaluate(snapshot.config[key], context, now, cb);
-  return Array.isArray(v) ? null : v;
-}
-
-export function getConfigExpression(
-  snapshotKey: string,
-  key: string
-): Expression {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.config[key];
-}
-
-export function getUsers(snapshotKey: string): Users {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.users;
-}
-
-export function getPermissions(snapshotKey: string): Permissions {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.permissions;
-}
-
-export function getUiConfig(snapshotKey: string): UiConfig {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.ui;
 }

lib/lock.ts

@@ -0,0 +1,50 @@
+import { collections } from "./db/db.ts";
+
+const CLOCK_SKEW_TOLERANCE = 30000;
+
+export async function acquireLock(
+  lockName: string,
+  ttl: number,
+  timeout = 0,
+  token = Math.random().toString(36).slice(2),
+): Promise<string> {
+  try {
+    const now = Date.now();
+    const r = await collections.locks.findOneAndUpdate(
+      { _id: lockName, value: token },
+      {
+        $set: {
+          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
+        },
+        $currentDate: { timestamp: true },
+      },
+      { upsert: true, returnDocument: "after" },
+    );
+    if (Math.abs(r.value.timestamp.getTime() - now) > CLOCK_SKEW_TOLERANCE)
+      throw new Error("Database clock skew too great");
+  } catch (err) {
+    if (err.code !== 11000) throw err;
+    if (!(timeout > 0)) return null;
+    const w = 50 + Math.random() * 50;
+    await new Promise((resolve) => setTimeout(resolve, w));
+    return acquireLock(lockName, ttl, timeout - w, token);
+  }
+
+  return token;
+}
+
+export async function releaseLock(
+  lockName: string,
+  token: string,
+): Promise<void> {
+  const res = await collections.locks.deleteOne({
+    _id: lockName,
+    value: token,
+  });
+  if (res.deletedCount !== 1) throw new Error("Lock expired");
+}
+
+export async function getToken(lockName: string): Promise<string> {
+  const res = await collections.locks.findOne({ _id: lockName });
+  return res?.value;
+}

lib/logger.ts

@@ -1,27 +1,7 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as fs from "fs";
-import * as os from "os";
-
-import * as config from "./config";
-import { getRequestOrigin } from "./forwarded";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as config from "./config.ts";
+import { getRequestOrigin } from "./forwarded.ts";
 import {
   SessionContext,
   AcsRequest,
@@ -29,7 +9,7 @@ import {
   CpeFault,
   InformRequest,
   Fault,
-} from "./types";
+} from "./types.ts";
 
 const REOPEN_EVERY = 60000;
 
@@ -156,13 +136,13 @@ export function close(): void {
 }
 
 export function flatten(
-  details: Record<string, unknown>
+  details: Record<string, unknown>,
 ): Record<string, unknown> {
   if (details.sessionContext) {
     const sessionContext = details.sessionContext as SessionContext;
     details.deviceId = sessionContext.deviceId;
     details.remoteAddress = getRequestOrigin(
-      sessionContext.httpRequest
+      sessionContext.httpRequest,
     ).remoteAddress;
     delete details.sessionContext;
   }
@@ -220,7 +200,7 @@ export function flatten(
   // For genieacs-ui
   if (details.context) {
     details.remoteAddress = getRequestOrigin(
-      details.context["req"]
+      details.context["req"],
     ).remoteAddress;
     if (details.context["state"].user)
       details.user = details.context["state"].user.username;
@@ -235,7 +215,7 @@ export function flatten(
 
 function formatJson(
   details: Record<string, unknown>,
-  systemd: boolean
+  systemd: boolean,
 ): string {
   if (systemd) {
     let severity = "";
@@ -251,7 +231,7 @@ function formatJson(
 
 function formatSimple(
   details: Record<string, unknown>,
-  systemd: boolean
+  systemd: boolean,
 ): string {
   const skip = {
     user: true,

lib/mongodb-functions.ts

@@ -1,684 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ObjectId } from "mongodb";
-import { map, parse, stringify, parseList } from "./common/expression-parser";
-import { likePatternToRegExp } from "./common/expression";
-import { Expression, Fault, Task } from "./types";
-import { decodeTag, encodeTag } from "./common";
-
-const isArray = Array.isArray;
-
-export function processDeviceFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
-    if (!isArray(exp)) return exp;
-
-    if (exp[0] === "PARAM") {
-      const p = exp[1];
-      if (p === "DeviceID.ID") return ["PARAM", "_id"];
-      else if (p === "DeviceID") return ["PARAM", "_deviceId"];
-      else if (p.startsWith("DeviceID."))
-        return ["PARAM", "_deviceId._" + p.slice(9)];
-      else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
-      else if (p === "Events.Registered") return ["PARAM", "_registered"];
-      else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
-      else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"];
-    } else if (
-      isArray(exp[1]) &&
-      exp[1][0] === "PARAM" &&
-      exp[1][1].startsWith("Tags.")
-    ) {
-      const t = decodeTag(exp[1][1].slice(5));
-      if (exp[0] === "IS NULL") return ["_tags", t, false];
-      else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
-      else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
-      else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
-    } else if (
-      [
-        "=",
-        "<>",
-        ">",
-        ">=",
-        "<",
-        "<=",
-        "LIKE",
-        "NOT LIKE",
-        "IS NULL",
-        "IS NOT NULL",
-      ].includes(exp[0])
-    ) {
-      let e = map(exp, (ee) => {
-        if (
-          isArray(ee) &&
-          ee[0] === "PARAM" &&
-          typeof ee[1] === "string" &&
-          !ee[1].startsWith("_")
-        )
-          return ["PARAM", `${ee[1]}._value`];
-        return ee;
-      });
-
-      if (typeof e[2] === "number") {
-        const alt = (e as any[]).slice();
-        alt[2] = new Date(e[2]);
-        e = ["OR", e, alt];
-      }
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function processTasksFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
-    if (!isArray(exp)) return exp;
-    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
-      const e = exp.slice();
-      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectId(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
-        e[2] = new Date(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
-        e[2] = new Date(e[2]);
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function processFaultsFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
-    if (!isArray(exp)) return exp;
-    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
-      const e = exp.slice();
-      if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
-        e[2] = new Date(e[2]);
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
-  function recursive(filter, negate): Record<string, unknown> {
-    const op = filter[0];
-
-    if (op === "AND") {
-      filter = filter.filter((f) => f !== true);
-      if (filter.length === 2) return recursive(filter[1], negate);
-    }
-
-    if (op === "OR") {
-      filter = filter.filter((f) => f !== false);
-      if (filter.length === 2) return recursive(filter[1], negate);
-    }
-
-    if ((!negate && op === "AND") || (negate && op === "OR"))
-      return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
-
-    if ((!negate && op === "OR") || (negate && op === "AND"))
-      return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
-
-    if (op === "NOT") return recursive(filter[1], !negate);
-
-    if (isArray(filter[2]))
-      throw new Error(`Invalid RHS operand of ${op} clause`);
-
-    if (op === "LIKE" || op === "NOT LIKE") {
-      if (op === "NOT LIKE") negate = !negate;
-      let param;
-      let flags;
-      if (!Array.isArray(filter[1]))
-        throw new Error(`Invalid LHS operand of ${op} clause`);
-      if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
-        if (filter[2] !== filter[2].toUpperCase())
-          throw new Error(`Invalid RHS operand of ${op} clause`);
-        param = filter[1][2][1];
-        flags = "i";
-      } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
-        if (filter[2] !== filter[2].toLowerCase())
-          throw new Error(`Invalid RHS operand of ${op} clause`);
-        param = filter[1][2][1];
-        flags = "i";
-      } else if (filter[1][0] === "PARAM") {
-        param = filter[1][1];
-        flags = "";
-      } else {
-        throw new Error(`Invalid LHS operand of ${op} clause`);
-      }
-      const r = likePatternToRegExp(filter[2], filter[3], flags);
-      if (negate) return { [param]: { $nin: [r, null] } };
-      else return { [param]: r };
-    }
-
-    if (op === "_tags") {
-      let t = filter[2];
-      if (negate) t = !t;
-      if (t) return { _tags: filter[1] };
-      else return { _tags: { $ne: filter[1] } };
-    }
-
-    if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
-      throw new Error(`Invalid LHS operand of ${op} clause`);
-
-    if (op === "IS NULL") {
-      const p = filter[1][1];
-      return { [p]: null };
-    }
-
-    if (op === "IS NOT NULL") {
-      const p = filter[1][1];
-      return { [p]: { $ne: null } };
-    }
-
-    if (isArray(filter[2]) || filter[2] == null)
-      throw new Error(`Invalid RHS operand of ${op} clause`);
-
-    if (op === "=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (!negate) return { [p]: v };
-      else return { [p]: { $nin: [v, null] } };
-    }
-
-    if (op === "<>") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: v };
-      else return { [p]: { $nin: [v, null] } };
-    }
-
-    if (op === ">") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $lte: v } };
-      else return { [p]: { $gt: v } };
-    }
-
-    if (op === ">=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $lt: v } };
-      else return { [p]: { $gte: v } };
-    }
-
-    if (op === "<") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $gte: v } };
-      else return { [p]: { $lt: v } };
-    }
-
-    if (op === "<=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $gt: v } };
-      else return { [p]: { $lte: v } };
-    }
-
-    throw new Error(`Unrecognized operator ${op}`);
-  }
-
-  if (!isArray(exp)) {
-    if (exp === true) return {};
-    throw new Error("Primitives are not valid queries");
-  }
-
-  return recursive(exp, false);
-}
-
-export function processDeviceProjection(
-  projection: Record<string, 1>
-): Record<string, 1> {
-  if (!projection) return projection;
-  const p = {};
-  for (const [k, v] of Object.entries(projection)) {
-    if (k === "DeviceID.ID") {
-      p["_id"] = 1;
-    } else if (k.startsWith("DeviceID")) {
-      p["_deviceId._SerialNumber"] = v;
-      p["_deviceId._OUI"] = v;
-      p["_deviceId._ProductClass"] = v;
-      p["_deviceId._Manufacturer"] = v;
-    } else if (k.startsWith("Tags")) {
-      p["_tags"] = v;
-    } else if (k.startsWith("Events")) {
-      p["_lastInform"] = v;
-      p["_registered"] = v;
-      p["_lastBoot"] = v;
-      p["_lastBootstrap"] = v;
-    } else {
-      p[k] = v;
-    }
-  }
-
-  return p;
-}
-
-export function processDeviceSort(
-  sort: Record<string, number>
-): Record<string, number> {
-  if (!sort) return sort;
-  const s = {};
-  for (const [k, v] of Object.entries(sort)) {
-    if (k === "DeviceID.ID") s["_id"] = v;
-    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
-    else if (k === "Events.Inform") s["_lastInform"] = v;
-    else if (k === "Events.Registered") s["_registered"] = v;
-    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
-    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
-    else s[`${k}._value`] = v;
-  }
-
-  return s;
-}
-
-function parseDate(d: Date): number | string {
-  const n = +d;
-  return isNaN(n) ? "" + d : n;
-}
-
-interface FlatAttributes {
-  object?: boolean;
-  objectTimestamp?: number;
-  writable?: boolean;
-  writableTimestamp?: number;
-  value?: [string | number | boolean, string];
-  valueTimestamp?: number;
-  notification?: number;
-  notificationTimestamp?: number;
-  accessList?: string[];
-  accessListTimestamp?: number;
-}
-
-interface FlatDevice {
-  [param: string]: FlatAttributes;
-}
-
-export function flattenDevice(device: Record<string, unknown>): FlatDevice {
-  function recursive(
-    input,
-    root: string,
-    output: FlatDevice,
-    timestamp: number
-  ): void {
-    for (const [name, tree] of Object.entries(input)) {
-      if (!root) {
-        if (name === "_lastInform") {
-          output["Events.Inform"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_registered") {
-          output["Events.Registered"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_lastBoot") {
-          output["Events.1_BOOT"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_lastBootstrap") {
-          output["Events.0_BOOTSTRAP"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_id") {
-          output["DeviceID.ID"] = {
-            value: [tree as string, "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_deviceId") {
-          output["DeviceID.Manufacturer"] = {
-            value: [tree["_Manufacturer"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.OUI"] = {
-            value: [tree["_OUI"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.ProductClass"] = {
-            value: [tree["_ProductClass"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.SerialNumber"] = {
-            value: [tree["_SerialNumber"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_tags") {
-          output["Tags"] = {
-            writable: false,
-            writableTimestamp: timestamp,
-            object: true,
-            objectTimestamp: timestamp,
-          };
-
-          for (const t of tree as string[]) {
-            output[`Tags.${encodeTag(t)}`] = {
-              value: [true, "xsd:boolean"],
-              valueTimestamp: timestamp,
-              writable: true,
-              writableTimestamp: timestamp,
-              object: false,
-              objectTimestamp: timestamp,
-            };
-          }
-        }
-      }
-
-      if (name.startsWith("_")) continue;
-
-      let childrenTimestamp = timestamp;
-
-      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
-      else if (+input["_timestamp"] > timestamp)
-        childrenTimestamp = +input["_timestamp"];
-
-      const attrs: FlatAttributes = {};
-      if (tree["_value"] != null) {
-        attrs.value = [
-          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
-          tree["_type"],
-        ];
-        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
-        attrs.object = false;
-        attrs.objectTimestamp = childrenTimestamp;
-      } else if (tree["_object"] != null) {
-        attrs.object = tree["_object"];
-        attrs.objectTimestamp = childrenTimestamp;
-      }
-
-      if (tree["_writable"] != null) {
-        attrs.writable = tree["_writable"];
-        attrs.writableTimestamp = childrenTimestamp;
-      }
-
-      if (tree["_notification"] != null) {
-        attrs.notification = tree["_notification"];
-        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
-      }
-
-      if (tree["_accessList"] != null) {
-        attrs.accessList = tree["_accessList"];
-        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
-      }
-
-      const r = root ? `${root}.${name}` : name;
-      output[r] = attrs;
-
-      if (attrs.object || tree["object"] == null)
-        recursive(tree, r, output, childrenTimestamp);
-    }
-  }
-
-  const newDevice: FlatDevice = {};
-  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
-  recursive(device, "", newDevice, timestamp);
-  return newDevice;
-}
-
-export function flattenFault(fault: unknown): Fault {
-  const f = Object.assign({}, fault) as Fault;
-  if (f.timestamp) f.timestamp = +f.timestamp;
-  if (f["expiry"]) f["expiry"] = +f["expiry"];
-  return f as Fault;
-}
-
-export function flattenTask(task: unknown): Task {
-  const t = Object.assign({}, task) as Task;
-  t._id = "" + t._id;
-  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
-  if (t.expiry) t.expiry = +t.expiry;
-  return t;
-}
-
-export function convertOldPrecondition(
-  query: Record<string, unknown>
-): Expression {
-  function recursive(_query): Expression {
-    const expressions: Expression[] = [];
-    for (const [k, v] of Object.entries(_query)) {
-      if (k[0] === "$") {
-        if (k === "$and") {
-          const and: Expression = ["AND"];
-          for (const vv of Object.values(v)) and.push(recursive(vv));
-          expressions.push(and);
-        } else if (k === "$or") {
-          const or: Expression = ["OR"];
-          for (const vv of Object.values(v)) or.push(recursive(vv));
-          expressions.push(or);
-        } else {
-          throw new Error(`Operator ${k} not supported`);
-        }
-      } else if (k === "_tags") {
-        if (typeof v === "object") {
-          if (isArray(v)) throw new Error(`Invalid type`);
-          const conjs: Expression[] = [];
-          for (const [op, val] of Object.entries(v)) {
-            if (op === "$ne") {
-              if (typeof v["$ne"] !== "string")
-                throw new Error("Only string values are allowed for _tags");
-              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
-            } else if (op === "$eq") {
-              if (typeof v["$eq"] !== "string")
-                throw new Error("Only string values are allowed for _tags");
-              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
-            } else {
-              throw new Error(`Invalid tag query`);
-            }
-          }
-          if (conjs.length === 1) expressions.push(conjs[0]);
-          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
-        } else {
-          expressions.push([
-            "IS NOT NULL",
-            ["PARAM", `Tags.${encodeTag(v as string)}`],
-          ]);
-        }
-      } else if (k.startsWith("Tags.")) {
-        let exists: boolean;
-        if (typeof v === "boolean") exists = v;
-        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
-        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
-        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
-        else throw new Error(`Invalid tag query`);
-
-        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
-      } else if (typeof v === "object") {
-        if (isArray(v)) throw new Error(`Invalid type`);
-
-        const exps: Expression[] = [];
-        for (const [kk, vv] of Object.entries(v)) {
-          if (kk === "$eq") {
-            exps.push(["=", ["PARAM", k], vv]);
-          } else if (kk === "$ne") {
-            exps.push([
-              "OR",
-              ["<>", ["PARAM", k], vv],
-              ["IS NULL", ["PARAM", k]],
-            ]);
-          } else if (kk === "$lt") {
-            exps.push(["<", ["PARAM", k], vv]);
-          } else if (kk === "$lte") {
-            exps.push(["<=", ["PARAM", k], vv]);
-          } else if (kk === "$gt") {
-            exps.push([">", ["PARAM", k], vv]);
-          } else if (kk === "$gte") {
-            exps.push([">=", ["PARAM", k], vv]);
-          } else {
-            throw new Error(`Operator ${kk} not supported`);
-          }
-          if (!["string", "number", "boolean"].includes(typeof vv))
-            throw new Error(`Invalid value for ${kk} operator`);
-        }
-        if (exps.length === 1) {
-          expressions.push(exps[0]);
-        } else if (exps.length > 1) {
-          const and: Expression = ["AND"];
-          expressions.push(and.concat(exps));
-        }
-      } else {
-        expressions.push(["=", ["PARAM", k], v]);
-      }
-    }
-    if (expressions.length === 1) return expressions[0];
-    if (expressions.length === 0) return true;
-    return ["AND", ...expressions];
-  }
-
-  // empty filter
-  if (!Object.keys(query).length) return true;
-
-  return recursive(query);
-}
-
-export function flattenPreset(
-  preset: Record<string, unknown>
-): Record<string, unknown> {
-  const p = Object.assign({}, preset);
-  if (p.precondition) {
-    try {
-      // Try parse to check expression validity
-      parse(p.precondition as string);
-    } catch (error) {
-      p.precondition = convertOldPrecondition(
-        JSON.parse(p.precondition as string)
-      );
-      p.precondition = (p.precondition as string).length
-        ? stringify(p.precondition as Expression)
-        : "";
-    }
-  }
-
-  if (p.events) {
-    const e = [];
-    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
-    p.events = e.join(", ");
-  }
-
-  const provision = p.configurations[0];
-  if (
-    (p.configurations as any[]).length === 1 &&
-    provision.type === "provision" &&
-    provision.name &&
-    provision.name.length
-  ) {
-    p.provision = provision.name;
-    p.provisionArgs = provision.args
-      ? provision.args.map((a) => stringify(a)).join(", ")
-      : "";
-  }
-
-  delete p.configurations;
-  return p;
-}
-
-export function flattenFile(
-  file: Record<string, unknown>
-): Record<string, unknown> {
-  const f = {};
-  f["_id"] = file["_id"];
-  if (file.metadata) {
-    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
-    f["metadata.oui"] = file["metadata"]["oui"] || "";
-    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
-    f["metadata.version"] = file["metadata"]["version"] || "";
-  }
-  return f;
-}
-
-export function preProcessPreset(
-  data: Record<string, unknown>
-): Record<string, unknown> {
-  const preset = Object.assign({}, data);
-
-  if (!preset.precondition) preset.precondition = "";
-  // Try parse to check expression validity
-  parse(preset.precondition as string);
-
-  preset.weight = parseInt(preset.weight as string) || 0;
-
-  const events = {};
-  if (preset.events) {
-    for (let e of (preset.events as string).split(",")) {
-      let v = true;
-      e = e.trim();
-      if (e.startsWith("-")) {
-        v = false;
-        e = e.slice(1).trim();
-      }
-      if (e) events[e] = v;
-    }
-  }
-
-  preset.events = events;
-
-  if (!preset.provision) throw new Error("Invalid preset provision");
-
-  const configuration = {
-    type: "provision",
-    name: preset.provision,
-    args: null,
-  };
-
-  if (preset.provisionArgs)
-    configuration.args = parseList(preset.provisionArgs as string);
-
-  delete preset.provision;
-  delete preset.provisionArgs;
-  preset.configurations = [configuration];
-  return preset;
-}

lib/ping.ts

@@ -1,24 +1,6 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { platform } from "os";
-import { exec } from "child_process";
+import { platform } from "node:os";
+import { exec } from "node:child_process";
+import { domainToASCII } from "node:url";
 
 export interface PingResult {
   packetsTransmitted: number;
@@ -30,65 +12,87 @@ export interface PingResult {
   mdev: number;
 }
 
-export function ping(
-  host: string,
-  callback: (err: Error, res?: PingResult, stdout?: string) => void
-): void {
-  let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
-  host = host.replace("[", "").replace("]", "");
-  switch (platform()) {
+function isValidHost(host: string): boolean {
+  // Valid chars in IPv4, IPv6, domain names
+  if (/^[a-zA-Z0-9\-.:[\]-]+$/.test(host)) return true;
+
+  // Check if input is an IDN convert to Punycode
+  // Can't merge with above because domainToASCII doesn't accept IP addresses
+  return /^[a-zA-Z0-9\-.:[\]-]+$/.test(domainToASCII(host));
+}
+
+export function parsePing(osPlatform: string, stdout: string): PingResult {
+  let parseRegExp1: RegExp, parseRegExp2: RegExp, parsed: PingResult;
+  switch (osPlatform) {
     case "linux":
-      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
       parseRegExp1 =
-        /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss[^]*= ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
       parseRegExp2 =
-        /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss/;
       break;
 
     case "freebsd":
-      // Send a single packet because on FreeBSD only superuser can send
-      // packets that are only 200 ms apart.
-      cmd = `ping -t 1 -c 3 ${host}`;
       parseRegExp1 =
         /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
       parseRegExp2 =
         /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
       break;
+  }
 
+  const m1 = stdout.match(parseRegExp1);
+  if (m1) {
+    parsed = {
+      packetsTransmitted: +m1[1],
+      packetsReceived: +m1[2],
+      packetLoss: +m1[3],
+      min: +m1[4],
+      avg: +m1[5],
+      max: +m1[6],
+      mdev: +m1[7],
+    };
+  } else {
+    const m2 = stdout.match(parseRegExp2);
+    if (m2) {
+      parsed = {
+        packetsTransmitted: +m2[1],
+        packetsReceived: +m2[2],
+        packetLoss: +m2[3],
+        min: null,
+        avg: null,
+        max: null,
+        mdev: null,
+      };
+    }
+  }
+  return parsed;
+}
+
+export function ping(
+  host: string,
+  callback: (err: Error, res?: PingResult, stdout?: string) => void,
+): void {
+  // Validate input to prevent possible remote code execution
+  // Credit to Alex Hordijk for reporting this vulnerability
+  if (!isValidHost(host)) return callback(new Error("Invalid host"));
+  host = host.replace("[", "").replace("]", "");
+  let cmd: string;
+
+  switch (platform()) {
+    case "linux":
+      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
+      break;
+    case "freebsd":
+      // Send a single packet because on FreeBSD only superuser can send
+      // packets that are only 200 ms apart.
+      cmd = `ping -t 1 -c 3 ${host}`;
+      break;
     default:
       return callback(new Error("Platform not supported"));
   }
 
   exec(cmd, (err, stdout) => {
-    let parsed: PingResult;
-    if (stdout) {
-      const m1 = stdout.match(parseRegExp1);
-      if (m1) {
-        parsed = {
-          packetsTransmitted: +m1[1],
-          packetsReceived: +m1[2],
-          packetLoss: +m1[3],
-          min: +m1[4],
-          avg: +m1[5],
-          max: +m1[6],
-          mdev: +m1[7],
-        };
-      } else {
-        const m2 = stdout.match(parseRegExp2);
-        if (m2) {
-          parsed = {
-            packetsTransmitted: +m2[1],
-            packetsReceived: +m2[2],
-            packetLoss: +m2[3],
-            min: null,
-            avg: null,
-            max: null,
-            mdev: null,
-          };
-        }
-      }
-    }
-
-    callback(err, parsed, stdout);
+    if (err) return callback(err);
+    const parsed: PingResult = parsePing(platform(), stdout);
+    return callback(err, parsed, stdout);
   });
 }

lib/query.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 function isObject(obj: any): boolean {
   return Object.prototype.toString.call(obj) === "[object Object]";
 }
@@ -54,6 +35,17 @@ function normalize(input): any {
   return input;
 }
 
+const EXPAND_OPS = new Set([
+  "$eq",
+  "$gt",
+  "$gte",
+  "$in",
+  "$lt",
+  "$lte",
+  "$ne",
+  "$nin",
+]);
+
 function expandValue(value: unknown): unknown[] {
   if (Array.isArray(value)) {
     let a = [];
@@ -71,7 +63,8 @@ function expandValue(value: unknown): unknown[] {
   const values = [];
   for (const [k, v] of Object.entries(value)) {
     keys.push(k);
-    values.push(expandValue(v));
+    if (EXPAND_OPS.has(k)) values.push(expandValue(v));
+    else values.push([v]);
     indices.push(0);
   }
 
@@ -106,7 +99,7 @@ function permute(param, val): any[] {
 }
 
 export function expand(
-  query: Record<string, unknown>
+  query: Record<string, unknown>,
 ): Record<string, unknown> {
   const newQuery = {};
   for (const [k, v] of Object.entries(query)) {
@@ -120,7 +113,7 @@ export function expand(
         if (v && (v["$ne"] != null || v["$not"] != null)) {
           if (Object.keys(v).length > 1)
             throw new Error("Cannot mix $ne or $not with other operators");
-          for (const c of conditions) newQuery["$an"].push(c);
+          for (const c of conditions) newQuery["$and"].push(c);
         } else {
           newQuery["$and"].push({ $or: conditions });
         }
@@ -135,7 +128,7 @@ export function expand(
 
 export function sanitizeQueryTypes(
   query: Record<string, unknown>,
-  types: Record<string, (v: unknown) => unknown>
+  types: Record<string, (v: unknown) => unknown>,
 ): Record<string, unknown> {
   for (const [k, v] of Object.entries(query)) {
     if (k[0] === "$") {

lib/sandbox.ts

@@ -1,30 +1,11 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as vm from "vm";
+import * as vm from "node:vm";
 import seedrandom from "seedrandom";
-import * as device from "./device";
-import * as extensions from "./extensions";
-import * as logger from "./logger";
-import * as scheduling from "./scheduling";
-import Path from "./common/path";
-import { Fault, SessionContext, ScriptResult } from "./types";
+import * as device from "./device.ts";
+import * as extensions from "./extensions.ts";
+import * as logger from "./logger.ts";
+import * as scheduling from "./scheduling.ts";
+import Path from "./common/path.ts";
+import { Fault, SessionContext, ScriptResult } from "./types.ts";
 
 // Used for throwing to exit user script and commit
 const COMMIT = Symbol();
@@ -34,7 +15,7 @@ const EXT = Symbol();
 
 const UNDEFINED = undefined;
 
-const context = vm.createContext();
+const context = vm.createContext(undefined, { microtaskMode: "afterEvaluate" });
 
 let state;
 
@@ -50,7 +31,7 @@ function runExtension(sessionContext, key, extCall): Promise<Fault> {
   }
 
   let prom = re.get(key);
-  if (!prom) {
+  if (prom == null) {
     re.set(
       key,
       (prom = new Promise((resolve, reject) => {
@@ -62,7 +43,7 @@ function runExtension(sessionContext, key, extCall): Promise<Fault> {
             resolve(fault);
           })
           .catch(reject);
-      }))
+      })),
     );
   }
 
@@ -78,7 +59,7 @@ class SandboxDate {
       number?,
       number?,
       number?,
-      number?
+      number?,
     ]
   ) {
     if (argumentList.length) return new Date(...argumentList);
@@ -143,7 +124,7 @@ class ParameterWrapper {
             unpacked = device.unpack(
               state.sessionContext.deviceData,
               path,
-              state.revision
+              state.revision,
             );
           }
 
@@ -151,7 +132,7 @@ class ParameterWrapper {
 
           const attr = state.sessionContext.deviceData.attributes.get(
             unpacked[0],
-            state.revision
+            state.revision,
           )[attrName];
 
           if (!attr) return UNDEFINED;
@@ -170,7 +151,7 @@ class ParameterWrapper {
           unpacked = device.unpack(
             state.sessionContext.deviceData,
             path,
-            state.revision
+            state.revision,
           );
         }
 
@@ -189,7 +170,7 @@ class ParameterWrapper {
           unpacked = device.unpack(
             state.sessionContext.deviceData,
             path,
-            state.revision
+            state.revision,
           );
         }
 
@@ -207,7 +188,7 @@ class ParameterWrapper {
         unpacked = device.unpack(
           state.sessionContext.deviceData,
           path,
-          state.revision
+          state.revision,
         );
       }
 
@@ -220,7 +201,7 @@ class ParameterWrapper {
 function declare(
   path: string,
   timestamps: { [attr: string]: number },
-  values: { [attr: string]: any }
+  values: { [attr: string]: any },
 ): ParameterWrapper {
   state.uncommitted = true;
   if (!timestamps) timestamps = {};
@@ -255,6 +236,7 @@ function declare(
   }
 
   for (const [attrName, attrTimestamp] of Object.entries(timestamps)) {
+    if (!(attrTimestamp >= 1)) continue;
     if (attrName === "path") {
       declaration.pathGet = attrTimestamp;
     } else {
@@ -285,7 +267,7 @@ function commit(): void {
     throw COMMIT;
   } else if (state.revision > state.maxRevision + 1) {
     throw new Error(
-      "Declare function should not be called from within a try/catch block"
+      "Declare function should not be called from within a try/catch block",
     );
   }
 }
@@ -350,12 +332,12 @@ function errorToFault(err: Error): Fault {
     fault.detail["stack"] = err.stack;
     // Trim the stack trace at the self-executing anonymous wrapper function
     const stackTrimIndex = fault.detail["stack"].match(
-      /\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/
+      /\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/,
     );
     if (stackTrimIndex) {
       fault.detail["stack"] = fault.detail["stack"].slice(
         0,
-        stackTrimIndex.index
+        stackTrimIndex.index,
       );
     }
   }
@@ -369,7 +351,7 @@ export async function run(
   sessionContext: SessionContext,
   startRevision: number,
   maxRevision: number,
-  extCounter = 0
+  extCounter = 0,
 ): Promise<ScriptResult> {
   state = {
     sessionContext: sessionContext,
@@ -390,7 +372,7 @@ export async function run(
   let ret, status;
 
   try {
-    ret = script.runInContext(context, { displayErrors: false });
+    ret = script.runInContext(context, { displayErrors: false, timeout: 50 });
     status = 0;
   } catch (err) {
     if (err === COMMIT) {
@@ -414,7 +396,7 @@ export async function run(
   await Promise.all(
     Object.entries(_state.extensions).map(async ([k, v]) => {
       fault = (await runExtension(_state.sessionContext, k, v)) || fault;
-    })
+    }),
   );
 
   if (fault) {
@@ -434,7 +416,7 @@ export async function run(
       sessionContext,
       startRevision,
       maxRevision,
-      extCounter - _state.extCounter
+      extCounter - _state.extCounter,
     );
   }
 

lib/scheduling.ts

@@ -1,24 +1,5 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as crypto from "crypto";
-import * as later from "later";
+import * as crypto from "node:crypto";
+import * as later from "@breejs/later";
 
 function md532(str): number {
   const digest = crypto.createHash("md5").update(str).digest();
@@ -37,7 +18,7 @@ export function variance(deviceId: string, vrnc: number): number {
 export function interval(
   timestamp: number,
   intrvl: number,
-  offset = 0
+  offset = 0,
 ): number {
   return Math.trunc((timestamp + offset) / intrvl) * intrvl - offset;
 }
@@ -52,7 +33,7 @@ export function parseCron(cronExp: string): any {
 export function cron(
   timestamp: number,
   schedule: unknown,
-  offset = 0
+  offset = 0,
 ): number[] {
   // TODO later.js doesn't throw erorr if expression is invalid!
   const ret = [0, 0];

lib/server.ts

@@ -1,31 +1,13 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as fs from "fs";
-import * as http from "http";
-import * as https from "https";
-import { Socket } from "net";
-import * as path from "path";
-import { ROOT_DIR } from "./config";
+import { readFileSync } from "node:fs";
+import * as http from "node:http";
+import * as https from "node:https";
+import { Socket } from "node:net";
+import * as path from "node:path";
+import { ROOT_DIR } from "./config.ts";
 
 let server: http.Server | https.Server;
-let listener: (...args) => void;
+let listener: http.RequestListener;
+let stopping = false;
 
 function closeServer(timeout, callback): void {
   if (!server) return void callback();
@@ -58,23 +40,70 @@ interface ServerOptions {
   ssl?: { key: string; cert: string };
   timeout?: number;
   keepAliveTimeout?: number;
+  requestTimeout?: number;
   onConnection?: (socket: Socket) => void;
+  onClientError?: (err: Error, socket: Socket) => void;
+}
+
+interface SocketEndpoint {
+  localAddress: string;
+  localPort: number;
+  remoteAddress: string;
+  remotePort: number;
+  remoteFamily: "IPv4" | "IPv6";
+}
+
+// Save this info as they're not accessible after a socket has been closed
+const socketEndpoints: WeakMap<Socket, SocketEndpoint> = new WeakMap();
+
+type Promisify<T extends (...args: any) => any> = (
+  ...args: Parameters<T>
+) => Promise<ReturnType<T>>;
+
+function getValidPrivKeys(value: string): Buffer[] {
+  return value.split(":").map((str) => {
+    str = str.trim();
+    const buf = str.startsWith("-----BEGIN ")
+      ? Buffer.from(str)
+      : readFileSync(path.resolve(ROOT_DIR, str));
+    return buf;
+  });
+}
+
+function getValidCerts(value: string): Buffer[] {
+  return value.split(":").map((str) => {
+    str = str.trim();
+    const buf = str.startsWith("-----BEGIN ")
+      ? Buffer.from(str)
+      : readFileSync(path.resolve(ROOT_DIR, str));
+    return buf;
+  });
 }
 
 export function start(
   options: ServerOptions,
-  _listener: http.RequestListener
+  _listener: Promisify<http.RequestListener>,
 ): void {
-  listener = _listener;
+  listener = (req, res) => {
+    if (stopping) res.setHeader("Connection", "close");
+    _listener(req, res).catch((err) => {
+      try {
+        res.socket.unref();
+        if (res.headersSent) {
+          res.writeHead(500, { Connection: "close" });
+          res.end(`${err.name}: ${err.message}`);
+        }
+      } catch (err) {
+        // Ignore
+      }
+      throw err;
+    });
+  };
 
   if (options.ssl) {
     const opts = {
-      key: options.ssl.key
-        .split(":")
-        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
-      cert: options.ssl.cert
-        .split(":")
-        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
+      key: getValidPrivKeys(options.ssl.key),
+      cert: getValidCerts(options.ssl.cert),
     };
 
     server = https.createServer(opts, listener);
@@ -85,13 +114,38 @@ export function start(
     if (options.onConnection) server.on("connection", options.onConnection);
   }
 
+  server.on("connection", (socket: Socket) => {
+    socketEndpoints.set(socket, {
+      localAddress: socket.localAddress,
+      localPort: socket.localPort,
+      remoteAddress: socket.remoteAddress,
+      remotePort: socket.remotePort,
+      remoteFamily: socket.remoteFamily as "IPv4" | "IPv6",
+    });
+  });
+
+  if (options.onClientError) {
+    server.on("clientError", (err, socket: Socket) => {
+      if (err["code"] !== "ECONNRESET" && socket.writable)
+        socket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n");
+
+      // As per Node docs: This event is guaranteed to be passed an instance
+      // of the <net.Socket> class
+      options.onClientError(err, socket as Socket);
+    });
+  }
+
   server.timeout = options.timeout || 0;
   if (options.keepAliveTimeout != null)
     server.keepAliveTimeout = options.keepAliveTimeout;
+  if (options.requestTimeout != null)
+    server.requestTimeout = options.requestTimeout;
+
   server.listen({ port: options.port, host: options.host });
 }
 
-export function stop(): Promise<void> {
+export function stop(terminateConnections = true): Promise<void> {
+  stopping = terminateConnections;
   return new Promise((resolve, reject) => {
     setTimeout(() => {
       reject(new Error("Could not close server in a timely manner"));
@@ -99,3 +153,8 @@ export function stop(): Promise<void> {
     closeServer(20000, resolve);
   });
 }
+
+export function getSocketEndpoints(socket: Socket): SocketEndpoint {
+  // TLSSocket keeps a reference to the raw TCP socket in _parent
+  return socketEndpoints.get(socket["_parent"] ?? socket);
+}

lib/soap.ts

@@ -1,30 +1,11 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 import {
   parseXml,
   Element,
   parseAttrs,
   encodeEntities,
   decodeEntities,
-} from "./xml-parser";
-import memoize from "./common/memoize";
+} from "./xml-parser.ts";
+import memoize from "./common/memoize.ts";
 import { version as VERSION } from "../package.json";
 import {
   InformRequest,
@@ -34,22 +15,22 @@ import {
   SoapMessage,
   TransferCompleteRequest,
   AcsRequest,
-  GetParameterNamesResponse,
-  GetParameterValuesResponse,
-  GetParameterAttributesResponse,
-  SetParameterValuesResponse,
-  SetParameterAttributesResponse,
-  AddObjectResponse,
-  DeleteObject,
-  DeleteObjectResponse,
-  RebootResponse,
-  FactoryResetResponse,
-  DownloadResponse,
+  type GetParameterNamesResponse,
+  type GetParameterValuesResponse,
+  type GetParameterAttributesResponse,
+  type SetParameterValuesResponse,
+  type SetParameterAttributesResponse,
+  type AddObjectResponse,
+  type DeleteObject,
+  type DeleteObjectResponse,
+  type RebootResponse,
+  type FactoryResetResponse,
+  type DownloadResponse,
   GetRPCMethodsRequest,
   RequestDownloadRequest,
   AcsResponse,
-} from "./types";
-import Path from "./common/path";
+} from "./types.ts";
+import Path from "./common/path.ts";
 
 const SERVER_NAME = `GenieACS/${VERSION}`;
 
@@ -158,7 +139,7 @@ const getValueType = memoize((str: string) => {
 });
 
 function parameterValueList(
-  xml: Element
+  xml: Element,
 ): [Path, string | number | boolean, string][] {
   return xml.children
     .map<[Path, string | number | boolean, string]>((e) => {
@@ -291,7 +272,7 @@ function GetParameterNamesResponse(xml): GetParameterNamesResponse {
   return {
     name: "GetParameterNamesResponse",
     parameterList: parameterInfoList(
-      xml.children.find((n) => n.localName === "ParameterList")
+      xml.children.find((n) => n.localName === "ParameterList"),
     ),
   };
 }
@@ -308,7 +289,7 @@ function GetParameterValuesResponse(xml: Element): GetParameterValuesResponse {
   return {
     name: "GetParameterValuesResponse",
     parameterList: parameterValueList(
-      xml.children.find((n) => n.localName === "ParameterList")
+      xml.children.find((n) => n.localName === "ParameterList"),
     ),
   };
 }
@@ -322,12 +303,12 @@ function GetParameterAttributes(methodRequest): string {
 }
 
 function GetParameterAttributesResponse(
-  xml: Element
+  xml: Element,
 ): GetParameterAttributesResponse {
   return {
     name: "GetParameterAttributesResponse",
     parameterList: parameterAttributeList(
-      xml.children.find((n) => n.localName === "ParameterList")
+      xml.children.find((n) => n.localName === "ParameterList"),
     ),
   };
 }
@@ -509,19 +490,19 @@ function Download(methodRequest): string {
   }</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
     methodRequest.url
   }</URL><Username>${encodeEntities(
-    methodRequest.username || ""
+    methodRequest.username || "",
   )}</Username><Password>${encodeEntities(
-    methodRequest.password || ""
+    methodRequest.password || "",
   )}</Password><FileSize>${
     methodRequest.fileSize || "0"
   }</FileSize><TargetFileName>${encodeEntities(
-    methodRequest.targetFileName || ""
+    methodRequest.targetFileName || "",
   )}</TargetFileName><DelaySeconds>${
     methodRequest.delaySeconds || "0"
   }</DelaySeconds><SuccessURL>${encodeEntities(
-    methodRequest.successUrl || ""
+    methodRequest.successUrl || "",
   )}</SuccessURL><FailureURL>${encodeEntities(
-    methodRequest.failureUrl || ""
+    methodRequest.failureUrl || "",
   )}</FailureURL></cwmp:Download>`;
 }
 
@@ -730,6 +711,18 @@ function RequestDownloadResponse(): string {
   return "<cwmp:RequestDownloadResponse></cwmp:RequestDownloadResponse>";
 }
 
+function AcsFault(f: CpeFault): string {
+  return `<soap-env:Body:Fault><faultcode>${encodeEntities(
+    f.faultCode,
+  )}</faultcode><faultstring>${encodeEntities(
+    f.faultString,
+  )}</faultstring><detail><cwmp:Fault><FaultCode>${encodeEntities(
+    f.detail.faultCode,
+  )}</FaultCode><FaultString>${encodeEntities(
+    f.detail.faultString,
+  )}</FaultString></cwmp:Fault></detail></soap-env:Body:Fault>`;
+}
+
 function faultStruct(xml: Element): FaultStruct {
   let faultCode: string,
     faultString: string,
@@ -827,7 +820,7 @@ function fault(xml: Element): CpeFault {
 
 export function request(
   body: string,
-  warn: Record<string, unknown>[]
+  warn: Record<string, unknown>[],
 ): SoapMessage {
   warnings = warn;
 
@@ -838,6 +831,7 @@ export function request(
     cpeRequest: null,
     cpeFault: null,
     cpeResponse: null,
+    unknownMethod: null,
   };
 
   if (!body.length) return rpc;
@@ -884,7 +878,7 @@ export function request(
         const attrs = memoizedParseAttrs(e.attrs);
         const attr = namespace
           ? attrs.find(
-              (s) => s.namespace === "xmlns" && s.localName === namespace
+              (s) => s.namespace === "xmlns" && s.localName === namespace,
             )
           : attrs.find((s) => s.name === "xmlns");
 
@@ -959,7 +953,8 @@ export function request(
       rpc.cpeFault = fault(methodElement);
       break;
     default:
-      throw new Error(`8000 Method not supported ${methodElement.localName}`);
+      rpc.unknownMethod = methodElement.localName;
+      break;
   }
 
   return rpc;
@@ -987,6 +982,7 @@ export function response(rpc: {
   id: string;
   acsRequest?: AcsRequest;
   acsResponse?: AcsResponse;
+  acsFault?: CpeFault;
   cwmpVersion?: string;
 }): { code: number; headers: Record<string, string>; data: string } {
   const headers = {
@@ -1015,7 +1011,7 @@ export function response(rpc: {
         throw new Error(
           `Unknown method response type ${
             (rpc.acsResponse as AcsResponse).name
-          }`
+          }`,
         );
     }
   } else if (rpc.acsRequest) {
@@ -1052,9 +1048,11 @@ export function response(rpc: {
         break;
       default:
         throw new Error(
-          `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`
+          `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`,
         );
     }
+  } else if (rpc.acsFault) {
+    body = AcsFault(rpc.acsFault);
   }
 
   headers["Content-Type"] = 'text/xml; charset="utf-8"';

lib/types.ts

@@ -1,28 +1,9 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
-import PathSet from "./common/path-set";
-import VersionedMap from "./versioned-map";
-import InstanceSet from "./instance-set";
-import { IncomingMessage, ServerResponse } from "http";
-import { Script } from "vm";
+import { IncomingMessage, ServerResponse } from "node:http";
+import { Script } from "node:vm";
+import Path from "./common/path.ts";
+import PathSet from "./common/path-set.ts";
+import VersionedMap from "./versioned-map.ts";
+import InstanceSet from "./instance-set.ts";
 
 export type Expression = string | number | boolean | null | any[];
 
@@ -97,7 +78,7 @@ export type VirtualParameterDeclaration = [
     value?: [string | number | boolean, string?];
     notification?: number;
     accessList?: string[];
-  }?
+  }?,
 ];
 
 export interface SyncState {
@@ -139,7 +120,7 @@ export interface SessionContext {
     AttributeTimestamps,
     AttributeValues,
     AttributeTimestamps,
-    AttributeValues
+    AttributeValues,
   ][][];
   revisions: number[];
   rpcCount: number;
@@ -160,7 +141,6 @@ export interface SessionContext {
   authState: number;
   tasks?: Task[];
   operations?: { [commandKey: string]: Operation };
-  cacheUntil?: number;
   syncState?: SyncState;
   lastActivity?: number;
   extendLock?: number;
@@ -180,7 +160,7 @@ export interface Task {
   fileName?: string;
   targetFileName?: string;
   expiry?: number;
-  provisions?: (string | number | boolean)[][];
+  provisions?: [string, ...Expression[]][];
 }
 
 export interface Operation {
@@ -456,7 +436,7 @@ export type Clear = [
     notification?: number;
     accessList?: number;
   }?,
-  number?
+  number?,
 ];
 
 export interface Preset {
@@ -526,6 +506,7 @@ export interface SoapMessage {
   cpeRequest?: CpeRequest;
   cpeFault?: CpeFault;
   cpeResponse?: CpeResponse;
+  unknownMethod?: string;
 }
 
 export interface ScriptResult {

lib/ui.ts

@@ -1,46 +1,22 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { constants } from "zlib";
+import { constants } from "node:zlib";
 import Koa from "koa";
-import Router from "koa-router";
+import Router from "@koa/router";
 import * as jwt from "jsonwebtoken";
-import koaStatic from "koa-static";
+import koaSend from "koa-send";
 import koaCompress from "koa-compress";
-import koaBodyParser from "koa-bodyparser";
+import koaBodyParser from "@koa/bodyparser";
 import koaJwt from "koa-jwt";
-import * as config from "./config";
-import api from "./ui/api";
-import Authorizer from "./common/authorizer";
-import * as logger from "./logger";
-import * as localCache from "./local-cache";
-import { PermissionSet } from "./types";
-import { authLocal } from "./ui/api-functions";
-import * as init from "./init";
+import * as config from "./config.ts";
+import api from "./ui/api.ts";
+import Authorizer from "./common/authorizer.ts";
+import * as logger from "./logger.ts";
+import * as localCache from "./ui/local-cache.ts";
+import { PermissionSet } from "./types.ts";
+import { authLocal } from "./api-functions.ts";
+import * as init from "./init.ts";
 import { version as VERSION } from "../package.json";
-import memoize from "./common/memoize";
-
-declare module "koa" {
-  interface Request {
-    body: any;
-  }
-}
+import memoize from "./common/memoize.ts";
+import { APP_JS, APP_CSS, FAVICON_PNG } from "../build/assets.ts";
 
 const koa = new Koa();
 const router = new Router();
@@ -53,22 +29,26 @@ const getAuthorizer = memoize(
     const roles: string[] = JSON.parse(rolesStr);
     const allPermissions = localCache.getPermissions(snapshot);
     const permissionSets: PermissionSet[] = roles.map((r) =>
-      Object.values(allPermissions[r] || {})
+      Object.values(allPermissions[r] || {}),
     );
     return new Authorizer(permissionSets);
-  }
+  },
 );
 
-koa.on("error", async (err, ctx) => {
+koa.on("error", (err, ctx) => {
   setTimeout(() => {
     // Ignored errors resulting from aborted requests
     if (ctx?.req.aborted) return;
+
+    // Ignore client errors (e.g. malicious path)
+    if (err.status === 400) return;
+
     throw err;
   });
 });
 
 koa.use(async (ctx, next) => {
-  const configSnapshot = await localCache.getCurrentSnapshot();
+  const configSnapshot = await localCache.getRevision();
   ctx.state.configSnapshot = configSnapshot;
   ctx.set("X-Config-Snapshot", configSnapshot);
   ctx.set("GenieACS-Version", VERSION);
@@ -89,7 +69,7 @@ koa.use(
 
       return true;
     },
-  })
+  }),
 );
 
 koa.use(async (ctx, next) => {
@@ -109,7 +89,7 @@ koa.use(async (ctx, next) => {
 
   ctx.state.authorizer = getAuthorizer(
     ctx.state.configSnapshot,
-    JSON.stringify(roles)
+    JSON.stringify(roles),
   );
 
   return next();
@@ -221,6 +201,10 @@ router.post("/init", async (ctx) => {
 });
 
 router.get("/", async (ctx) => {
+  // koa-router seems to tolerate double slashes in the URL but that can
+  // be problematic when using relatives asset paths in HTML
+  if (ctx.path.endsWith("//")) return;
+
   const permissionSets: PermissionSet[] =
     ctx.state.authorizer.getPermissionSets();
 
@@ -228,12 +212,12 @@ router.get("/", async (ctx) => {
   if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
     wizard = '<script>window.location.hash = "#!/wizard";</script>';
 
-  ctx.body = `
+  ctx.body = `<!DOCTYPE html>
   <html>
     <head>
       <title>GenieACS</title>
-      <link rel="shortcut icon" type="image/png" href="favicon.png" />
-      <link rel="stylesheet" href="app.css">
+      <link rel="shortcut icon" type="image/png" href="${FAVICON_PNG}" />
+      <link rel="stylesheet" href="${APP_CSS}">
     </head>
     <body>
     <noscript>GenieACS UI requires JavaScript to work. Please enable JavaScript in your browser.</noscript>
@@ -244,11 +228,11 @@ router.get("/", async (ctx) => {
         window.configSnapshot = ${JSON.stringify(ctx.state.configSnapshot)};
         window.genieacsVersion = ${JSON.stringify(VERSION)};
         window.username = ${JSON.stringify(
-          ctx.state.user ? ctx.state.user.username : ""
+          ctx.state.user ? ctx.state.user.username : "",
         )};
         window.permissionSets = ${JSON.stringify(permissionSets)};
       </script>
-      <script type="module" src="app.js"></script>${wizard} 
+      <script type="module" src="${APP_JS}"></script>${wizard} 
     </body>
   </html>
   `;
@@ -263,14 +247,26 @@ koa.use(
       flush: constants.Z_SYNC_FLUSH,
     },
     br: {
+      flush: constants.BROTLI_OPERATION_FLUSH,
       params: {
         [constants.BROTLI_PARAM_QUALITY]: 5,
       },
     },
-  })
+  }),
 );
 
 koa.use(router.routes());
-koa.use(koaStatic(config.ROOT_DIR + "/public"));
+koa.use(async (ctx, next) => {
+  await next();
+  if (ctx.method !== "HEAD" && ctx.method !== "GET") return;
+  if (ctx.body != null || ctx.status !== 404) return;
+  if (/(?:^|[\\/])\.\.(?:[\\/]|$)/.test(ctx.path)) return;
+
+  try {
+    await koaSend(ctx, ctx.path, { root: config.ROOT_DIR + "/public" });
+  } catch (err) {
+    if (err.status !== 404) throw err;
+  }
+});
 
 export const listener = koa.callback();

lib/ui/api-functions.ts

@@ -1,211 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ObjectId } from "mongodb";
-import * as db from "./db";
-import { del } from "../cache";
-import { getUsers } from "../local-cache";
-import { hashPassword } from "../auth";
-import {
-  insertTasks,
-  awaitSessionEnd,
-  awaitSessionStart,
-  connectionRequest,
-  deleteDevice,
-} from "../api-functions";
-import { Task } from "../types";
-
-async function deleteFault(id): Promise<void> {
-  const deviceId = id.split(":", 1)[0];
-  const channel = id.slice(deviceId.length + 1);
-
-  await Promise.all([
-    db.deleteFault(id),
-    channel.startsWith("task_")
-      ? db.deleteTask(new ObjectId(channel.slice(5)))
-      : null,
-  ]);
-
-  await del(`${deviceId}_tasks_faults_operations`);
-}
-
-export async function deleteResource(
-  resource: string,
-  id: string
-): Promise<void> {
-  switch (resource) {
-    case "devices":
-      await deleteDevice(id);
-      break;
-
-    case "files":
-      await db.deleteFile(id);
-      break;
-
-    case "faults":
-      await deleteFault(id);
-      break;
-
-    case "provisions":
-      await db.deleteProvision(id);
-      break;
-
-    case "presets":
-      await db.deletePreset(id);
-      break;
-
-    case "virtualParameters":
-      await db.deleteVirtualParameter(id);
-      break;
-
-    case "config":
-      await db.deleteConfig(id);
-      break;
-
-    case "permissions":
-      await db.deletePermission(id);
-      break;
-
-    case "users":
-      await db.deleteUser(id);
-      break;
-  }
-
-  await del("presets_hash");
-}
-
-export async function postTasks(
-  deviceId: string,
-  tasks: Task[],
-  timeout: number,
-  device: Record<string, { value?: [boolean | number | string, string] }>
-): Promise<{ connectionRequest: string; tasks: any[] }> {
-  for (const task of tasks) {
-    delete task._id;
-    task["device"] = deviceId;
-  }
-
-  tasks = await insertTasks(tasks);
-  const statuses = tasks.map((t) => {
-    return { _id: t._id, status: "pending" };
-  });
-  const lastInform = Date.now();
-  const notInSession = await awaitSessionEnd(deviceId, 30000);
-  if (!notInSession) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  await del(`${deviceId}_tasks_faults_operations`);
-
-  const status = await connectionRequest(deviceId, device);
-
-  if (status) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: status,
-      tasks: statuses,
-    };
-  }
-
-  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
-  if (!sessionStarted) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "No contact from CPE",
-      tasks: statuses,
-    };
-  }
-
-  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
-  if (!sessionEnded) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  const promises = [];
-  for (const s of statuses) {
-    promises.push(db.query("tasks", ["=", ["PARAM", "_id"], s._id]));
-    promises.push(
-      db.query("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
-    );
-  }
-
-  const res = await Promise.all(promises);
-  for (const [i, r] of statuses.entries()) {
-    if (res[i * 2].length === 0) {
-      r.status = "done";
-    } else if (res[i * 2 + 1].length === 1) {
-      r.status = "fault";
-      r["fault"] = res[i * 2 + 1][0];
-    }
-  }
-
-  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
-
-  return { connectionRequest: "OK", tasks: statuses };
-}
-
-export async function putResource(
-  resource: string,
-  id: string,
-  data: Record<string, unknown>
-): Promise<void> {
-  if (resource === "presets") {
-    await db.putPreset(id, data);
-  } else if (resource === "provisions") {
-    await db.putProvision(id, data);
-  } else if (resource === "virtualParameters") {
-    await db.putVirtualParameter(id, data);
-  } else if (resource === "config") {
-    await db.putConfig(id, data);
-  } else if (resource === "permissions") {
-    await db.putPermission(id, data);
-  } else if (resource === "users") {
-    delete data["password"];
-    delete data["salt"];
-    await db.putUser(id, data);
-  }
-
-  await del("presets_hash");
-}
-
-export function authLocal(
-  snapshot: string,
-  username: string,
-  password: string
-): Promise<boolean> {
-  return new Promise((resolve, reject) => {
-    const users = getUsers(snapshot);
-    const user = users[username];
-    if (!user?.password) return void resolve(null);
-    hashPassword(password, user.salt)
-      .then((hash) => {
-        if (hash === user.password) resolve(true);
-        else resolve(false);
-      })
-      .catch(reject);
-  });
-}

lib/ui/api.ts

@@ -1,37 +1,22 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as stream from "stream";
-import Router from "koa-router";
-import * as db from "./db";
-import * as apiFunctions from "./api-functions";
-import { evaluate, and, extractParams } from "../common/expression";
-import { parse } from "../common/expression-parser";
-import * as logger from "../logger";
-import { getConfig } from "../local-cache";
-import { QueryOptions, Expression } from "../types";
-import { generateSalt, hashPassword } from "../auth";
-import { del } from "../cache";
-import Authorizer from "../common/authorizer";
-import { ping } from "../ping";
-import { decodeTag } from "../common";
-import { stringify as yamlStringify } from "../common/yaml";
+import { Readable } from "node:stream";
+import Router from "@koa/router";
+import { ObjectId } from "mongodb";
+import * as db from "./db.ts";
+import * as apiFunctions from "../api-functions.ts";
+import { evaluate, and, extractParams } from "../common/expression/util.ts";
+import { parse } from "../common/expression/parser.ts";
+import * as logger from "../logger.ts";
+import { getConfig } from "../ui/local-cache.ts";
+import { Expression, Task } from "../types.ts";
+import { generateSalt, hashPassword } from "../auth.ts";
+import { del } from "../cache.ts";
+import Authorizer from "../common/authorizer.ts";
+import { ping } from "../ping.ts";
+import { decodeTag } from "../util.ts";
+import { stringify as yamlStringify } from "../common/yaml.ts";
+import { ResourceLockedError } from "../common/errors.ts";
+import { acquireLock, releaseLock } from "../lock.ts";
+import { collections } from "../db/db.ts";
 
 const router = new Router();
 export default router;
@@ -93,23 +78,22 @@ router.get(`/devices/:id.csv`, async (ctx) => {
     return void (ctx.status = 403);
   }
 
-  const res = await db.query("devices", filter);
-  if (!res[0]) return void (ctx.status = 404);
+  const { value: device } = await db.query("devices", filter).next();
+  if (!device) return void (ctx.status = 404);
 
   ctx.type = "text/csv";
   ctx.attachment(
     `device-${ctx.params.id}-${new Date()
       .toISOString()
-      .replace(/[:.]/g, "")}.csv`
+      .replace(/[:.]/g, "")}.csv`,
   );
 
-  ctx.body = new stream.PassThrough();
-  ctx.body.write(
-    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp,Notification,Notification timestamp,Access list,Access list timestamp\n"
-  );
+  const lines: string[] = [
+    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp,Notification,Notification timestamp,Access list,Access list timestamp",
+  ];
 
-  for (const k of Object.keys(res[0]).sort()) {
-    const p = res[0][k];
+  for (const k of Object.keys(device).sort()) {
+    const p = device[k];
     let value = "";
     let type = "";
     if (p.value) {
@@ -125,7 +109,7 @@ router.get(`/devices/:id.csv`, async (ctx) => {
       new Date(p.objectTimestamp).toJSON(),
       p.writable,
       new Date(p.writableTimestamp).toJSON(),
-      `"${value.toString().replace(/"/g, '""')}"`,
+      `"${String(value).replace(/"/g, '""')}"`,
       type,
       new Date(p.valueTimestamp).toJSON(),
       p.notification,
@@ -133,9 +117,9 @@ router.get(`/devices/:id.csv`, async (ctx) => {
       p.accessList ? p.accessList.join(", ") : "",
       new Date(p.accessListTimestamp).toJSON(),
     ];
-    ctx.body.write(row.map((r) => (r != null ? r : "")).join(",") + "\n");
+    lines.push(row.map((r) => (r != null ? r : "")).join(","));
   }
-  ctx.body.end();
+  ctx.body = lines.join("\n");
   logger.accessInfo(log);
 });
 
@@ -178,7 +162,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
   router.get(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
-    const options: QueryOptions = {};
+    const options: Parameters<typeof db.query>[2] = {};
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
       filter = and(filter, parse(singleParam(ctx.request.query.filter)));
@@ -216,23 +200,23 @@ for (const [resource, flags] of Object.entries(resources)) {
       ]);
     }
 
-    ctx.body = new stream.PassThrough();
-    ctx.type = "application/json";
-
-    let c = 0;
-    ctx.body.write("[\n");
-    await db.query(resource, filter, options, (obj) => {
-      ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
-    });
-    ctx.body.end("]");
-
     logger.accessInfo(log);
+    ctx.type = "application/json";
+    ctx.body = Readable.from(
+      (async function* () {
+        let c = 0;
+        yield "[\n";
+        for await (const obj of db.query(resource, filter, options))
+          yield (c++ ? "," : "") + JSON.stringify(obj) + "\n";
+        yield "]";
+      })(),
+    );
   });
 
   // CSV download
   router.get(`/${resource}.csv`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
-    const options: QueryOptions = { projection: {} };
+    const options: Parameters<typeof db.query>[2] = { projection: {} };
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
       filter = and(filter, parse(singleParam(ctx.request.query.filter)));
@@ -256,7 +240,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     }
 
     const columns: Record<string, Expression> = JSON.parse(
-      singleParam(ctx.request.query.columns)
+      singleParam(ctx.request.query.columns),
     );
     const now = Date.now();
 
@@ -276,61 +260,62 @@ for (const [resource, flags] of Object.entries(resources)) {
       ]);
     }
 
-    ctx.body = new stream.PassThrough();
+    logger.accessInfo(log);
     ctx.type = "text/csv";
     ctx.attachment(
-      `${resource}-${new Date(now).toISOString().replace(/[:.]/g, "")}.csv`
+      `${resource}-${new Date(now).toISOString().replace(/[:.]/g, "")}.csv`,
     );
 
-    ctx.body.write(
-      Object.keys(columns).map((k) => `"${k.replace(/"/, '""')}"`) + "\n"
+    ctx.body = Readable.from(
+      (async function* () {
+        yield Object.keys(columns).map((k) => `"${k.replace(/"/, '""')}"`) +
+          "\n";
+        for await (const obj of db.query(resource, filter, options)) {
+          const arr = Object.values(columns).map((exp) => {
+            const v = evaluate(exp, obj, null, (e) => {
+              if (Array.isArray(e)) {
+                if (e[0] === "PARAM") {
+                  if (resource === "devices") {
+                    if (e[1] === "Tags") {
+                      const tags = [];
+                      for (const p in obj)
+                        if (p.startsWith("Tags."))
+                          tags.push(decodeTag(p.slice(5)));
+
+                      return tags.join(", ");
+                    }
+                    if (e === exp) {
+                      const p = obj[e[1]];
+                      if (
+                        p &&
+                        p.value &&
+                        p.value[1] === "xsd:dateTime" &&
+                        typeof p.value[0] === "number"
+                      )
+                        return new Date(p.value[0]).toJSON();
+                    }
+                  } else if (resource === "faults") {
+                    if (e[1] === "detail") return yamlStringify(obj["detail"]);
+                  }
+                } else if (e[0] === "FUNC") {
+                  if (e[1] === "DATE_STRING") {
+                    if (e[2] && !Array.isArray(e[2]))
+                      return new Date(e[2]).toJSON();
+                  }
+                }
+              }
+
+              return e;
+            });
+
+            if (Array.isArray(v) || v == null) return "";
+            if (typeof v === "string") return `"${v.replace(/"/g, '""')}"`;
+            return v;
+          });
+          yield arr.join(",") + "\n";
+        }
+      })(),
     );
-    await db.query(resource, filter, options, (obj) => {
-      const arr = Object.values(columns).map((exp) => {
-        const v = evaluate(exp, obj, null, (e) => {
-          if (Array.isArray(e)) {
-            if (e[0] === "PARAM") {
-              if (resource === "devices") {
-                if (e[1] === "Tags") {
-                  const tags = [];
-                  for (const p in obj)
-                    if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
-
-                  return tags.join(", ");
-                }
-                if (e === exp) {
-                  const p = obj[e[1]];
-                  if (
-                    p &&
-                    p.value &&
-                    p.value[1] === "xsd:dateTime" &&
-                    typeof p.value[0] === "number"
-                  )
-                    return new Date(p.value[0]).toJSON();
-                }
-              } else if (resource === "faults") {
-                if (e[1] === "detail") return yamlStringify(obj["detail"]);
-              }
-            } else if (e[0] === "FUNC") {
-              if (e[1] === "DATE_STRING") {
-                if (e[2] && !Array.isArray(e[2]))
-                  return new Date(e[2]).toJSON();
-              }
-            }
-          }
-
-          return e;
-        });
-
-        if (Array.isArray(v) || v == null) return "";
-        if (typeof v === "string") return `"${v.replace(/"/g, '""')}"`;
-        return v;
-      });
-      ctx.body.write(arr.join(",") + "\n");
-    });
-    ctx.body.end();
-
-    logger.accessInfo(log);
   });
 
   router.head(`/${resource}/:id`, async (ctx) => {
@@ -351,9 +336,9 @@ for (const [resource, flags] of Object.entries(resources)) {
       return void (ctx.status = 403);
     }
 
-    const res = await db.query(resource, filter);
+    const count = await db.count(resource, filter);
 
-    if (!res.length) return void (ctx.status = 404);
+    if (!count) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = "";
@@ -377,12 +362,12 @@ for (const [resource, flags] of Object.entries(resources)) {
       return void (ctx.status = 403);
     }
 
-    const res = await db.query(resource, filter);
+    const { value: res } = await db.query(resource, filter).next();
 
-    if (!res.length) return void (ctx.status = 404);
+    if (!res) return void (ctx.status = 404);
 
     logger.accessInfo(log);
-    ctx.body = res[0];
+    ctx.body = res;
   });
 
   if (flags & RESOURCE_DELETE) {
@@ -403,16 +388,27 @@ for (const [resource, flags] of Object.entries(resources)) {
         logUnauthorizedWarning(log);
         return void (ctx.status = 403);
       }
-      const res = await db.query(resource, filter);
-      if (!res.length) return void (ctx.status = 404);
+      const { value: res } = await db.query(resource, filter).next();
+      if (!res) return void (ctx.status = 404);
 
-      const validate = authorizer.getValidator(resource, res[0]);
+      const validate = authorizer.getValidator(resource, res);
       if (!validate("delete")) {
         logUnauthorizedWarning(log);
         return void (ctx.status = 403);
       }
 
-      await apiFunctions.deleteResource(resource, ctx.params.id);
+      try {
+        await apiFunctions.deleteResource(resource, ctx.params.id);
+      } catch (err) {
+        if (err instanceof ResourceLockedError) {
+          log.message += " failed";
+          logger.accessWarn(log);
+          ctx.status = 503;
+          ctx.body = err.message;
+          return;
+        }
+        throw err;
+      }
 
       logger.accessInfo(log);
 
@@ -535,33 +531,69 @@ router.put("/files/:id", async (ctx) => {
 });
 
 router.post("/devices/:id/tasks", async (ctx) => {
+  const deviceId = ctx.params.id;
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Commit tasks",
     context: ctx,
-    deviceId: ctx.params.id,
+    deviceId: deviceId,
     tasks: null,
   };
 
-  const filter = and(authorizer.getFilter("devices", 3), [
-    "=",
-    ["PARAM", "DeviceID.ID"],
-    ctx.params.id,
-  ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
     return void (ctx.status = 403);
   }
-  const devices = await db.query("devices", filter);
-  if (!devices.length) return void (ctx.status = 404);
-  const device = devices[0];
 
-  const validate = authorizer.getValidator("devices", device);
-  for (const t of ctx.request.body) {
-    if (!validate("task", t)) {
-      logUnauthorizedWarning(log);
-      return void (ctx.status = 403);
+  const socketTimeout: number = ctx.socket.timeout;
+
+  // Extend socket timeout while waiting for session
+  if (socketTimeout) ctx.socket.setTimeout(300000);
+
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000, 30000);
+  if (!token) {
+    log.message += " failed";
+    logger.accessWarn(log);
+    ctx.body = "Device is in session";
+    ctx.status = 503;
+    // Restore socket timeout
+    if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
+    return;
+  }
+
+  let device;
+
+  let statuses: { _id: string; status: string }[];
+
+  try {
+    const filter = and(authorizer.getFilter("devices", 3), [
+      "=",
+      ["PARAM", "DeviceID.ID"],
+      deviceId,
+    ]);
+    device = (await db.query("devices", filter).next()).value;
+    if (!device) return void (ctx.status = 404);
+
+    const validate = authorizer.getValidator("devices", device);
+    for (const t of ctx.request.body) {
+      if (!validate("task", t)) {
+        logUnauthorizedWarning(log);
+        return void (ctx.status = 403);
+      }
     }
+
+    let tasks = ctx.request.body as Task[];
+
+    for (const task of tasks) {
+      delete task._id;
+      task["device"] = deviceId;
+    }
+
+    tasks = await apiFunctions.insertTasks(tasks);
+
+    statuses = tasks.map((t) => ({ _id: t._id, status: "pending" }));
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
   }
 
   const onlineThreshold = getConfig(
@@ -584,30 +616,46 @@ router.post("/devices/:id/tasks", async (ctx) => {
         }
       }
       return exp;
-    }
+    },
   ) as number;
 
-  const socketTimeout: number = ctx.socket["timeout"];
+  const lastInform = device["Events.Inform"].value[0] as number;
 
-  // Disable socket timeout while waiting for postTasks()
-  if (socketTimeout) ctx.socket.setTimeout(0);
+  let status = await apiFunctions.connectionRequest(deviceId, device);
+  if (!status) {
+    const sessionStarted = await apiFunctions.awaitSessionStart(
+      deviceId,
+      lastInform,
+      onlineThreshold,
+    );
+    if (!sessionStarted) {
+      status = "No contact from CPE";
+    } else {
+      const sessionEnded = await apiFunctions.awaitSessionEnd(deviceId, 120000);
+      if (!sessionEnded) status = "Session took too long to complete";
+    }
+  }
 
-  const res = await apiFunctions.postTasks(
-    ctx.params.id,
-    ctx.request.body,
-    onlineThreshold,
-    device
-  );
+  if (!status) {
+    const promises = statuses.map((t) =>
+      collections.faults.count({ _id: `${deviceId}:task_${t._id}` }),
+    );
+
+    const res = await Promise.all(promises);
+    for (const [i, r] of statuses.entries())
+      r.status = res[i] ? "fault" : "done";
+  }
+
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
 
   // Restore socket timeout
   if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
 
-  log.tasks = res.tasks.map((t) => t._id).join(",");
-
+  log.tasks = statuses.map((t) => t._id).join(",");
   logger.accessInfo(log);
 
-  ctx.set("Connection-Request", res.connectionRequest);
-  ctx.body = res.tasks;
+  ctx.set("Connection-Request", status || "OK");
+  ctx.body = statuses;
 });
 
 router.post("/devices/:id/tags", async (ctx) => {
@@ -628,10 +676,10 @@ router.post("/devices/:id/tags", async (ctx) => {
     logUnauthorizedWarning(log);
     return void (ctx.status = 403);
   }
-  const res = await db.query("devices", filter);
-  if (!res.length) return void (ctx.status = 404);
+  const { value: res } = await db.query("devices", filter).next();
+  if (!res) return void (ctx.status = 404);
 
-  const validate = authorizer.getValidator("devices", res[0]);
+  const validate = authorizer.getValidator("devices", res);
   if (!validate("tags", ctx.request.body)) {
     logUnauthorizedWarning(log);
     return void (ctx.status = 403);
@@ -652,6 +700,7 @@ router.post("/devices/:id/tags", async (ctx) => {
 });
 
 router.get("/ping/:host", async (ctx) => {
+  if (!ctx.state.user) return void (ctx.status = 401);
   return new Promise<void>((resolve) => {
     ping(ctx.params.host, (err, parsed) => {
       if (parsed) {
@@ -680,7 +729,7 @@ router.put("/users/:id/password", async (ctx) => {
       !(await apiFunctions.authLocal(
         ctx.state.configSnapshot,
         username,
-        ctx.request.body.authPassword
+        ctx.request.body.authPassword,
       ))
     ) {
       logUnauthorizedWarning(log);
@@ -693,17 +742,17 @@ router.put("/users/:id/password", async (ctx) => {
     return void (ctx.status = 403);
   }
 
-  const filter = and(authorizer.getFilter("users", 3), [
-    "=",
-    ["PARAM", RESOURCE_IDS.users],
-    username,
-  ]);
-  const res = await db.query("users", filter);
-  if (!res.length) return void (ctx.status = 404);
-
   const newPassword = ctx.request.body.newPassword;
+
   if (ctx.state.user) {
-    const validate = authorizer.getValidator("users", res[0]);
+    const filter = and(authorizer.getFilter("users", 3), [
+      "=",
+      ["PARAM", RESOURCE_IDS.users],
+      username,
+    ]);
+    const { value: res } = await db.query("users", filter).next();
+    if (!res) return void (ctx.status = 404);
+    const validate = authorizer.getValidator("users", res);
     if (!validate("password", { password: newPassword })) {
       logUnauthorizedWarning(log);
       return void (ctx.status = 403);
@@ -714,7 +763,7 @@ router.put("/users/:id/password", async (ctx) => {
   const password = await hashPassword(newPassword, salt);
   await db.putUser(username, { password, salt });
 
-  await del("presets_hash");
+  await del("ui-local-cache-hash");
 
   logger.accessInfo(log);
   ctx.body = "";

lib/ui/db.ts

@@ -1,182 +1,422 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
+import { Script } from "node:vm";
+import { Readable } from "node:stream";
+import { Collection, ObjectId, WithoutId } from "mongodb";
+import { encodeTag } from "../util.ts";
+import { evaluate } from "../common/expression/util.ts";
+import { Expression, Fault, Task } from "../types.ts";
+import { collections, filesBucket } from "../db/db.ts";
+import { convertOldPrecondition, optimizeProjection } from "../db/util.ts";
+import * as MongoTypes from "../db/types.ts";
+import { parse, parseList, stringify } from "../common/expression/parser.ts";
+import { toMongoQuery } from "../db/synth.ts";
 
-import { Db, GridFSBucket, ObjectId } from "mongodb";
-import { Script } from "vm";
-import { onConnect } from "../db";
-import * as mongodbFunctions from "../mongodb-functions";
-import * as expression from "../common/expression";
-import { QueryOptions, Expression } from "../types";
-import { Readable, Writable } from "stream";
-import { minimize } from "../common/boolean-expression";
+function processDeviceProjection(
+  projection: Record<string, 1>,
+): Record<string, 1> {
+  if (!projection) return projection;
+  const p = {};
+  for (const [k, v] of Object.entries(projection)) {
+    if (k === "DeviceID.ID") {
+      p["_id"] = 1;
+    } else if (k.startsWith("DeviceID")) {
+      p["_deviceId._SerialNumber"] = v;
+      p["_deviceId._OUI"] = v;
+      p["_deviceId._ProductClass"] = v;
+      p["_deviceId._Manufacturer"] = v;
+    } else if (k.startsWith("Tags")) {
+      p["_tags"] = v;
+    } else if (k.startsWith("Events")) {
+      p["_lastInform"] = v;
+      p["_registered"] = v;
+      p["_lastBoot"] = v;
+      p["_lastBootstrap"] = v;
+    } else {
+      p[k] = v;
+    }
+  }
 
-const RESOURCE_COLLECTION = {
-  files: "fs.files",
-};
+  return p;
+}
 
-let db: Db;
+function processDeviceSort(
+  sort: Record<string, number>,
+): Record<string, number> {
+  if (!sort) return sort;
+  const s = {};
+  for (const [k, v] of Object.entries(sort)) {
+    if (k === "DeviceID.ID") s["_id"] = v;
+    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
+    else if (k === "Events.Inform") s["_lastInform"] = v;
+    else if (k === "Events.Registered") s["_registered"] = v;
+    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
+    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
+    else s[`${k}._value`] = v;
+  }
 
-onConnect(async (_db) => {
-  db = _db;
-});
+  return s;
+}
 
-export function query(
-  resource: string,
-  filter: Expression,
-  options?: QueryOptions
-): Promise<any[]>;
-export function query(
-  resource: string,
-  filter: Expression,
-  options: QueryOptions,
-  callback: (doc: any) => void
-): Promise<void>;
-export function query(
+function parseDate(d: Date): number | string {
+  const n = +d;
+  return isNaN(n) ? "" + d : n;
+}
+
+interface FlatAttributes {
+  object?: boolean;
+  objectTimestamp?: number;
+  writable?: boolean;
+  writableTimestamp?: number;
+  value?: [string | number | boolean, string];
+  valueTimestamp?: number;
+  notification?: number;
+  notificationTimestamp?: number;
+  accessList?: string[];
+  accessListTimestamp?: number;
+}
+
+export interface FlatDevice {
+  [param: string]: FlatAttributes;
+}
+
+export function flattenDevice(device: Record<string, unknown>): FlatDevice {
+  function recursive(
+    input,
+    root: string,
+    output: FlatDevice,
+    timestamp: number,
+  ): void {
+    for (const [name, tree] of Object.entries(input)) {
+      if (!root) {
+        if (name === "_lastInform") {
+          output["Events.Inform"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_registered") {
+          output["Events.Registered"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_lastBoot") {
+          output["Events.1_BOOT"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_lastBootstrap") {
+          output["Events.0_BOOTSTRAP"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_id") {
+          output["DeviceID.ID"] = {
+            value: [tree as string, "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_deviceId") {
+          output["DeviceID.Manufacturer"] = {
+            value: [tree["_Manufacturer"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.OUI"] = {
+            value: [tree["_OUI"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.ProductClass"] = {
+            value: [tree["_ProductClass"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.SerialNumber"] = {
+            value: [tree["_SerialNumber"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_tags") {
+          output["Tags"] = {
+            writable: false,
+            writableTimestamp: timestamp,
+            object: true,
+            objectTimestamp: timestamp,
+          };
+
+          for (const t of tree as string[]) {
+            output[`Tags.${encodeTag(t)}`] = {
+              value: [true, "xsd:boolean"],
+              valueTimestamp: timestamp,
+              writable: true,
+              writableTimestamp: timestamp,
+              object: false,
+              objectTimestamp: timestamp,
+            };
+          }
+        }
+      }
+
+      if (name.startsWith("_")) continue;
+
+      let childrenTimestamp = timestamp;
+
+      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
+      else if (+input["_timestamp"] > timestamp)
+        childrenTimestamp = +input["_timestamp"];
+
+      const attrs: FlatAttributes = {};
+      if (tree["_value"] != null) {
+        attrs.value = [
+          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
+          tree["_type"],
+        ];
+        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
+        attrs.object = false;
+        attrs.objectTimestamp = childrenTimestamp;
+      } else if (tree["_object"] != null) {
+        attrs.object = tree["_object"];
+        attrs.objectTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_writable"] != null) {
+        attrs.writable = tree["_writable"];
+        attrs.writableTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_notification"] != null) {
+        attrs.notification = tree["_notification"];
+        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      if (tree["_accessList"] != null) {
+        attrs.accessList = tree["_accessList"];
+        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      const r = root ? `${root}.${name}` : name;
+      output[r] = attrs;
+
+      if (attrs.object || tree["object"] == null)
+        recursive(tree, r, output, childrenTimestamp);
+    }
+  }
+
+  const newDevice: FlatDevice = {};
+  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
+  recursive(device, "", newDevice, timestamp);
+  return newDevice;
+}
+
+function flattenFault(fault: unknown): Fault {
+  const f = Object.assign({}, fault) as Fault;
+  if (f.timestamp) f.timestamp = +f.timestamp;
+  if (f["expiry"]) f["expiry"] = +f["expiry"];
+  return f as Fault;
+}
+
+function flattenTask(task: unknown): Task {
+  const t = Object.assign({}, task) as Task;
+  t._id = "" + t._id;
+  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
+  if (t.expiry) t.expiry = +t.expiry;
+  return t;
+}
+
+function flattenPreset(
+  preset: Record<string, unknown>,
+): Record<string, unknown> {
+  const p = Object.assign({}, preset);
+  if (p.precondition) {
+    try {
+      // Try parse to check expression validity
+      parse(p.precondition as string);
+    } catch (error) {
+      p.precondition = convertOldPrecondition(
+        JSON.parse(p.precondition as string),
+      );
+      p.precondition = (p.precondition as string).length
+        ? stringify(p.precondition as Expression)
+        : "";
+    }
+  }
+
+  if (p.events) {
+    const e = [];
+    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
+    p.events = e.join(", ");
+  }
+
+  const provision = p.configurations[0];
+  if (
+    (p.configurations as any[]).length === 1 &&
+    provision.type === "provision" &&
+    provision.name &&
+    provision.name.length
+  ) {
+    p.provision = provision.name;
+    p.provisionArgs = provision.args
+      ? provision.args.map((a) => stringify(a)).join(", ")
+      : "";
+  }
+
+  delete p.configurations;
+  return p;
+}
+
+function flattenFile(file: Record<string, unknown>): Record<string, unknown> {
+  const f = {};
+  f["_id"] = file["_id"];
+  if (file.metadata) {
+    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
+    f["metadata.oui"] = file["metadata"]["oui"] || "";
+    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
+    f["metadata.version"] = file["metadata"]["version"] || "";
+  }
+  return f;
+}
+
+function preProcessPreset(data: Record<string, unknown>): MongoTypes.Preset {
+  const preset = Object.assign({}, data);
+
+  if (!preset.precondition) preset.precondition = "";
+  // Try parse to check expression validity
+  parse(preset.precondition as string);
+
+  preset.weight = parseInt(preset.weight as string) || 0;
+
+  const events = {};
+  if (preset.events) {
+    for (let e of (preset.events as string).split(",")) {
+      let v = true;
+      e = e.trim();
+      if (e.startsWith("-")) {
+        v = false;
+        e = e.slice(1).trim();
+      }
+      if (e) events[e] = v;
+    }
+  }
+
+  preset.events = events;
+
+  if (!preset.provision) throw new Error("Invalid preset provision");
+
+  const configuration = {
+    type: "provision",
+    name: preset.provision,
+    args: null,
+  };
+
+  if (preset.provisionArgs)
+    configuration.args = parseList(preset.provisionArgs as string);
+
+  delete preset.provision;
+  delete preset.provisionArgs;
+  preset.configurations = [configuration];
+  return preset as unknown as MongoTypes.Preset;
+}
+
+interface QueryOptions {
+  projection?: any;
+  skip?: number;
+  limit?: number;
+  sort?: {
+    [param: string]: number;
+  };
+}
+
+export async function* query(
   resource: string,
   filter: Expression,
   options?: QueryOptions,
-  callback?: (doc: any) => void
-): Promise<void | any[]> {
+): AsyncGenerator<any, void, undefined> {
   options = options || {};
-  let q;
-  filter = expression.evaluate(filter, null, Date.now());
-  filter = minimize(filter, true);
+  filter = evaluate(filter, null, Date.now());
+  const q = toMongoQuery(filter, resource);
+  if (!q) return;
 
-  if (Array.isArray(filter)) {
-    if (resource === "devices") {
-      filter = mongodbFunctions.processDeviceFilter(filter);
-    } else if (resource === "tasks") {
-      filter = mongodbFunctions.processTasksFilter(filter);
-    } else if (resource === "faults") {
-      filter = mongodbFunctions.processFaultsFilter(filter);
-    } else if (resource === "users") {
-      // Protect against brute force, and dictionary attacks
-      const params = expression.extractParams(filter);
-      if (params.includes("password") || params.includes("salt"))
-        return Promise.reject(new Error("Invalid users filter"));
-    }
+  const collection = collections[resource] as Collection<any>;
+  const cursor = collection.find(q);
+  if (options.projection) {
+    let projection = options.projection;
+    if (resource === "devices")
+      projection = processDeviceProjection(options.projection);
 
-    q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (!filter) {
-    return Promise.resolve([]);
+    if (resource === "presets") projection.configurations = 1;
+    projection = optimizeProjection(projection);
+    cursor.project(projection);
   }
 
-  return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-    const cursor = collection.find(q);
-    if (options.projection) {
-      let projection = options.projection;
-      if (resource === "devices") {
-        projection = mongodbFunctions.processDeviceProjection(
-          options.projection
-        );
-      }
+  if (resource === "users") cursor.project({ password: 0, salt: 0 });
 
-      if (resource === "presets") projection.configurations = 1;
-      cursor.project(projection);
-    }
+  if (options.skip) cursor.skip(options.skip);
+  if (options.limit) cursor.limit(options.limit);
 
-    if (resource === "users") cursor.project({ password: 0, salt: 0 });
-
-    if (options.skip) cursor.skip(options.skip);
-    if (options.limit) cursor.limit(options.limit);
-
-    if (options.sort) {
-      let s = Object.entries(options.sort)
-        .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
-        .reduce(
-          (obj, [k, v]) =>
-            Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
-          {}
-        );
-
-      if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
-      cursor.sort(s);
-    }
-
-    if (!callback) {
-      cursor.toArray((err, docs) => {
-        if (err) return reject(err);
-        if (resource === "devices")
-          docs = docs.map((d) => mongodbFunctions.flattenDevice(d));
-        else if (resource === "faults")
-          docs = docs.map((d) => mongodbFunctions.flattenFault(d));
-        else if (resource === "tasks")
-          docs = docs.map((d) => mongodbFunctions.flattenTask(d));
-        else if (resource === "presets")
-          docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
-        else if (resource === "files")
-          docs = docs.map((d) => mongodbFunctions.flattenFile(d));
-        return resolve(docs);
-      });
-    } else {
-      cursor.forEach(
-        (doc) => {
-          if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc);
-          else if (resource === "faults")
-            doc = mongodbFunctions.flattenFault(doc);
-          else if (resource === "tasks")
-            doc = mongodbFunctions.flattenTask(doc);
-          else if (resource === "presets")
-            doc = mongodbFunctions.flattenPreset(doc);
-          else if (resource === "files")
-            doc = mongodbFunctions.flattenFile(doc);
-          callback(doc);
-        },
-        (err) => {
-          if (err) reject(err);
-          else resolve();
-        }
+  if (options.sort) {
+    let s = Object.entries(options.sort)
+      .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
+      .reduce(
+        (obj, [k, v]) =>
+          Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
+        {},
       );
-    }
-  });
+
+    if (resource === "devices") s = processDeviceSort(s);
+    cursor.sort(s);
+  }
+
+  for await (let doc of cursor) {
+    if (resource === "devices") doc = flattenDevice(doc);
+    else if (resource === "faults") doc = flattenFault(doc);
+    else if (resource === "tasks") doc = flattenTask(doc);
+    else if (resource === "presets") doc = flattenPreset(doc);
+    else if (resource === "files") doc = flattenFile(doc);
+
+    yield doc;
+  }
 }
 
 export function count(resource: string, filter: Expression): Promise<number> {
-  let q;
-  filter = expression.evaluate(filter, null, Date.now());
-  filter = minimize(filter, true);
-
-  if (Array.isArray(filter)) {
-    if (resource === "devices")
-      filter = mongodbFunctions.processDeviceFilter(filter);
-    else if (resource === "tasks")
-      filter = mongodbFunctions.processTasksFilter(filter);
-    else if (resource === "faults")
-      filter = mongodbFunctions.processFaultsFilter(filter);
-    q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (!filter) {
-    return Promise.resolve(0);
-  }
-
-  return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-    collection.find(q).count((err, c) => {
-      if (err) reject(err);
-      else resolve(c);
-    });
-  });
+  const collection = collections[resource] as Collection<any>;
+  filter = evaluate(filter, null, Date.now());
+  const q = toMongoQuery(filter, resource);
+  if (!q) return Promise.resolve(0);
+  return collection.countDocuments(q);
 }
 
 export async function updateDeviceTags(
   deviceId: string,
-  tags: Record<string, boolean>
+  tags: Record<string, boolean>,
 ): Promise<void> {
   const add = [];
   const pull = [];
@@ -186,54 +426,29 @@ export async function updateDeviceTags(
     if (onOff) add.push(tag);
     else pull.push(tag);
   }
-
-  const collection = db.collection("devices");
   const object = {};
 
   if (add?.length) object["$addToSet"] = { _tags: { $each: add } };
   if (pull?.length) object["$pullAll"] = { _tags: pull };
 
-  await collection.updateOne({ _id: deviceId }, object);
+  await collections.devices.updateOne({ _id: deviceId }, object);
 }
 
-function putResource(resource, id, object): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-    collection.replaceOne({ _id: id }, object, { upsert: true }, (err) => {
-      if (err) return void reject(err);
-      resolve();
-    });
-  });
-}
-
-function deleteResource(
-  resource: string,
-  id: string | ObjectId
-): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-    collection.deleteOne({ _id: id }, (err) => {
-      if (err) return void reject(err);
-      resolve();
-    });
-  });
-}
-
-export function putPreset(
+export async function putPreset(
   id: string,
-  object: Record<string, unknown>
+  object: Record<string, unknown>,
 ): Promise<void> {
-  object = mongodbFunctions.preProcessPreset(object);
-  return putResource("presets", id, object);
+  const p = preProcessPreset(object);
+  await collections.presets.replaceOne({ _id: id }, p, { upsert: true });
 }
 
-export function deletePreset(id: string): Promise<void> {
-  return deleteResource("presets", id);
+export async function deletePreset(id: string): Promise<void> {
+  await collections.presets.deleteOne({ _id: id });
 }
 
-export function putProvision(
+export async function putProvision(
   id: string,
-  object: Record<string, unknown>
+  object: { script: string },
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
@@ -244,21 +459,23 @@ export function putProvision(
   } catch (err) {
     if (err.stack?.startsWith(`${id}:`)) {
       return Promise.reject(
-        new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`)
+        new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`),
       );
     }
     return Promise.reject(err);
   }
-  return putResource("provisions", id, object);
+  await collections.provisions.replaceOne({ _id: id }, object, {
+    upsert: true,
+  });
 }
 
-export function deleteProvision(id: string): Promise<void> {
-  return deleteResource("provisions", id);
+export async function deleteProvision(id: string): Promise<void> {
+  await collections.provisions.deleteOne({ _id: id });
 }
 
-export function putVirtualParameter(
+export async function putVirtualParameter(
   id: string,
-  object: Record<string, unknown>
+  object: { script: string },
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
@@ -269,103 +486,108 @@ export function putVirtualParameter(
   } catch (err) {
     if (err.stack?.startsWith(`${id}:`)) {
       return Promise.reject(
-        new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`)
+        new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`),
       );
     }
     return Promise.reject(err);
   }
-  return putResource("virtualParameters", id, object);
-}
-
-export function deleteVirtualParameter(id: string): Promise<void> {
-  return deleteResource("virtualParameters", id);
-}
-
-export function putConfig(
-  id: string,
-  object: Record<string, unknown>
-): Promise<void> {
-  return putResource("config", id, object);
-}
-
-export function deleteConfig(id: string): Promise<void> {
-  return deleteResource("config", id);
-}
-
-export function putPermission(
-  id: string,
-  object: Record<string, unknown>
-): Promise<void> {
-  return putResource("permissions", id, object);
-}
-
-export function deletePermission(id: string): Promise<void> {
-  return deleteResource("permissions", id);
-}
-
-export function putUser(
-  id: string,
-  object: Record<string, unknown>
-): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const collection = db.collection("users");
-    // update instead of replace to keep the password if not set by user
-    collection.updateOne(
-      { _id: id },
-      { $set: object },
-      { upsert: true },
-      (err) => {
-        if (err) return void reject(err);
-        resolve();
-      }
-    );
+  await collections.virtualParameters.replaceOne({ _id: id }, object, {
+    upsert: true,
   });
 }
 
-export function deleteUser(id: string): Promise<void> {
-  return deleteResource("users", id);
+export async function deleteVirtualParameter(id: string): Promise<void> {
+  await collections.virtualParameters.deleteOne({ _id: id });
+}
+
+export async function putConfig(
+  id: string,
+  object: WithoutId<MongoTypes.Config>,
+): Promise<void> {
+  await collections.config.replaceOne({ _id: id }, object, { upsert: true });
+}
+
+export async function deleteConfig(id: string): Promise<void> {
+  await collections.config.deleteOne({ _id: id });
+}
+
+export async function putPermission(
+  id: string,
+  object: WithoutId<MongoTypes.Permission>,
+): Promise<void> {
+  await collections.permissions.replaceOne({ _id: id }, object, {
+    upsert: true,
+  });
+}
+
+export async function deletePermission(id: string): Promise<void> {
+  await collections.permissions.deleteOne({ _id: id });
+}
+
+export async function putUser(
+  id: string,
+  object: Partial<WithoutId<MongoTypes.User>>,
+): Promise<void> {
+  // update instead of replace to keep the password if not set by user
+  await collections.users.updateOne(
+    { _id: id },
+    { $set: object },
+    { upsert: true },
+  );
+}
+
+export async function deleteUser(id: string): Promise<void> {
+  await collections.users.deleteOne({ _id: id });
 }
 
 export function downloadFile(filename: string): Readable {
-  const bucket = new GridFSBucket(db);
-  return bucket.openDownloadStreamByName(filename);
+  return filesBucket.openDownloadStreamByName(filename);
 }
 
 export function putFile(
   filename: string,
   metadata: Record<string, string>,
-  contentStream: Readable
+  contentStream: Readable,
 ): Promise<void> {
   return new Promise((resolve, reject) => {
-    const bucket = new GridFSBucket(db);
-    const uploadStream = bucket.openUploadStreamWithId(
+    const uploadStream = filesBucket.openUploadStreamWithId(
       filename as unknown as ObjectId,
       filename,
       {
         metadata: metadata,
-      }
+      },
     );
-    uploadStream.on("error", reject);
-    contentStream.on("error", reject);
-    uploadStream.on("finish", resolve);
-    contentStream.pipe(uploadStream as Writable);
-  });
-}
 
-export function deleteFile(filename: string): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const bucket = new GridFSBucket(db);
-    bucket.delete(filename as any, (err) => {
-      if (err) return void reject(err);
-      resolve();
+    let readableEnded = false;
+    contentStream.on("end", () => {
+      readableEnded = true;
     });
+    contentStream.on("close", () => {
+      // In Node versions prior to 15, the stream will not emit an error if the
+      // connection is closed before the stream is finished.
+      // For Node 12.9+ we can just use stream.readableEnded
+      if (!readableEnded)
+        uploadStream.destroy(new Error("Stream closed prematurely"));
+    });
+
+    contentStream.on("error", (err) => {
+      uploadStream.destroy(err);
+    });
+
+    uploadStream.on("error", reject);
+    uploadStream.on("finish", resolve);
+    contentStream.pipe(uploadStream);
   });
 }
 
-export function deleteFault(id: string): Promise<void> {
-  return deleteResource("faults", id);
+export async function deleteFile(filename: string): Promise<void> {
+  await filesBucket.delete(filename as any);
 }
 
-export function deleteTask(id: ObjectId): Promise<void> {
-  return deleteResource("tasks", id);
+export async function deleteFault(id: string): Promise<void> {
+  await collections.faults.deleteOne({ _id: id });
+}
+
+export async function deleteTask(id: ObjectId): Promise<void> {
+  await collections.tasks.deleteOne({ _id: id });
 }

lib/ui/local-cache.ts

@@ -0,0 +1,187 @@
+import * as crypto from "node:crypto";
+import * as config from "../config.ts";
+import { collections } from "../db/db.ts";
+import * as expression from "../common/expression/util.ts";
+import { parse } from "../common/expression/parser.ts";
+import { Expression, Users, Permissions, Config, UiConfig } from "../types.ts";
+import { LocalCache } from "../local-cache.ts";
+
+interface Snapshot {
+  permissions: Permissions;
+  users: Users;
+  config: Config;
+  ui: UiConfig;
+}
+
+async function fetchPermissions(): Promise<[string, Permissions]> {
+  const perms = await collections.permissions.find().toArray();
+  perms.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(perms))
+    .digest("hex");
+  const permissions: Permissions = {};
+
+  for (const p of perms) {
+    if (!permissions[p.role]) permissions[p.role] = {};
+    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
+
+    permissions[p.role][p.access][p.resource] = {
+      access: p.access,
+      filter: parse(p.filter || "true"),
+    };
+    if (p.validate)
+      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
+  }
+
+  return [h, permissions];
+}
+
+async function fetchUsers(): Promise<[string, Users]> {
+  const _users = await collections.users.find().toArray();
+  _users.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(_users))
+    .digest("hex");
+  const users = {};
+
+  for (const user of _users) {
+    users[user._id] = {
+      password: user.password,
+      salt: user.salt,
+      roles: user.roles.split(",").map((s) => s.trim()),
+    };
+  }
+
+  return [h, users];
+}
+
+async function fetchConfig(): Promise<[string, Config, UiConfig]> {
+  const conf = await collections.config.find().toArray();
+  conf.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(conf)).digest("hex");
+
+  const ui = {
+    filters: {},
+    device: {},
+    index: {},
+    overview: {},
+    pageSize: null,
+  };
+
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    const val = expression.evaluate(parse(c.value));
+    _config[c._id] = val;
+    if (c._id.startsWith("ui.")) {
+      const keys = c._id.split(".");
+      if (!(keys[1] in ui)) continue;
+      // remove the first key(ui)
+      keys.shift();
+      let ref = ui;
+      while (keys.length > 1) {
+        const k = keys.shift();
+        if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
+        ref = ref[k];
+      }
+      ref[keys[0]] = val;
+    }
+  }
+
+  return [h, _config, ui];
+}
+
+const localCache = new LocalCache<Snapshot>("ui-local-cache-hash", refresh);
+
+async function refresh(): Promise<[string, Snapshot]> {
+  const res = await Promise.all([
+    fetchPermissions(),
+    fetchUsers(),
+    fetchConfig(),
+  ]);
+
+  const h = crypto.createHash("md5");
+  for (const r of res) h.update(r[0]);
+
+  const snapshot = {
+    permissions: res[0][1],
+    users: res[1][1],
+    config: res[2][1],
+    ui: res[2][2],
+  };
+
+  return [h.digest("hex"), snapshot];
+}
+
+export async function getRevision(): Promise<string> {
+  return await localCache.getRevision();
+}
+
+export function getConfig(
+  revision: string,
+  key: string,
+  context: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Expression,
+): string | number | boolean | null {
+  const snapshot = localCache.get(revision);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const oldOpts = {
+    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
+    "cwmp.debug": "DEBUG",
+    "cwmp.retryDelay": "RETRY_DELAY",
+    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
+    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
+    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
+    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
+    "cwmp.cookiesPath": "COOKIES_PATH",
+    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
+    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
+    "cwmp.connectionRequestAllowBasicAuth":
+      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
+    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
+    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
+  };
+
+  if (!(key in snapshot.config)) {
+    if (key in oldOpts) {
+      let id;
+      if (context?.["id"]) {
+        id = context["id"];
+      } else if (cb) {
+        id = cb(["PARAM", "DeviceID.ID"]);
+        if (Array.isArray(id)) id = null;
+      }
+      return config.get(oldOpts[key], id);
+    }
+    return null;
+  }
+
+  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+  return Array.isArray(v) ? null : v;
+}
+
+export function getConfigExpression(revision: string, key: string): Expression {
+  const snapshot = localCache.get(revision);
+  return snapshot.config[key];
+}
+
+export function getUsers(revision: string): Users {
+  const snapshot = localCache.get(revision);
+  return snapshot.users;
+}
+
+export function getPermissions(revision: string): Permissions {
+  const snapshot = localCache.get(revision);
+  return snapshot.permissions;
+}
+
+export function getUiConfig(revision: string): UiConfig {
+  const snapshot = localCache.get(revision);
+  return snapshot.ui;
+}

lib/util.ts

@@ -1,24 +1,7 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
+import { EventEmitter } from "node:events";
 
 export function generateDeviceId(
-  deviceIdStruct: Record<string, string>
+  deviceIdStruct: Record<string, string>,
 ): string {
   // Escapes everything except alphanumerics and underscore
   function esc(str): string {
@@ -52,7 +35,7 @@ export function encodeTag(tag: string): string {
   return encodeURIComponent(tag)
     .replace(
       /[!~*'().]/g,
-      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
+      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase(),
     )
     .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")
     .replace(/%/g, "0x");
@@ -61,3 +44,27 @@ export function encodeTag(tag: string): string {
 export function decodeTag(tag: string): string {
   return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%"));
 }
+
+export function once(
+  emitter: EventEmitter,
+  event: string,
+  timeout: number,
+): Promise<unknown[]> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(new Error(`Event ${event} timed out after ${timeout} ms`));
+    }, timeout);
+
+    emitter.once(event, (...args: unknown[]) => {
+      clearTimeout(timer);
+      resolve(args);
+    });
+  });
+}
+
+export function setTimeoutPromise(delay: number, ref = true): Promise<void> {
+  return new Promise((resolve) => {
+    const timerId = setTimeout(resolve, delay);
+    if (!ref) timerId.unref();
+  });
+}

lib/versioned-map.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 const NONEXISTENT = Symbol();
 const UNDEFINED = undefined;
 
@@ -26,10 +7,10 @@ interface Revisions<V> {
 }
 
 export default class VersionedMap<K, V> {
-  private declare _sizeDiff: number[];
-  private declare _revision: number;
-  private declare map: Map<K, (V | symbol)[]>;
-  public declare dirty: number;
+  declare private _sizeDiff: number[];
+  declare private _revision: number;
+  declare private map: Map<K, (V | symbol)[]>;
+  declare public dirty: number;
 
   public constructor() {
     this._sizeDiff = [0];
@@ -131,6 +112,8 @@ export default class VersionedMap<K, V> {
       prev = v;
     }
 
+    if (prev === NONEXISTENT && !res.delete) return null;
+
     return res;
   }
 
@@ -138,9 +121,11 @@ export default class VersionedMap<K, V> {
     const del = revisionsObj.delete || 0;
     const mutations = Object.keys(revisionsObj).reduce(
       (acc, cur) => (cur === "delete" ? acc : acc | (1 << +cur)),
-      del
+      del,
     );
 
+    if (!mutations) return;
+
     const revisions = [];
 
     let prev: V | symbol = NONEXISTENT;
@@ -168,6 +153,7 @@ export default class VersionedMap<K, V> {
 
   public *diff(): IterableIterator<[K, V, V]> {
     for (const [key, revisions] of this.map) {
+      if (revisions.length <= 1) continue;
       let first = revisions[0];
       let last = revisions[revisions.length - 1];
       if (first === NONEXISTENT && last === NONEXISTENT) continue;

lib/xml-parser.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 const CHAR_SINGLE_QUOTE = 39;
 const CHAR_DOUBLE_QUOTE = 34;
 const CHAR_LESS_THAN = 60;

lib/xmpp-client.ts

@@ -0,0 +1,460 @@
+import * as net from "node:net";
+import * as tls from "node:tls";
+import { EventEmitter } from "node:events";
+import { createHash, createHmac, randomBytes } from "node:crypto";
+import { parseXml, Element, parseAttrs } from "./xml-parser.ts";
+
+function encodeBase64(str: string): string {
+  return Buffer.from(str).toString("base64");
+}
+
+function decodeBase64(str: string): string {
+  return Buffer.from(str, "base64").toString();
+}
+
+function detectStreamTag(data: string): number {
+  const i1 = data.indexOf("<stream:stream");
+  if (i1 < 0) throw new Error("Cannot detect opening stream tag");
+  const i2 = data.indexOf(">", i1);
+  if (i2 < 0) throw new Error("Cannot detect opening stream tag");
+  return i2 + 1;
+}
+
+function xmppStream<T>(
+  socket: net.Socket,
+  callback: Generator<void, T, Element>,
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const onError = (err: Error): void => {
+      socket.removeListener("error", onError);
+      reject(err);
+    };
+    socket.on("error", onError);
+
+    const onData = (chunk: Buffer): void => {
+      try {
+        const str = chunk.toString("utf8");
+        const xml = parseXml(str);
+        const { value, done } = callback.next(xml.children[0]);
+        if (done) {
+          socket.removeListener("error", onError);
+          socket.removeListener("data", onData);
+          resolve(value as T);
+        }
+      } catch (err) {
+        socket.removeListener("error", onError);
+        socket.removeListener("data", onData);
+        reject(err);
+      }
+    };
+
+    socket.once("data", (chunk: Buffer) => {
+      try {
+        const str = chunk.toString("utf8");
+        const i = detectStreamTag(str);
+        const streamTagStr = str.slice(0, i);
+        const xml = parseXml(streamTagStr + "</stream:stream>");
+        callback.next(xml.children[0]);
+        chunk = chunk.slice(Buffer.byteLength(streamTagStr));
+        if (chunk.length) onData(chunk);
+        socket.on("data", onData);
+      } catch (err) {
+        socket.removeListener("error", onError);
+        socket.removeListener("data", onData);
+        reject(err);
+      }
+    });
+
+    try {
+      callback.next();
+    } catch (err) {
+      socket.removeListener("error", onError);
+      socket.removeListener("data", onData);
+      reject(err);
+    }
+  });
+}
+
+const INT_1 = Buffer.from([0, 0, 0, 1]);
+const saltedPasswordCache = {
+  password: "",
+  iterationCount: 0,
+  saltBase64: "",
+  salted: Buffer.allocUnsafe(0),
+};
+
+function saltPassword(
+  password: string,
+  saltBase64: string,
+  iteractionCount: number,
+): Buffer {
+  if (
+    password === saltedPasswordCache.password &&
+    saltBase64 === saltedPasswordCache.saltBase64 &&
+    iteractionCount === saltedPasswordCache.iterationCount
+  )
+    return saltedPasswordCache.salted;
+
+  const hi = createHmac("sha1", password)
+    .update(Buffer.concat([Buffer.from(saltBase64, "base64"), INT_1]))
+    .digest();
+  let hi2: Buffer = hi;
+  for (let i = 1; i < iteractionCount; ++i) {
+    hi2 = createHmac("sha1", password).update(hi2).digest();
+    for (const [j, b] of hi2.entries()) hi[j] ^= b;
+  }
+
+  saltedPasswordCache.saltBase64 = saltBase64;
+  saltedPasswordCache.password = password;
+  saltedPasswordCache.iterationCount = iteractionCount;
+  saltedPasswordCache.salted = hi;
+
+  return hi;
+}
+
+function* loginPlain(
+  socket: net.Socket,
+  username: string,
+  password: string,
+): Generator<void, void, Element> {
+  socket.write(
+    `<auth xmlns="urn:ietf:params:xml:ns:xmpp-sasl" mechanism="PLAIN">${encodeBase64(
+      `\x00${username}\x00${password}`,
+    )}</auth>`,
+  );
+  const res1 = yield;
+
+  if (
+    res1.name === "failure" &&
+    res1.children.some((c) => c.name === "not-authorized")
+  )
+    throw new Error("Not authorized");
+
+  if (res1.name !== "success")
+    throw new Error(`Unexpected response ${res1.name}`);
+}
+
+function* loginScram(
+  socket: net.Socket,
+  username: string,
+  password: string,
+): Generator<void, void, Element> {
+  const cnonce = randomBytes(8).toString("base64");
+  const gs2Header = "n,,";
+  const clientFirstMessageBare = `n=${username},r=${cnonce}`;
+  const clientFirstMessage = gs2Header + clientFirstMessageBare;
+  socket.write(
+    `<auth xmlns="urn:ietf:params:xml:ns:xmpp-sasl" mechanism="SCRAM-SHA-1">${encodeBase64(
+      clientFirstMessage,
+    )}</auth>`,
+  );
+  const res1 = yield;
+  if (res1.name !== "challenge")
+    throw new Error(`Unexpected element ${res1.name}`);
+  const serverFirstMessage = decodeBase64(res1.text);
+
+  let iterationCount: number;
+  let saltBase64: string;
+  let nonce: string;
+  for (const s of serverFirstMessage.split(",")) {
+    if (s.startsWith("i=")) iterationCount = parseInt(s.slice(2));
+    else if (s.startsWith("s=")) saltBase64 = s.slice(2);
+    else if (s.startsWith("r=")) nonce = s.slice(2);
+  }
+  if (iterationCount == null || isNaN(iterationCount))
+    throw new Error("Invalid iteration count");
+  if (saltBase64 == null) throw new Error("Missing salt");
+  if (nonce == null) throw new Error("Missing nonce");
+
+  const saltedPassword = saltPassword(password, saltBase64, iterationCount);
+  const clientKey = createHmac("sha1", saltedPassword)
+    .update("Client Key")
+    .digest();
+  const storedKey = createHash("sha1").update(clientKey).digest();
+
+  const clientFinalMessageWithoutProof = `c=${encodeBase64(
+    gs2Header,
+  )},r=${nonce}`;
+  const authMessage = `${clientFirstMessageBare},${serverFirstMessage},${clientFinalMessageWithoutProof}`;
+  const clientSignature = createHmac("sha1", storedKey)
+    .update(authMessage)
+    .digest();
+
+  const clientProof = Buffer.from(clientKey);
+  for (const [i, b] of clientSignature.entries()) clientProof[i] ^= b;
+
+  const clientFinalMessage = `${clientFinalMessageWithoutProof},p=${clientProof.toString(
+    "base64",
+  )}`;
+  socket.write(
+    `<response xmlns="urn:ietf:params:xml:ns:xmpp-sasl">${encodeBase64(
+      clientFinalMessage,
+    )}</response>`,
+  );
+  const res2 = yield;
+
+  if (
+    res2.name === "failure" &&
+    res2.children.some((c) => c.name === "not-authorized")
+  )
+    throw new Error("Not authorized");
+
+  if (res2.name !== "success")
+    throw new Error(`Unexpected response ${res2.name}`);
+
+  const serverKey = createHmac("sha1", saltedPassword)
+    .update("Server Key")
+    .digest();
+  const serverSignature = createHmac("sha1", serverKey)
+    .update(authMessage)
+    .digest("base64");
+
+  if (!decodeBase64(res2.text).endsWith(serverSignature))
+    throw new Error("Invalid server signature");
+}
+
+function* starttls(socket: net.Socket): Generator<void, void, Element> {
+  socket.write("<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>");
+  const res1 = yield;
+  if (res1.name !== "proceed") throw new Error("Failed to initiate STARTTLS");
+}
+
+function* bind(
+  socket: net.Socket,
+  resource: string,
+): Generator<void, void, Element> {
+  const id = randomBytes(8).toString("base64");
+  socket.write(
+    `<iq id='${id}' type='set'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'><resource>${resource}</resource></bind></iq>`,
+  );
+  const res1 = yield;
+  if (res1.name !== "iq") throw new Error(`Unexpected element ${res1.name}`);
+  const attrs1 = parseAttrs(res1.attrs);
+  const idAttr = attrs1.find((a) => a.name === "id");
+  if (!idAttr || idAttr.value !== id) throw new Error("Invalid ID");
+  const typeAttr = attrs1.find((a) => a.name === "type");
+  if (!typeAttr) throw new Error("Missing type attribute");
+  if (typeAttr.value !== "result") throw new Error("Cannot bind to resource");
+}
+
+const STATUS_RESTART_STREAM = 1;
+const STATUS_STARTTLS = 2;
+
+function* init(
+  socket: net.Socket,
+  host: string,
+  username: string,
+  password: string,
+  resource: string,
+): Generator<void, number, Element> {
+  socket.write(
+    `<?xml version='1.0'?><stream:stream from='${username}@${host}' to='${host}' version='1.0' xml:lang='en' xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams'>`,
+  );
+  const open = yield;
+  if (open.name !== "stream:stream")
+    throw new Error(`Unexpected element ${open.name}`);
+  const features = yield;
+  if (features.name !== "stream:features")
+    throw new Error(`Unexpected element ${features.name}`);
+  for (const feature of features.children) {
+    if (feature.name === "starttls") {
+      if (feature.children.some((c) => c.name === "required")) {
+        yield* starttls(socket);
+        return STATUS_STARTTLS;
+      }
+    } else if (feature.name === "mechanisms") {
+      const mechanisms: Set<string> = new Set(
+        feature.children.map((c) => c.text),
+      );
+      if (mechanisms.has("PLAIN")) {
+        yield* loginPlain(socket, username, password);
+        return STATUS_RESTART_STREAM;
+      } else if (mechanisms.has("SCRAM-SHA-1")) {
+        yield* loginScram(socket, username, password);
+        return STATUS_RESTART_STREAM;
+      } else {
+        throw new Error("No supported SASL method");
+      }
+    } else if (feature.name === "bind") {
+      yield* bind(socket, resource);
+      return 0;
+    }
+  }
+  return 0;
+}
+
+function upgradeTls(socket: net.Socket, host: string): Promise<tls.TLSSocket> {
+  return new Promise((resolve, reject) => {
+    socket.on("error", reject);
+    const newSocket = tls.connect({ socket, host }, () => {
+      socket.removeListener("error", reject);
+      resolve(newSocket);
+    });
+  });
+}
+
+interface XmppClientOptions {
+  host: string;
+  port?: number;
+  username?: string;
+  password?: string;
+  resource?: string;
+  timeout?: number;
+}
+
+export default class XmppClient extends EventEmitter {
+  private _socket: net.Socket;
+  private _host: string;
+  private _username: string;
+  private _resource: string;
+  private _iqStanzaCallbacks: Map<
+    string,
+    (err: Error, r?: { rawRes: string; res: Element }) => void
+  >;
+
+  private constructor() {
+    super();
+    this._socket = null;
+    this._host = null;
+    this._username = null;
+    this._resource = null;
+    this._iqStanzaCallbacks = new Map();
+  }
+
+  static async connect(opts: XmppClientOptions): Promise<XmppClient> {
+    function connectSocket(host: string, port: number): Promise<net.Socket> {
+      return new Promise((resolve, reject) => {
+        const socket = new net.Socket();
+        socket.on("error", reject);
+        socket.connect(port, host, () => {
+          socket.removeListener("error", reject);
+          resolve(socket);
+        });
+      });
+    }
+
+    let socket = await connectSocket(opts.host, opts.port || 5222);
+    try {
+      let status = 1;
+      while (status) {
+        if (status === STATUS_STARTTLS)
+          socket = await upgradeTls(socket, opts.host);
+        status = await xmppStream(
+          socket,
+          init(socket, opts.host, opts.username, opts.password, opts.resource),
+        );
+      }
+    } catch (err) {
+      socket.destroy();
+      throw err;
+    }
+
+    const client = new XmppClient();
+    client._socket = socket;
+    client._host = opts.host;
+    client._username = opts.username;
+    client._resource = opts.resource;
+    socket.on("data", client._onData.bind(client));
+    socket.on("error", client._onError.bind(client));
+    if (opts.timeout)
+      socket.setTimeout(opts.timeout, client.close.bind(client));
+    return client;
+  }
+
+  close(): void {
+    this._socket.end("</stream:stream>");
+    this._socket.removeAllListeners("data");
+    this._socket.removeAllListeners("error");
+    this.emit("close");
+  }
+
+  ref(): void {
+    this._socket.ref();
+  }
+
+  unref(): void {
+    this._socket.unref();
+  }
+
+  get host(): string {
+    return this._host;
+  }
+
+  get username(): string {
+    return this._username;
+  }
+
+  get resource(): string {
+    return this._resource;
+  }
+
+  private _onData(chunk: Buffer): void {
+    try {
+      let close = false;
+      let str = chunk.toString("utf8");
+      if (str.endsWith("</stream:stream>")) {
+        str = str.slice(0, -16);
+        close = true;
+      }
+      const xml = parseXml(str);
+      const idx = xml.children.map((c) => c.bodyIndex);
+      for (const [i, c] of xml.children.entries()) {
+        const s = str.slice(idx[i], idx[i + 1]);
+        if (c.name === "iq") {
+          const attrs = parseAttrs(c.attrs);
+          const id = attrs.find((a) => a.name === "id");
+          if (id) {
+            const cb = this._iqStanzaCallbacks.get(id.value);
+            if (cb) cb(null, { rawRes: s, res: c });
+          }
+        }
+        this.emit("stanza", c, s);
+      }
+      if (close) {
+        this._socket.removeAllListeners("data");
+        this._socket.removeAllListeners("error");
+        this.emit("close");
+      }
+    } catch (err) {
+      this._socket.removeAllListeners("data");
+      this._socket.removeAllListeners("error");
+      this.emit("error", err);
+    }
+  }
+
+  private _onError(err: Error): void {
+    this._socket.end();
+    this.emit("error", err);
+    for (const cb of this._iqStanzaCallbacks.values()) cb(err);
+  }
+
+  send(msg: string): void {
+    this._socket.write(msg);
+  }
+
+  sendIqStanza(
+    from: string,
+    to: string,
+    type: string,
+    body: string,
+    timeout = 3000,
+  ): Promise<{ rawReq: string; rawRes: string; res: Element }> {
+    return new Promise((resolve, reject) => {
+      const id = randomBytes(8).toString("base64");
+      const rawReq = `<iq from="${from}" to="${to}" id="${id}" type="${type}">${body}</iq>`;
+      this.send(rawReq);
+      const t = setTimeout(() => {
+        this._iqStanzaCallbacks.delete(id);
+        reject(
+          new Error("Did not receive IQ stanza response in a timely manner"),
+        );
+      }, timeout);
+      this._iqStanzaCallbacks.set(id, (err, r) => {
+        this._iqStanzaCallbacks.delete(id);
+        clearTimeout(t);
+        if (err) reject(err);
+        else resolve(Object.assign(r, { rawReq }));
+      });
+    });
+  }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.7",
+  "version": "1.2.14",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -12,73 +12,63 @@
     "CWMP",
     "ACS"
   ],
-  "author": "Zaid Abdulla <zaid@genieacs.com>",
+  "author": {
+    "name": "GenieACS Inc.",
+    "url": "https://genieacs.com"
+  },
   "license": "AGPL-3.0",
   "private": true,
-  "directories": {
-    "bin": "bin"
+  "bin": {
+    "genieacs-cwmp": "bin/genieacs-cwmp",
+    "genieacs-fs": "bin/genieacs-fs",
+    "genieacs-nbi": "bin/genieacs-nbi",
+    "genieacs-ui": "bin/genieacs-ui"
   },
   "dependencies": {
-    "espresso-iisojs": "^1.0.4",
+    "@breejs/later": "^4.2.0",
+    "@koa/bodyparser": "^5.1.2",
+    "@koa/router": "^13.1.1",
+    "bson": "^4.7.2",
+    "espresso-iisojs": "^1.0.8",
     "iconv-lite": "^0.6.3",
-    "ipaddr.js": "^2.0.1",
-    "jsonwebtoken": "^8.5.1",
-    "koa": "^2.13.1",
-    "koa-bodyparser": "^4.3.0",
-    "koa-compress": "^5.1.0",
-    "koa-jwt": "^4.0.1",
-    "koa-router": "^10.1.1",
-    "koa-static": "^5.0.0",
-    "later": "^1.2.0",
-    "mongodb": "^4.1.2",
-    "parsimmon": "^1.18.0",
+    "ipaddr.js": "^2.3.0",
+    "jsonwebtoken": "^9.0.3",
+    "koa": "^2.16.4",
+    "koa-compress": "^5.2.0",
+    "koa-jwt": "^4.0.3",
+    "koa-send": "^5.0.1",
+    "mongodb": "^4.16.0",
+    "parsimmon": "^1.18.1",
     "seedrandom": "^3.0.5"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^20.0.0",
-    "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^13.0.4",
-    "@rollup/plugin-typescript": "^8.2.5",
-    "@types/codemirror": "^5.60.2",
-    "@types/jsonwebtoken": "^8.5.5",
-    "@types/koa": "^2.13.4",
-    "@types/koa-compress": "^4.0.3",
-    "@types/koa-router": "^7.4.4",
-    "@types/koa-static": "^4.0.2",
-    "@types/mithril": "^2.0.8",
-    "@types/node": "^16.9.2",
-    "@types/parsimmon": "^1.10.6",
-    "@types/seedrandom": "^3.0.1",
-    "@typescript-eslint/eslint-plugin": "^4.31.1",
-    "@typescript-eslint/parser": "^4.31.1",
-    "ava": "^3.15.0",
-    "codemirror": "^5.62.3",
-    "cssnano": "^5.0.8",
-    "eslint": "^7.32.0",
-    "eslint-config-prettier": "^8.3.0",
-    "esm": "^3.2.25",
-    "mithril": "^2.0.4",
-    "postcss": "^8.3.6",
-    "postcss-import": "^14.0.2",
-    "postcss-preset-env": "^6.7.0",
-    "prettier": "^2.4.1",
-    "rollup": "^2.56.3",
-    "rollup-plugin-terser": "^7.0.2",
-    "sql.js": "^1.6.1",
-    "svgo": "^2.6.1",
-    "ts-node": "^10.2.1",
-    "typescript": "^4.4.3",
+    "@types/codemirror": "^5.60.17",
+    "@types/jsonwebtoken": "^9.0.10",
+    "@types/koa": "^2.15.0",
+    "@types/koa-compress": "^4.0.7",
+    "@types/mithril": "^2.2.7",
+    "@types/node": "^20.19.37",
+    "@types/parsimmon": "^1.10.9",
+    "@types/seedrandom": "^3.0.8",
+    "@typescript-eslint/eslint-plugin": "^7.18.0",
+    "@typescript-eslint/parser": "^7.18.0",
+    "codemirror": "^5.65.21",
+    "esbuild": "^0.27.4",
+    "eslint": "^8.57.1",
+    "eslint-config-prettier": "^9.1.2",
+    "mithril": "^2.3.8",
+    "prettier": "^3.8.1",
+    "sql.js": "^1.14.1",
+    "svgo": "^3.3.3",
+    "typescript": "^5.9.3",
     "yaml": "^1.10.2"
   },
   "engines": {
     "node": ">=12.13.0"
   },
   "scripts": {
-    "test": "ava --verbose --color",
-    "lint-eslint": "eslint --fix \"bin/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\"",
-    "lint-prettier": "prettier --prose-wrap always --write .",
-    "lint-tsc": "tsc --noEmit",
-    "lint": "npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
-    "build": "node -r esm -r ts-node/register/transpile-only build/build"
+    "test": "esbuild --log-level=warning --bundle --platform=node --target=node18 --packages=external --sourcemap=inline --outdir=test test/*.ts && node --test --enable-source-maps test/*.js && rm test/*.js",
+    "lint": "esbuild build/lint.ts --bundle --platform=node --target=node16 --packages=external | node",
+    "build": "esbuild build/build.ts --bundle --platform=node --target=node12 --packages=external | node"
   }
 }

test/auth.ts

@@ -1,8 +1,9 @@
-import ava from "ava";
-import { randomBytes } from "crypto";
-import * as auth from "../lib/auth";
+import test from "node:test";
+import assert from "node:assert";
+import { randomBytes } from "node:crypto";
+import * as auth from "../lib/auth.ts";
 
-ava("digest", (t) => {
+void test("digest", () => {
   const username = "test";
   const password = "test";
   const uri = "/";
@@ -17,8 +18,6 @@ ava("digest", (t) => {
     `Digest realm="${realm}",nonce="${nonce}",qop="auth-int"`,
   ];
 
-  t.plan(challenges.length);
-
   for (const challenge of challenges) {
     const wwwAuthHeader = auth.parseWwwAuthenticateHeader(challenge);
     const solution = auth.solveDigest(
@@ -27,10 +26,10 @@ ava("digest", (t) => {
       uri,
       method,
       body,
-      wwwAuthHeader
+      wwwAuthHeader,
     );
     const authHeader = auth.parseAuthorizationHeader(solution);
-    t.is(
+    assert.strictEqual(
       authHeader["response"],
       auth.digest(
         username,
@@ -42,8 +41,8 @@ ava("digest", (t) => {
         authHeader["qop"],
         body,
         authHeader["cnonce"],
-        authHeader["nc"]
-      )
+        authHeader["nc"],
+      ),
     );
   }
 });

test/boolean-expression.ts

@@ -1,118 +0,0 @@
-import ava, { TestInterface } from "ava";
-import { minimize, unionDiff } from "../lib/common/boolean-expression";
-import { parse, stringify } from "../lib/common/expression-parser";
-import initSqlJs from "sql.js";
-
-const test = ava as TestInterface<{
-  query: (filter: string) => Promise<Set<number>>;
-}>;
-
-const STRING_VALUES = [null, "", "a", "ab", "ab10", "ab-10"];
-const DECIMAL_VALUES = [null, 0, -10, 10];
-
-test.before(async (t) => {
-  const sql = await initSqlJs();
-  const db = new sql.Database();
-
-  t.context.query = async (filter: string): Promise<Set<number>> => {
-    const res = db.exec(`SELECT id FROM test WHERE ${filter}`);
-    if (!res.length) return new Set();
-    return new Set(res[0].values.flat());
-  };
-
-  db.run(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY, string STRING, decimal DECIMAL(4,2))"
-  );
-
-  const stmt = db.prepare("INSERT INTO test (string, decimal) VALUES (?, ?)");
-  const count = STRING_VALUES.length * DECIMAL_VALUES.length;
-  for (let i = 0; i < count; ++i) {
-    const str = i % STRING_VALUES.length;
-    const dec = Math.trunc(i / STRING_VALUES.length) % DECIMAL_VALUES.length;
-    stmt.run([STRING_VALUES[str], DECIMAL_VALUES[dec]]);
-  }
-  stmt.free();
-});
-
-function setsEqual(set1: Set<number>, set2: Set<number>): boolean {
-  if (set1.size !== set2.size) return false;
-  for (const s of set1) if (!set2.has(s)) return false;
-  return true;
-}
-
-function getPermutations(...arrs: any[][]): any[][] {
-  const count = arrs.reduce((total, arr) => total * arr.length, 1);
-  const res = [];
-  for (let i = 0; i < count; ++i) {
-    let j = i;
-    const row = [];
-    for (const arr of arrs) {
-      const v = arr[j % arr.length];
-      j = Math.trunc(j / arr.length);
-      row.push(v);
-    }
-    res.push(row);
-  }
-  return res;
-}
-
-test("minimize", async (t) => {
-  const cases: string[] = [];
-
-  cases.push("null");
-  cases.push("false");
-  cases.push("true");
-  cases.push("string");
-  cases.push("(string + decimal) IS NULL");
-  cases.push("(string + decimal) = NULL");
-  cases.push("COALESCE(string, decimal) = 0");
-
-  for (const [s1, s2, s3, op1, op2] of getPermutations(
-    STRING_VALUES.filter((s) => s),
-    STRING_VALUES.filter((s) => s),
-    STRING_VALUES.filter((s) => s),
-    [">", "=", "<"],
-    ["<>", ">="]
-  )) {
-    cases.push(
-      `string ${op1} "${s1}" OR string ='${s2}' OR NOT string ${op2} '${s3}'`
-    );
-  }
-
-  for (const [s1, s2] of getPermutations(
-    STRING_VALUES.filter((s) => s),
-    STRING_VALUES.filter((s) => s)
-  ))
-    cases.push(`string > "${s1}" AND string < '${s2}'`);
-
-  for (const c of cases) {
-    const res1 = await t.context.query(c);
-    const min = stringify(minimize(parse(c), true));
-    const res2 = await t.context.query(min);
-    t.true(setsEqual(res1, res2));
-  }
-});
-
-test("unionDiff", async (t) => {
-  const cases = [
-    "true",
-    "decimal > 0",
-    "decimal > 10",
-    "UPPER(string || decimal) LIKE 'AB10'",
-    "COALESCE(string, decimal) = 0",
-  ];
-
-  for (const [c1, c2] of getPermutations(cases, cases)) {
-    const res1 = await t.context.query(c1);
-    const res2 = await t.context.query(c2);
-    const [union, diff] = unionDiff(parse(c1), parse(c2));
-    const res3 = await t.context.query(stringify(union));
-    const res4 = await t.context.query(stringify(diff));
-
-    const unionSet = new Set([...res1, ...res2]);
-    const diffSet = new Set(Array.from(res2).filter((r) => !res1.has(r)));
-
-    t.true(setsEqual(res3, unionSet));
-    t.true(setsEqual(res4, diffSet));
-  }
-});

test/db.ts

@@ -0,0 +1,168 @@
+import test from "node:test";
+import assert from "node:assert";
+import { EJSON } from "bson";
+import { Filter } from "mongodb";
+import { stringify, parse } from "../lib/common/expression/parser.ts";
+import { convertOldPrecondition } from "../lib/db/util.ts";
+import { toMongoQuery } from "../lib/db/synth.ts";
+
+void test("convertOldPrecondition", () => {
+  const tests = [
+    [{}, "true"],
+    [{ test: "test" }, 'test = "test"'],
+    [{ test: { $eq: "test" } }, 'test = "test"'],
+    [{ test: { $ne: "test" } }, 'test <> "test" OR test IS NULL'],
+    [{ test: { $gte: "test" } }, 'test >= "test"'],
+    [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
+    [{ "Tags.test": false }, "Tags.test IS NULL"],
+    [{ "Tags.test": { $exists: true } }, "Tags.test IS NOT NULL"],
+    [{ "Tags.test": { $exists: false } }, "Tags.test IS NULL"],
+    [{ "Tags.test": { $ne: true } }, "Tags.test IS NULL"],
+    [{ "Tags.test": { $ne: false } }, "Tags.test IS NOT NULL"],
+    [{ "Tags.test": { $eq: true } }, "Tags.test IS NOT NULL"],
+    [{ "Tags.test": { $eq: false } }, "Tags.test IS NULL"],
+    [{ _tags: "test" }, "Tags.test IS NOT NULL"],
+    [{ _tags: { $ne: "test" } }, "Tags.test IS NULL"],
+    [{ _tags: { $eq: "test" } }, "Tags.test IS NOT NULL"],
+    [
+      { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
+      'test = "test" AND (test <> "test" OR test IS NULL)',
+    ],
+    [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
+    [
+      { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
+      'test = "test" OR (test <> "test" OR test IS NULL)',
+    ],
+    [
+      { test: { $gte: "test1", $ne: "test2" } },
+      'test >= "test1" AND (test <> "test2" OR test IS NULL)',
+    ],
+    [
+      { test: "test", test2: { $ne: "test2" } },
+      'test = "test" AND (test2 <> "test2" OR test2 IS NULL)',
+    ],
+  ];
+
+  const shouldFailTests = [
+    [{ test: { $gee: "test" } }, "Operator $gee not supported"],
+    [{ test: [] }, "Invalid type"],
+    [{ "Tags.test": { $gt: true } }, "Invalid tag query"],
+    [{ _tags: [] }, "Invalid type"],
+    [{ _tags: { $gt: "test" } }, "Invalid tag query"],
+    [{ $nor: [] }, "Operator $nor not supported"],
+  ] as [Filter<unknown>, string][];
+
+  for (const t of tests) {
+    assert.strictEqual(
+      stringify(convertOldPrecondition(t[0] as Record<string, unknown>)),
+      t[1],
+    );
+  }
+
+  for (const t of shouldFailTests) {
+    const func = (): void => {
+      convertOldPrecondition(t[0]);
+    };
+    assert.throws(func, new Error(t[1]));
+  }
+});
+
+void test("toMongoQuery", async () => {
+  const queries: [string, Filter<unknown> | false][] = [
+    ["true", {}],
+    ["Tags.tag1 = true", { _tags: { $eq: "tag1" } }],
+    ["Tags.tag1 <> false", { _tags: { $eq: "tag1" } }],
+    ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }],
+    ["Tags.tag1 = 123", false],
+    ["Param1 = 'value1'", { "Param1._value": { $eq: "value1" } }],
+    [
+      "Param1 <> 'value1'",
+      {
+        "Param1._value": { $ne: "value1" },
+        $and: [{ "Param1._value": { $ne: null } }],
+      },
+    ],
+    [
+      "Param1 <> 1657844103524",
+      {
+        "Param1._value": { $ne: 1657844103524 },
+
+        $and: [
+          {
+            "Param1._value": { $ne: { $date: "2022-07-15T00:15:03.524Z" } },
+          },
+          { "Param1._value": { $ne: null } },
+        ],
+      },
+    ],
+    [
+      "Param1 = 1657844103524",
+      {
+        $or: [
+          { "Param1._value": { $eq: { $date: "2022-07-15T00:15:03.524Z" } } },
+          { "Param1._value": { $eq: 1657844103524 } },
+        ],
+      },
+    ],
+    ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
+    ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
+    [
+      "Param1 LIKE 'value'",
+      {
+        "Param1._value": {
+          $regularExpression: { options: "s", pattern: "^value$" },
+        },
+      },
+    ],
+
+    [
+      "LOWER(Param1) LIKE 'value'",
+      {
+        "Param1._value": {
+          $regularExpression: { options: "is", pattern: "^value$" },
+        },
+      },
+    ],
+    [
+      "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')",
+      {
+        $or: [
+          {
+            "Param2._value": { $ne: null },
+            $and: [{ "Param2._value": { $ne: "value1" } }],
+            "Param1._value": { $eq: "value2" },
+          },
+          {
+            "Param1._value": { $ne: "value2" },
+            $and: [{ "Param1._value": { $ne: null } }],
+          },
+        ],
+      },
+    ],
+    [
+      "Param1 <> 'value2' OR Param1 IS NULL",
+      { "Param1._value": { $ne: "value2" } },
+    ],
+  ];
+
+  for (const [expStr, expect] of queries) {
+    const exp = parse(expStr);
+    let query = toMongoQuery(exp, "devices");
+    if (query) query = EJSON.serialize(query);
+    assert.deepStrictEqual(query, expect);
+  }
+
+  const failQueries: [any, string][] = [
+    ["Param1 = Param2", "Right-hand operand must be a literal value"],
+    ["Param1 LIKE Param2", "Right-hand operand of 'LIKE' must be a string"],
+    ["NOW() = 1", "Left-hand operand must be a parameter"],
+    ["param{param2} = 1", "Left-hand operand must be a parameter"],
+  ];
+
+  for (const [expStr, err] of failQueries) {
+    const exp = parse(expStr);
+    assert.throws(() => toMongoQuery(exp, "devices"), {
+      message: err,
+    });
+  }
+});

test/device.ts

@@ -1,28 +1,28 @@
-import ava from "ava";
-import Path from "../lib/common/path";
-import * as device from "../lib/device";
-import PathSet from "../lib/common/path-set";
-import VersionedMap from "../lib/versioned-map";
-import { Attributes } from "../lib/types";
+import test from "node:test";
+import assert from "node:assert";
+import Path from "../lib/common/path.ts";
+import * as device from "../lib/device.ts";
+import PathSet from "../lib/common/path-set.ts";
+import VersionedMap from "../lib/versioned-map.ts";
+import { Attributes } from "../lib/types.ts";
 
-ava("getAliasDeclarations", (t) => {
+void test("getAliasDeclarations", () => {
   const path = Path.parse("a.[aa:10,bb.[aaa:100].cc:1].b");
   const decs = device.getAliasDeclarations(path, 99);
 
   const expected = ["a.*.b", "a.*.aa", "a.*.bb.*.cc", "a.*.bb.*.aaa"];
 
-  t.plan(24);
   for (const [i, d] of decs.entries()) {
-    t.is(d.path.toString(), expected[i]);
-    t.is(d.pathGet, 99);
-    t.is(d.pathSet, null);
-    t.deepEqual(d.attrGet, i ? { value: 99 } : null);
-    t.is(d.attrSet, null);
-    t.is(d.defer, true);
+    assert.strictEqual(d.path.toString(), expected[i]);
+    assert.strictEqual(d.pathGet, 99);
+    assert.strictEqual(d.pathSet, null);
+    assert.deepStrictEqual(d.attrGet, i ? { value: 99 } : null);
+    assert.strictEqual(d.attrSet, null);
+    assert.strictEqual(d.defer, true);
   }
 });
 
-ava("unpack", (t) => {
+void test("unpack", () => {
   const now = Date.now();
   const deviceData = {
     paths: new PathSet(),
@@ -85,13 +85,13 @@ ava("unpack", (t) => {
   let unpacked: Path[];
   unpacked = device.unpack(
     deviceData,
-    Path.parse("a.[b:b,c:c].a.[b:b1,c:c1].a")
+    Path.parse("a.[b:b,c:c].a.[b:b1,c:c1].a"),
   );
-  t.is(unpacked.length, 2);
-  t.is(unpacked[0].toString(), "a.1.a.1.a");
-  t.is(unpacked[1].toString(), "a.2.a.1.a");
+  assert.strictEqual(unpacked.length, 2);
+  assert.strictEqual(unpacked[0].toString(), "a.1.a.1.a");
+  assert.strictEqual(unpacked[1].toString(), "a.2.a.1.a");
 
   unpacked = device.unpack(deviceData, Path.parse("a.*.a.[b:c1,c:b1].a"));
-  t.is(unpacked.length, 1);
-  t.is(unpacked[0].toString(), "a.2.a.2.a");
+  assert.strictEqual(unpacked.length, 1);
+  assert.strictEqual(unpacked[0].toString(), "a.2.a.2.a");
 });

test/mongodb-functions.ts

@@ -1,140 +0,0 @@
-import ava from "ava";
-import * as mongodbFunctions from "../lib/mongodb-functions";
-import { stringify, parse } from "../lib/common/expression-parser";
-
-ava("convertOldPrecondition", (t) => {
-  const tests = [
-    [{}, "true"],
-    [{ test: "test" }, 'test = "test"'],
-    [{ test: { $eq: "test" } }, 'test = "test"'],
-    [{ test: { $ne: "test" } }, 'test <> "test" OR test IS NULL'],
-    [{ test: { $gte: "test" } }, 'test >= "test"'],
-    [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
-    [{ "Tags.test": false }, "Tags.test IS NULL"],
-    [{ "Tags.test": { $exists: true } }, "Tags.test IS NOT NULL"],
-    [{ "Tags.test": { $exists: false } }, "Tags.test IS NULL"],
-    [{ "Tags.test": { $ne: true } }, "Tags.test IS NULL"],
-    [{ "Tags.test": { $ne: false } }, "Tags.test IS NOT NULL"],
-    [{ "Tags.test": { $eq: true } }, "Tags.test IS NOT NULL"],
-    [{ "Tags.test": { $eq: false } }, "Tags.test IS NULL"],
-    [{ _tags: "test" }, "Tags.test IS NOT NULL"],
-    [{ _tags: { $ne: "test" } }, "Tags.test IS NULL"],
-    [{ _tags: { $eq: "test" } }, "Tags.test IS NOT NULL"],
-    [
-      { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" AND (test <> "test" OR test IS NULL)',
-    ],
-    [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
-    [
-      { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" OR (test <> "test" OR test IS NULL)',
-    ],
-    [
-      { test: { $gte: "test1", $ne: "test2" } },
-      'test >= "test1" AND (test <> "test2" OR test IS NULL)',
-    ],
-    [
-      { test: "test", test2: { $ne: "test2" } },
-      'test = "test" AND (test2 <> "test2" OR test2 IS NULL)',
-    ],
-  ];
-
-  const shouldFailTests = [
-    [{ test: { $gee: "test" } }, "Operator $gee not supported"],
-    [{ test: [] }, "Invalid type"],
-    [{ "Tags.test": { $gt: true } }, "Invalid tag query"],
-    [{ _tags: [] }, "Invalid type"],
-    [{ _tags: { $gt: "test" } }, "Invalid tag query"],
-    [{ $nor: [] }, "Operator $nor not supported"],
-  ];
-
-  t.plan(tests.length + 2 * shouldFailTests.length);
-  for (const test of tests) {
-    t.is(
-      stringify(
-        mongodbFunctions.convertOldPrecondition(
-          test[0] as Record<string, unknown>
-        )
-      ),
-      test[1]
-    );
-  }
-
-  for (const test of shouldFailTests) {
-    const func = (): void => {
-      mongodbFunctions.convertOldPrecondition(
-        test[0] as Record<string, unknown>
-      );
-    };
-    const error = t.throws(func, { instanceOf: Error });
-    t.is(error.message, test[1]);
-  }
-});
-
-ava("filterToMongoQuery", async (t) => {
-  const queries: [string, Record<string, unknown>][] = [
-    ["true", {}],
-    ["Tags.tag1 = true", { _tags: "tag1" }],
-    ["Tags.tag1 <> false", { _tags: "tag1" }],
-    ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }],
-    ["Tags.tag1 = 123", { "Tags.tag1": 123 }],
-    ["Param1 = 'value1'", { "Param1._value": "value1" }],
-    ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }],
-    ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
-    ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
-    ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }],
-    ["LOWER(Param1) LIKE 'value'", { "Param1._value": /^value$/i }],
-    [
-      "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')",
-      {
-        $or: [
-          { "Param1._value": { $nin: ["value2", null] } },
-          {
-            $and: [
-              { "Param2._value": { $nin: ["value1", null] } },
-              { "Param1._value": { $gte: "value2" } },
-            ],
-          },
-        ],
-      },
-    ],
-    [
-      "Param1 <> 'value2' OR Param1 IS NULL",
-      {
-        $or: [
-          { "Param1._value": { $nin: ["value2", null] } },
-          { "Param1._value": null },
-        ],
-      },
-    ],
-  ];
-
-  for (const [expStr, expect] of queries) {
-    const exp = parse(expStr);
-    const query = mongodbFunctions.filterToMongoQuery(
-      mongodbFunctions.processDeviceFilter(exp)
-    );
-    t.deepEqual(query, expect);
-  }
-
-  const failQueries: [any, string][] = [
-    ["Param1 = Param2", "Invalid RHS operand of = clause"],
-    ["Param1 LIKE Param2", "Invalid RHS operand of LIKE clause"],
-    ["1 LIKE '1'", "Invalid LHS operand of LIKE clause"],
-    ["1 = 2", "Invalid LHS operand of = clause"],
-    ["1 IS NULL", "Invalid LHS operand of IS NULL clause"],
-    ["false", "Primitives are not valid queries"],
-    ["UPPER(Param1) LIKE 'value'", "Invalid RHS operand of LIKE clause"],
-  ];
-
-  for (const [expStr, err] of failQueries) {
-    const exp = parse(expStr);
-    t.throws(
-      () =>
-        mongodbFunctions.filterToMongoQuery(
-          mongodbFunctions.processDeviceFilter(exp)
-        ),
-      { message: err }
-    );
-  }
-});

test/pagination.ts

@@ -0,0 +1,120 @@
+import test from "node:test";
+import assert from "node:assert";
+import initSqlJs from "sql.js/dist/sql-asm.js";
+import { parse, stringify } from "../lib/common/expression/parser.ts";
+import {
+  bookmarkToExpression,
+  paginate,
+  toBookmark,
+} from "../lib/common/expression/pagination.ts";
+import { Expression } from "../lib/types.ts";
+import { covers, minimize } from "../lib/common/expression/synth.ts";
+
+const VALUES = [null, -1, false, "a"];
+const PARAMS = ["param1", "param2"];
+
+let db;
+
+async function query(filter: string): Promise<{ id: string }[]> {
+  if (!db) {
+    const sql = await initSqlJs();
+    db = new sql.Database();
+
+    db.run(`CREATE TABLE test (id INTEGER PRIMARY KEY, ${PARAMS.join(", ")})`);
+
+    const stmt = db.prepare(
+      `INSERT INTO test (${PARAMS.join(", ")}) VALUES (${PARAMS.map(
+        () => "?",
+      ).join(", ")})`,
+    );
+    const count = VALUES.length ** PARAMS.length;
+    for (let i = 0; i < count; ++i) {
+      const values: (boolean | number | string)[] = [];
+      for (let j = 0; j < PARAMS.length; ++j)
+        values.push(VALUES[Math.trunc(i / VALUES.length ** j) % VALUES.length]);
+      stmt.run(values);
+    }
+    stmt.free();
+  }
+
+  const res = db.exec(`SELECT * FROM test WHERE ${filter}`);
+  if (!res.length) return [];
+  return res[0].values.map((row) =>
+    Object.fromEntries(row.map((v, i) => [res[0].columns[i], v])),
+  );
+}
+
+function getAllSortOrders(columns: string[]): Array<Record<string, number>> {
+  const sortOrders: Array<Record<string, number>> = [];
+
+  function generateOrders(
+    remaining: string[],
+    current: Record<string, number>,
+  ): void {
+    if (remaining.length === 0) {
+      sortOrders.push({ ...current });
+      return;
+    }
+
+    for (const column of remaining) {
+      const newRemaining = remaining.filter((c) => c !== column);
+      current[column] = -1;
+      generateOrders(newRemaining, current);
+      current[column] = 1;
+      generateOrders(newRemaining, current);
+      delete current[column];
+    }
+  }
+
+  generateOrders(columns, {});
+  return sortOrders;
+}
+
+async function testPaginate(
+  q1: Expression,
+  q2: Expression,
+  sort: Record<string, number>,
+): Promise<void> {
+  const orderBy = Object.entries(sort)
+    .map(([k, v]) => `${k} ${v > 0 ? "ASC" : "DESC"}`)
+    .join(", ");
+
+  const allMatches = await query(`${stringify(q2)} ORDER BY ${orderBy}`);
+  const [fulfilled, diff] = paginate(q1, q2, sort);
+  assert.ok(covers(q1, fulfilled));
+
+  const fulfilledMatches = await query(
+    `${stringify(fulfilled)} ORDER BY ${orderBy}`,
+  );
+
+  const diffMatches = await query(`${stringify(diff)} ORDER BY ${orderBy}`);
+  assert.deepStrictEqual(allMatches, [...fulfilledMatches, ...diffMatches]);
+
+  if (allMatches.length === fulfilledMatches.length) return;
+
+  const nextMatches = allMatches.slice(
+    fulfilledMatches.length,
+    fulfilledMatches.length + 1,
+  );
+  const bookmark = toBookmark(sort, nextMatches[nextMatches.length - 1]);
+  const bookmarkFilter = bookmarkToExpression(bookmark, sort);
+  const capped = ["AND", q2, bookmarkFilter];
+
+  assert.ok(!covers(q1, capped));
+  const cappedMatches = await query(
+    `(${stringify(capped)}) ORDER BY ${orderBy}`,
+  );
+  assert.deepStrictEqual(cappedMatches, [...fulfilledMatches, ...nextMatches]);
+  const min = minimize(["OR", q1, capped]);
+  await testPaginate(min, q2, sort);
+}
+
+void test("paginate", async () => {
+  const cases: [string, string][] = [["param2 < 'a'", "param1 >= 'a'"]];
+  const params = ["id", ...PARAMS];
+  const sortOrders = getAllSortOrders(params);
+
+  for (const [q1, q2] of cases)
+    for (const sort of sortOrders)
+      await testPaginate(parse(q1), parse(q2), sort);
+});

test/path-set.ts

@@ -1,34 +1,35 @@
-import ava from "ava";
-import Path from "../lib/common/path";
-import PathSet from "../lib/common/path-set";
+import test from "node:test";
+import assert from "node:assert";
+import Path from "../lib/common/path.ts";
+import PathSet from "../lib/common/path-set.ts";
 
-ava("depth", (t) => {
+void test("depth", () => {
   const pathSet = new PathSet();
-  t.is(pathSet.depth, 0);
+  assert.strictEqual(pathSet.depth, 0);
   pathSet.add(Path.parse(""));
-  t.is(pathSet.depth, 1);
+  assert.strictEqual(pathSet.depth, 1);
   pathSet.add(Path.parse("a"));
-  t.is(pathSet.depth, 2);
+  assert.strictEqual(pathSet.depth, 2);
 });
 
-ava("add", (t) => {
+void test("add", () => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a"));
   pathSet.add(Path.parse("a"));
-  t.is(pathSet.find(Path.parse("a"), true, true, 99).length, 1);
+  assert.strictEqual(pathSet.find(Path.parse("a"), true, true, 99).length, 1);
 });
 
-ava("get", (t) => {
+void test("get", () => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a.*"));
   pathSet.add(Path.parse("a.a"));
   pathSet.add(Path.parse("*.*"));
 
-  t.is(pathSet.get(Path.parse("a.*")).toString(), "a.*");
-  t.is(pathSet.get(Path.parse("*.a")), null);
+  assert.strictEqual(pathSet.get(Path.parse("a.*")).toString(), "a.*");
+  assert.strictEqual(pathSet.get(Path.parse("*.a")), null);
 });
 
-ava("find", (t) => {
+void test("find", () => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a"));
   pathSet.add(Path.parse("a.*"));
@@ -36,33 +37,33 @@ ava("find", (t) => {
   pathSet.add(Path.parse("*.a"));
   pathSet.add(Path.parse("*.*"));
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse(""), true, true, 1).map((p) => p.toString()),
-    ["a"]
+    ["a"],
   );
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse(""), false, false, 2).map((p) => p.toString()),
-    ["a", "a.*", "a.a", "*.a", "*.*"]
+    ["a", "a.*", "a.a", "*.a", "*.*"],
   );
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse("a.*"), false, false).map((p) => p.toString()),
-    ["a.*"]
+    ["a.*"],
   );
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse("a.*"), false, true).map((p) => p.toString()),
-    ["a.*", "a.a"]
+    ["a.*", "a.a"],
   );
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse("a.*"), true, false).map((p) => p.toString()),
-    ["a.*", "*.*"]
+    ["a.*", "*.*"],
   );
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse("a.*"), true, true).map((p) => p.toString()),
-    ["a.*", "a.a", "*.a", "*.*"]
+    ["a.*", "a.a", "*.a", "*.*"],
   );
 });

test/path.ts

@@ -1,44 +1,45 @@
-import ava from "ava";
-import Path from "../lib/common/path";
+import test from "node:test";
+import assert from "node:assert";
+import Path from "../lib/common/path.ts";
 
-ava("parse", (t) => {
-  t.throws(() => Path.parse("."));
-  t.throws(() => Path.parse("a "));
-  t.throws(() => Path.parse(".a"));
-  t.throws(() => Path.parse("a."));
-  t.throws(() => Path.parse("a..b"));
-  t.throws(() => Path.parse("b*"));
-  t.throws(() => Path.parse("*b"));
-  t.throws(() => Path.parse("a.b c.d"));
-  t.throws(() => Path.parse("a["));
-  t.throws(() => Path.parse("a[b"));
-  t.throws(() => Path.parse("a[b:"));
-  t.throws(() => Path.parse('a[b:"waef]'));
-  t.notThrows(() => Path.parse("[]"));
-  t.notThrows(() => Path.parse("[a:]"));
-  t.notThrows(() => Path.parse("*"));
-  t.notThrows(() => Path.parse(""));
+void test("parse", () => {
+  assert.throws(() => Path.parse("."));
+  assert.throws(() => Path.parse("a "));
+  assert.throws(() => Path.parse(".a"));
+  assert.throws(() => Path.parse("a."));
+  assert.throws(() => Path.parse("a..b"));
+  assert.throws(() => Path.parse("b*"));
+  assert.throws(() => Path.parse("*b"));
+  assert.throws(() => Path.parse("a.b c.d"));
+  assert.throws(() => Path.parse("a["));
+  assert.throws(() => Path.parse("a[b"));
+  assert.throws(() => Path.parse("a[b:"));
+  assert.throws(() => Path.parse('a[b:"waef]'));
+  assert.doesNotThrow(() => Path.parse("[]"));
+  assert.doesNotThrow(() => Path.parse("[a:]"));
+  assert.doesNotThrow(() => Path.parse("*"));
+  assert.doesNotThrow(() => Path.parse(""));
 });
 
-ava("toString", (t) => {
+void test("toString", () => {
   const path1 = Path.parse('abc.[ abc : 123 , 123 : abc , 123: " abc "].123');
   const path2 = Path.parse('abc.[123:  " abc ",abc:123,123:abc].123');
-  t.is(path1.toString(), path2.toString());
+  assert.strictEqual(path1.toString(), path2.toString());
 });
 
-ava("slice", (t) => {
+void test("slice", () => {
   const path = Path.parse("a.*.b.[x:y].c");
   const sliced = path.slice(1, -1);
-  t.is(sliced.toString(), '*.b.[x:"y"]');
-  t.is(sliced.alias, 0b100);
-  t.is(sliced.wildcard, 0b1);
+  assert.strictEqual(sliced.toString(), '*.b.[x:"y"]');
+  assert.strictEqual(sliced.alias, 0b100);
+  assert.strictEqual(sliced.wildcard, 0b1);
 });
 
-ava("concat", (t) => {
+void test("concat", () => {
   const path1 = Path.parse("a");
   const path2 = Path.parse("*.[a:b]");
   const concat = path1.concat(path2);
-  t.is(concat.toString(), 'a.*.[a:"b"]');
-  t.is(concat.alias, 0b100);
-  t.is(concat.wildcard, 0b10);
+  assert.strictEqual(concat.toString(), 'a.*.[a:"b"]');
+  assert.strictEqual(concat.alias, 0b100);
+  assert.strictEqual(concat.wildcard, 0b10);
 });

test/ping.ts

@@ -0,0 +1,37 @@
+import test from "node:test";
+import assert from "node:assert";
+import { parsePing } from "../lib/ping.ts";
+
+void test("linux Case-1", () => {
+  const platform = "linux";
+  const stdout =
+    "PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data\n64 bytes from 8.8.8.8: icmp_seq=1 ttl=118 time=6.66 ms\n64 bytes from 8.8.8.8: icmp_seq=2 ttl=118 time=6.50 ms\n64 bytes from 8.8.8.8: icmp_seq=3 ttl=118 time=6.38 ms\n\n--- 8.8.8.8 ping statistics ---\n3 packets transmitted, 3 received, 0% packet loss, time 402ms\nrtt min/avg/max/mdev = 6.381/6.511/6.656/0.112 ms";
+  const parsedResult = {
+    packetsTransmitted: 3,
+    packetsReceived: 3,
+    packetLoss: 0,
+    min: 6.381,
+    avg: 6.511,
+    max: 6.656,
+    mdev: 0.112,
+  };
+  const parsed = parsePing(platform, stdout);
+  assert.deepStrictEqual(parsedResult, parsed);
+});
+
+void test("linux Case-2", () => {
+  const platform = "linux";
+  const stdout =
+    "PING 10.251.9.108 (10.251.9.108): 56 data bytes\n64 bytes from 10.251.9.108: icmp_seq=0 ttl=57 time=36.758 ms\n64 bytes from 10.251.9.108: icmp_seq=1 ttl=57 time=69.094 ms\n64 bytes from 10.251.9.108: icmp_seq=2 ttl=57 time=28.868 ms\n--- 10.251.9.108 ping statistics ---3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max/stddev = 28.868/44.907/69.094/17.404 ms";
+  const parsedResult = {
+    packetsTransmitted: 3,
+    packetsReceived: 3,
+    packetLoss: 0,
+    min: 28.868,
+    avg: 44.907,
+    max: 69.094,
+    mdev: 17.404,
+  };
+  const parsed = parsePing(platform, stdout);
+  assert.deepStrictEqual(parsedResult, parsed);
+});

test/synth.ts

@@ -0,0 +1,151 @@
+import test from "node:test";
+import assert from "node:assert";
+import initSqlJs from "sql.js/dist/sql-asm.js";
+import { covers, minimize, unionDiff } from "../lib/common/expression/synth.ts";
+import { parse, stringify } from "../lib/common/expression/parser.ts";
+
+const STRING_VALUES = [null, "", "a", "ab", "ab10", "ab-10"];
+const DECIMAL_VALUES = [null, 0, -10, 10];
+
+let db;
+
+async function query(filter: string): Promise<Set<number>> {
+  if (!db) {
+    const sql = await initSqlJs();
+    db = new sql.Database();
+
+    db.run(
+      "CREATE TABLE test (id INTEGER PRIMARY KEY, string STRING, decimal DECIMAL(4,2))",
+    );
+
+    const stmt = db.prepare("INSERT INTO test (string, decimal) VALUES (?, ?)");
+    const count = STRING_VALUES.length * DECIMAL_VALUES.length;
+    for (let i = 0; i < count; ++i) {
+      const str = i % STRING_VALUES.length;
+      const dec = Math.trunc(i / STRING_VALUES.length) % DECIMAL_VALUES.length;
+      stmt.run([STRING_VALUES[str], DECIMAL_VALUES[dec]]);
+    }
+    stmt.free();
+  }
+
+  const res = db.exec(`SELECT id FROM test WHERE ${filter}`);
+  if (!res.length) return new Set();
+  return new Set(res[0].values.flat());
+}
+
+function setsEqual(set1: Set<number>, set2: Set<number>): boolean {
+  if (set1.size !== set2.size) return false;
+  for (const s of set1) if (!set2.has(s)) return false;
+  return true;
+}
+
+function getPermutations(...arrs: any[][]): any[][] {
+  const count = arrs.reduce((total, arr) => total * arr.length, 1);
+  const res = [];
+  for (let i = 0; i < count; ++i) {
+    let j = i;
+    const row = [];
+    for (const arr of arrs) {
+      const v = arr[j % arr.length];
+      j = Math.trunc(j / arr.length);
+      row.push(v);
+    }
+    res.push(row);
+  }
+  return res;
+}
+
+void test("minimize", async () => {
+  const cases: string[] = [];
+
+  cases.push("null");
+  cases.push("false");
+  cases.push("true");
+  cases.push("string");
+  cases.push("(string + decimal) IS NULL");
+  cases.push("(string + decimal) = NULL");
+  cases.push("COALESCE(string, decimal) = 0");
+
+  for (const [s1, s2, s3, op1, op2] of getPermutations(
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s),
+    [">", "=", "<"],
+    ["<>", ">="],
+  )) {
+    cases.push(
+      `string ${op1} "${s1}" OR string ='${s2}' OR NOT string ${op2} '${s3}'`,
+    );
+  }
+
+  for (const [s1, s2] of getPermutations(
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s),
+  ))
+    cases.push(`string > "${s1}" AND string < '${s2}'`);
+
+  for (const c of cases) {
+    const res1 = await query(c);
+    const min = stringify(minimize(parse(c), true));
+    const res2 = await query(min);
+    assert.strictEqual(setsEqual(res1, res2), true);
+  }
+});
+
+void test("unionDiff", async () => {
+  const cases = [
+    "true",
+    "decimal > 0",
+    "decimal > 10",
+    "UPPER(string || decimal) LIKE 'AB10'",
+    "COALESCE(string, decimal) = 0",
+  ];
+
+  for (const [c1, c2] of getPermutations(cases, cases)) {
+    const res1 = await query(c1);
+    const res2 = await query(c2);
+    const [union, diff] = unionDiff(parse(c1), parse(c2));
+    const res3 = await query(stringify(union));
+    const res4 = await query(stringify(diff));
+
+    const unionSet = new Set([...res1, ...res2]);
+    const diffSet = new Set(Array.from(res2).filter((r) => !res1.has(r)));
+
+    assert.strictEqual(setsEqual(res3, unionSet), true);
+    assert.strictEqual(setsEqual(res4, diffSet), true);
+  }
+});
+
+void test("covers", async () => {
+  assert.strictEqual(covers(false, false), true);
+  assert.strictEqual(covers(false, parse("decimal > 5 AND decimal < 3")), true);
+  assert.strictEqual(covers(parse("true"), parse("decimal > 0")), true);
+  assert.strictEqual(covers(parse("true"), parse("false")), true);
+  assert.strictEqual(covers(parse("false"), parse("false")), true);
+  assert.strictEqual(covers(parse("false"), parse("decimal > 0")), false);
+  assert.strictEqual(covers(parse("decimal >= 0"), parse("decimal > 0")), true);
+  assert.strictEqual(
+    covers(parse("decimal > 0"), parse("decimal >= 0")),
+    false,
+  );
+
+  const cases = [
+    ["decimal >= 0", "decimal > 0"],
+    ["decimal > 0", "decimal > 5"],
+    ["string IS NOT NULL", "string = 'a'"],
+    ["true", "decimal > 0"],
+  ];
+
+  for (const [c1, c2] of cases) {
+    const res1 = await query(c1);
+    const res2 = await query(c2);
+    const coversResult = covers(parse(c1), parse(c2));
+    const actuallyCovers = Array.from(res2).every((r) => res1.has(r));
+
+    assert.strictEqual(
+      coversResult,
+      actuallyCovers,
+      `covers(${c1}, ${c2}) should match actual coverage`,
+    );
+  }
+});

test/util.ts

@@ -1,7 +1,8 @@
-import ava from "ava";
-import * as common from "../lib/common";
+import test from "node:test";
+import assert from "node:assert";
+import * as common from "../lib/util.ts";
 
-ava("generateDeviceId", (t) => {
+void test("generateDeviceId", () => {
   const space = [" ", "%20"];
   const special = [";", "%3B"];
   const cases = [
@@ -28,14 +29,17 @@ ava("generateDeviceId", (t) => {
       `TestOUIWith${space[1]}_${special[1]}2912-TestSerialNumberWith${space[1]}_${special[1]}2912`,
     ],
   ];
-  t.plan(cases.length);
+
   for (const c of cases)
-    t.is(common.generateDeviceId(c[0] as Record<string, string>), c[1]);
+    assert.strictEqual(
+      common.generateDeviceId(c[0] as Record<string, string>),
+      c[1],
+    );
 });
 
-ava("escapeRegExp", (t) => {
-  t.is(
+void test("escapeRegExp", () => {
+  assert.strictEqual(
     common.escapeRegExp("\\ ^ $ * + ? . ( ) | { } [ ]"),
-    "\\\\ \\^ \\$ \\* \\+ \\? \\. \\( \\) \\| \\{ \\} \\[ \\]"
+    "\\\\ \\^ \\$ \\* \\+ \\? \\. \\( \\) \\| \\{ \\} \\[ \\]",
   );
 });

test/xml-parser.ts

@@ -1,16 +1,17 @@
-import ava from "ava";
+import test from "node:test";
+import assert from "node:assert";
 import {
   parseXmlDeclaration,
   decodeEntities,
   parseXml,
-} from "../lib/xml-parser";
+} from "../lib/xml-parser.ts";
 
-ava("parseXmlDeclaration", (t) => {
+void test("parseXmlDeclaration", () => {
   const buf = Buffer.from(
-    '<?xml version="1.0" encoding="UTF-8"?>\n<soap-env:Envelope />'
+    '<?xml version="1.0" encoding="UTF-8"?>\n<soap-env:Envelope />',
   );
   const attrs = parseXmlDeclaration(buf);
-  t.deepEqual(attrs, [
+  assert.deepStrictEqual(attrs, [
     {
       name: "version",
       namespace: "",
@@ -26,18 +27,18 @@ ava("parseXmlDeclaration", (t) => {
   ]);
 });
 
-ava("decodeEntities", (t) => {
-  t.is(
+void test("decodeEntities", () => {
+  assert.strictEqual(
     decodeEntities("&&amp;&lt;&gt;&quot;&apos;&gt;&#167;&#xd842;&#xDFB7;;"),
-    "&&<>\"'>§𠮷;"
+    "&&<>\"'>§𠮷;",
   );
 });
 
-ava("parse", (t) => {
+void test("parse", () => {
   const xml =
     '<?xml version="1.0"?>\n<a-b:c><d f="1<g>"/><!-- comment --><h >i</h></a-b:c>';
   const parsed = parseXml(xml);
-  t.deepEqual(parsed, {
+  assert.deepStrictEqual(parsed, {
     name: "root",
     namespace: "",
     localName: "root",

test/yaml.ts

@@ -1,14 +1,18 @@
-import ava from "ava";
+import test from "node:test";
+import assert from "node:assert";
 import * as yaml from "yaml";
-import { stringify } from "../lib/common/yaml";
+import { stringify } from "../lib/common/yaml.ts";
 
 import testCases from "./yaml-tests.json";
 
-ava("stringify", (t) => {
+void test("stringify", () => {
   for (const testCase of testCases) {
     let str = stringify(testCase);
     if (str.startsWith(">2")) str = ">3" + str.slice(2);
     if (str.startsWith("|2")) str = "|3" + str.slice(2);
-    t.deepEqual(yaml.parse(yaml.stringify(testCase)), yaml.parse(str));
+    assert.deepStrictEqual(
+      yaml.parse(yaml.stringify(testCase)),
+      yaml.parse(str),
+    );
   }
 });

tsconfig.json

@@ -1,15 +1,18 @@
 {
   "compilerOptions": {
-    "lib": ["ES2019", "dom"],
-    "module": "ES2020",
-    "moduleResolution": "node",
-    "target": "ES2019",
+    "lib": ["ES2022", "dom"],
+    "module": "ES2022",
+    "moduleResolution": "bundler",
+    "target": "ES2022",
     "resolveJsonModule": true,
     "allowSyntheticDefaultImports": true,
     "alwaysStrict": true,
     "noImplicitReturns": true,
     "strictFunctionTypes": true,
-    "allowJs": true
+    "isolatedModules": true,
+    "noEmit": true,
+    "allowImportingTsExtensions": true,
+    "skipLibCheck": true
   },
   "include": [
     "./bin/*.ts",

ui/admin-menu.ts

@@ -1,24 +1,4 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import m from "mithril";
-import { ClosureComponent, Component } from "mithril";
+import m, { ClosureComponent, Component } from "mithril";
 
 const component: ClosureComponent = (): Component => {
   return {
@@ -31,8 +11,8 @@ const component: ClosureComponent = (): Component => {
           m(
             "li",
             { class: active["presets"] },
-            m("a", { href: "#!/admin/presets" }, "Presets")
-          )
+            m("a", { href: "#!/admin/presets" }, "Presets"),
+          ),
         );
       }
 
@@ -41,8 +21,8 @@ const component: ClosureComponent = (): Component => {
           m(
             "li",
             { class: active["provisions"] },
-            m("a", { href: "#!/admin/provisions" }, "Provisions")
-          )
+            m("a", { href: "#!/admin/provisions" }, "Provisions"),
+          ),
         );
       }
 
@@ -51,8 +31,12 @@ const component: ClosureComponent = (): Component => {
           m(
             "li",
             { class: active["virtualParameters"] },
-            m("a", { href: "#!/admin/virtualParameters" }, "Virtual Parameters")
-          )
+            m(
+              "a",
+              { href: "#!/admin/virtualParameters" },
+              "Virtual Parameters",
+            ),
+          ),
         );
       }
 
@@ -61,8 +45,8 @@ const component: ClosureComponent = (): Component => {
           m(
             "li",
             { class: active["files"] },
-            m("a", { href: "#!/admin/files" }, "Files")
-          )
+            m("a", { href: "#!/admin/files" }, "Files"),
+          ),
         );
       }
 
@@ -71,8 +55,8 @@ const component: ClosureComponent = (): Component => {
           m(
             "li",
             { class: active["config"] },
-            m("a", { href: "#!/admin/config" }, "Config")
-          )
+            m("a", { href: "#!/admin/config" }, "Config"),
+          ),
         );
       }
 
@@ -81,8 +65,8 @@ const component: ClosureComponent = (): Component => {
           m(
             "li",
             { class: active["permissions"] },
-            m("a", { href: "#!/admin/permissions" }, "Permissions")
-          )
+            m("a", { href: "#!/admin/permissions" }, "Permissions"),
+          ),
         );
       }
 
@@ -91,8 +75,8 @@ const component: ClosureComponent = (): Component => {
           m(
             "li",
             { class: active["users"] },
-            m("a", { href: "#!/admin/users" }, "Users")
-          )
+            m("a", { href: "#!/admin/users" }, "Users"),
+          ),
         );
       }
 

ui/app.ts

@@ -1,43 +1,24 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 import m, { RouteResolver } from "mithril";
-import layout from "./layout";
-import * as store from "./store";
-import * as wizardPage from "./wizard-page";
-import * as loginPage from "./login-page";
-import * as overviewPage from "./overview-page";
-import * as devicesPage from "./devices-page";
-import * as devicePage from "./device-page";
-import * as errorPage from "./error-page";
-import * as faultsPage from "./faults-page";
-import * as presetsPage from "./presets-page";
-import * as provisionsPage from "./provisions-page";
-import * as virtualParametersPage from "./virtual-parameters-page";
-import * as filesPage from "./files-page";
-import * as configPage from "./config-page";
-import * as permissionsPage from "./permissions-page";
-import * as usersPage from "./users-page";
-import Authorizer from "../lib//common/authorizer";
-import * as notifications from "./notifications";
-import { contextifyComponent } from "./components";
-import { PermissionSet, UiConfig } from "../lib/types";
+import layout from "./layout.ts";
+import * as store from "./store.ts";
+import * as wizardPage from "./wizard-page.ts";
+import * as loginPage from "./login-page.ts";
+import * as overviewPage from "./overview-page.ts";
+import * as devicesPage from "./devices-page.ts";
+import * as devicePage from "./device-page.ts";
+import * as errorPage from "./error-page.ts";
+import * as faultsPage from "./faults-page.ts";
+import * as presetsPage from "./presets-page.ts";
+import * as provisionsPage from "./provisions-page.ts";
+import * as virtualParametersPage from "./virtual-parameters-page.ts";
+import * as filesPage from "./files-page.ts";
+import * as configPage from "./config-page.ts";
+import * as permissionsPage from "./permissions-page.ts";
+import * as usersPage from "./users-page.ts";
+import Authorizer from "../lib//common/authorizer.ts";
+import * as notifications from "./notifications.ts";
+import { contextifyComponent } from "./components.ts";
+import { PermissionSet, UiConfig } from "../lib/types.ts";
 
 declare global {
   interface Window {
@@ -137,7 +118,7 @@ m.route(document.body, "/overview", {
   "/admin/provisions": pagify("provisions", provisionsPage),
   "/admin/virtualParameters": pagify(
     "virtualParameters",
-    virtualParametersPage
+    virtualParametersPage,
   ),
   "/admin/files": pagify("files", filesPage),
   "/admin/config": pagify("config", configPage),

ui/autocomplete-compnent.ts

@@ -1,35 +1,16 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 type AutocompleteCallback = (
   value: string,
-  callback: (suggestions: { value: string; tip?: string }[]) => void
+  callback: (suggestions: { value: string; tip?: string }[]) => void,
 ) => void;
 
 export default class Autocomplete {
-  private declare callback: AutocompleteCallback;
-  private declare element: HTMLInputElement;
-  private declare hideTimeout: NodeJS.Timeout;
-  private declare visible: boolean;
-  private declare default: string;
-  private declare selection: number;
-  private declare container: HTMLElement;
+  declare private callback: AutocompleteCallback;
+  declare private element: HTMLInputElement;
+  declare private hideTimeout: NodeJS.Timeout;
+  declare private visible: boolean;
+  declare private default: string;
+  declare private selection: number;
+  declare private container: HTMLElement;
 
   public constructor(className: string, callback: AutocompleteCallback) {
     this.callback = callback;
@@ -99,9 +80,9 @@ export default class Autocomplete {
       if (this.visible) this.hide();
       return;
     }
-    this.container.style.left = `${domRect.left + window.pageXOffset}px`;
+    this.container.style.left = `${domRect.left + window.scrollX}px`;
     this.container.style.width = `${domRect.width}px`;
-    this.container.style.top = `${domRect.bottom + window.pageYOffset}px`;
+    this.container.style.top = `${domRect.bottom + window.scrollY}px`;
   }
 
   private hide(): void {
@@ -180,14 +161,14 @@ export default class Autocomplete {
       if (selectedElement) {
         this.container.scrollTop = Math.min(
           this.container.scrollTop,
-          selectedElement.offsetTop
+          selectedElement.offsetTop,
         );
 
         this.container.scrollTop = Math.max(
           this.container.scrollTop,
           selectedElement.offsetTop +
             selectedElement.scrollHeight -
-            this.container.clientHeight
+            this.container.clientHeight,
         );
       }
     });

ui/change-password-component.ts

@@ -1,26 +1,7 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 import { VnodeDOM, ClosureComponent } from "mithril";
-import * as notifications from "./notifications";
-import { changePassword } from "./store";
-import { m } from "./components";
+import * as notifications from "./notifications.ts";
+import { changePassword } from "./store.ts";
+import { m } from "./components.ts";
 
 interface Attrs {
   noAuth?: boolean;
@@ -52,7 +33,7 @@ const component: ClosureComponent<Attrs> = () => {
             oncreate: (_vnode) => {
               (_vnode.dom as HTMLSelectElement).focus();
             },
-          })
+          }),
         ),
       ];
 
@@ -75,8 +56,8 @@ const component: ClosureComponent<Attrs> = () => {
               oninput: (e) => {
                 vnode.state[f] = e.target.value;
               },
-            })
-          )
+            }),
+          ),
         );
       }
 
@@ -85,7 +66,7 @@ const component: ClosureComponent<Attrs> = () => {
         {
           type: "submit",
         },
-        "Change password"
+        "Change password",
       ) as VnodeDOM;
 
       form.push(m(".actions-bar", submit));
@@ -109,19 +90,19 @@ const component: ClosureComponent<Attrs> = () => {
               ) {
                 notifications.push(
                   "error",
-                  "Password confirm doesn't match new password"
+                  "Password confirm doesn't match new password",
                 );
               } else {
                 (submit.dom as HTMLFormElement).disabled = true;
                 changePassword(
                   vnode.state["username"],
                   vnode.state["newPassword"],
-                  vnode.state["authPassword"]
+                  vnode.state["authPassword"],
                 )
                   .then(() => {
                     notifications.push(
                       "success",
-                      "Password updated successfully"
+                      "Password updated successfully",
                     );
                     if (onPasswordChange) onPasswordChange();
                     (submit.dom as HTMLFormElement).disabled = false;
@@ -133,7 +114,7 @@ const component: ClosureComponent<Attrs> = () => {
               }
             },
           },
-          form
+          form,
         ),
       ];
 

ui/code-editor-component.ts

@@ -1,26 +1,7 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 import { Editor } from "codemirror";
 import { ClosureComponent } from "mithril";
-import { m } from "./components";
-import { codeMirror } from "./dynamic-loader";
+import { m } from "./components.ts";
+import { codeMirror } from "./dynamic-loader.ts";
 
 interface Attrs {
   id: string;
@@ -54,7 +35,7 @@ const component: ClosureComponent<Attrs> = () => {
                   if (vnode.attrs.onSubmit) vnode.attrs.onSubmit(_vnode.dom);
                 },
               },
-            }
+            },
           );
 
           if (vnode.attrs.onChange) {

ui/components.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 import m, {
   Static,
   Attributes,
@@ -26,20 +7,20 @@ import m, {
   ClosureComponent,
   Vnode,
 } from "mithril";
-import parameter from "./components/parameter";
-import parameterList from "./components/parameter-list";
-import parameterTable from "./components/parameter-table";
-import overviewDot from "./components/overview-dot";
-import container from "./components/container";
-import summonButton from "./components/summon-button";
-import deviceFaults from "./components/device-faults";
-import allParameters from "./components/all-parameters";
-import deviceActions from "./components/device-actions";
-import tags from "./components/tags";
-import ping from "./components/ping";
-import deviceLink from "./components/device-link";
-import longTextComponent from "./long-text-component";
-import loading from "./components/loading";
+import parameter from "./components/parameter.ts";
+import parameterList from "./components/parameter-list.ts";
+import parameterTable from "./components/parameter-table.ts";
+import overviewDot from "./components/overview-dot.ts";
+import container from "./components/container.ts";
+import summonButton from "./components/summon-button.ts";
+import deviceFaults from "./components/device-faults.ts";
+import allParameters from "./components/all-parameters.ts";
+import deviceActions from "./components/device-actions.ts";
+import tags from "./components/tags.ts";
+import ping from "./components/ping.ts";
+import deviceLink from "./components/device-link.ts";
+import longTextComponent from "./long-text-component.ts";
+import loading from "./components/loading.ts";
 
 const comps = {
   parameter,
@@ -63,10 +44,11 @@ const vnodeContext = new WeakMap<Vnode, Attributes>();
 
 interface MC extends Static {
   context: {
-    (ctx: Attributes, selector: string, ...children: Children[]): Vnode<
-      any,
-      any
-    >;
+    (
+      ctx: Attributes,
+      selector: string,
+      ...children: Children[]
+    ): Vnode<any, any>;
     (
       ctx: Attributes,
       selector: string,
@@ -150,7 +132,7 @@ export function contextifyComponent(component: ComponentTypes): ComponentTypes {
                 title: "Click to print stack trace to console",
                 onclick: () => console.error(err),
               },
-              "Error!"
+              "Error!",
             );
           }
         };

ui/components/all-parameters.ts

@@ -1,28 +1,13 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ClosureComponent, Component } from "mithril";
-import { m } from "../components";
-import * as taskQueue from "../task-queue";
-import { parse } from "../../lib/common/expression-parser";
-import memoize from "../../lib/common/memoize";
-import { getIcon } from "../icons";
+import { ClosureComponent } from "mithril";
+import { m } from "../components.ts";
+import * as taskQueue from "../task-queue.ts";
+import { parse } from "../../lib/common/expression/parser.ts";
+import memoize from "../../lib/common/memoize.ts";
+import { getIcon } from "../icons.ts";
+import { QueryResponse, evaluateExpression } from "../store.ts";
+import debounce from "../../lib/common/debounce.ts";
+import { Expression } from "../../lib/types.ts";
+import { FlatDevice } from "../../lib/ui/db.ts";
 
 const memoizedParse = memoize(parse);
 
@@ -50,30 +35,39 @@ function orderKeysByDepth(device: Record<string, unknown>): string[][] {
   return res;
 }
 
-const component: ClosureComponent = (): Component => {
+interface Attrs {
+  device: FlatDevice;
+  limit: Expression;
+  deviceQuery: QueryResponse;
+}
+
+const component: ClosureComponent<Attrs> = () => {
+  let queryString: string;
+  const formQueryString = debounce((args: string[]) => {
+    queryString = args[args.length - 1];
+    m.redraw();
+  }, 500);
+
   return {
     view: (vnode) => {
-      const device = vnode.attrs["device"];
+      const device = vnode.attrs.device;
 
-      const limit = vnode.attrs["limit"] > 0 ? vnode.attrs["limit"] : 100;
+      const limit =
+        (evaluateExpression(vnode.attrs.limit, device) as number) || 100;
 
       const search = m("input", {
         type: "text",
         placeholder: "Search parameters",
         oninput: (e) => {
-          vnode.state["searchString"] = e.target.value;
+          formQueryString(e.target.value);
           e.redraw = false;
-          clearTimeout(vnode.state["timeout"]);
-          vnode.state["timeout"] = setTimeout(m.redraw, 500);
         },
       });
 
       const instanceRegex = /\.[0-9]+$/;
       let re;
-      if (vnode.state["searchString"]) {
-        const keywords = vnode.state["searchString"]
-          .split(" ")
-          .filter((s) => s);
+      if (queryString) {
+        const keywords = queryString.split(" ").filter((s) => s);
         if (keywords.length)
           re = new RegExp(keywords.map((s) => escapeRegExp(s)).join(".*"), "i");
       }
@@ -104,8 +98,8 @@ const component: ClosureComponent = (): Component => {
           val.push(
             m(
               "parameter",
-              Object.assign({ device: device, parameter: memoizedParse(k) })
-            )
+              Object.assign({ device: device, parameter: memoizedParse(k) }),
+            ),
           );
         } else if (p.object && p.writable) {
           if (instanceRegex.test(k)) {
@@ -117,13 +111,13 @@ const component: ClosureComponent = (): Component => {
                   onclick: () => {
                     taskQueue.queueTask({
                       name: "deleteObject",
-                      device: device["DeviceID.ID"].value[0],
+                      device: device["DeviceID.ID"].value[0] as string,
                       objectName: k,
                     });
                   },
                 },
-                getIcon("delete-instance")
-              )
+                getIcon("delete-instance"),
+              ),
             );
           } else {
             val.push(
@@ -134,13 +128,13 @@ const component: ClosureComponent = (): Component => {
                   onclick: () => {
                     taskQueue.queueTask({
                       name: "addObject",
-                      device: device["DeviceID.ID"].value[0],
+                      device: device["DeviceID.ID"].value[0] as string,
                       objectName: k,
                     });
                   },
                 },
-                getIcon("add-instance")
-              )
+                getIcon("add-instance"),
+              ),
             );
           }
         }
@@ -153,38 +147,38 @@ const component: ClosureComponent = (): Component => {
               onclick: () => {
                 taskQueue.queueTask({
                   name: "getParameterValues",
-                  device: device["DeviceID.ID"].value[0],
+                  device: device["DeviceID.ID"].value[0] as string,
                   parameterNames: [k],
                 });
               },
             },
-            getIcon("refresh")
-          )
+            getIcon("refresh"),
+          ),
         );
 
         return m(
           "tr",
           attrs,
           m("td.left", m("long-text", { text: k })),
-          m("td.right", val)
+          m("td.right", val),
         );
       });
 
       return m(
         "loading",
-        { queries: [vnode.attrs["deviceQuery"]] },
+        { queries: [vnode.attrs.deviceQuery] },
         m(
           ".all-parameters",
           m(
             "a.download-csv",
             {
               href: `api/devices/${encodeURIComponent(
-                device["DeviceID.ID"].value[0]
+                device["DeviceID.ID"].value[0],
               )}.csv`,
               download: "",
               style: "float: right;",
             },
-            "Download"
+            "Download",
           ),
           search,
           m(
@@ -192,10 +186,10 @@ const component: ClosureComponent = (): Component => {
             m("table", m("tbody", rows)),
             m(
               "m",
-              `Displaying ${filteredKeys.length} out of ${count} parameters.`
-            )
-          )
-        )
+              `Displaying ${filteredKeys.length} out of ${count} parameters.`,
+            ),
+          ),
+        ),
       );
     },
   };

ui/components/container.ts

@@ -1,45 +1,79 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
+import { Attributes, ClosureComponent } from "mithril";
+import { map } from "../../lib/common/expression/parser.ts";
+import memoize from "../../lib/common/memoize.ts";
+import { Expression } from "../../lib/types.ts";
+import { m } from "../components.ts";
+import { evaluateExpression, getTimestamp } from "../store.ts";
+import { FlatDevice } from "../../lib/ui/db.ts";
 
-import { ClosureComponent, Component } from "mithril";
-import { m } from "../components";
-import * as store from "../store";
+const evaluateAttributes = memoize(
+  (
+    attrs: Record<string, Expression>,
+    obj: Record<string, unknown>,
+    now: number, // eslint-disable-line @typescript-eslint/no-unused-vars
+  ): Attributes => {
+    const res: Attributes = {};
+    for (const [k, v] of Object.entries(attrs)) {
+      const vv = map(v, (e) => {
+        if (
+          Array.isArray(e) &&
+          e[0] === "FUNC" &&
+          e[1] === "ENCODEURICOMPONENT"
+        ) {
+          const a = evaluateExpression(e[2], obj);
+          if (a == null) return null;
+          return encodeURIComponent(a as string);
+        }
+        return e;
+      });
+      res[k] = evaluateExpression(vv, obj);
+    }
+    return res;
+  },
+);
 
-const component: ClosureComponent = (): Component => {
+interface Attrs {
+  device: FlatDevice;
+  filter: Expression;
+  components: unknown;
+}
+
+const component: ClosureComponent<Attrs> = () => {
   return {
     view: (vnode) => {
-      if (vnode.attrs["filter"]) {
-        if (
-          !store.evaluateExpression(
-            vnode.attrs["filter"],
-            vnode.attrs["device"]
-          )
-        )
-          return null;
+      const device = vnode.attrs.device;
+      if ("filter" in vnode.attrs) {
+        if (!evaluateExpression(vnode.attrs.filter, device || {})) return null;
       }
 
-      const children = Object.values(vnode.attrs["components"]).map((c) => {
+      const children = Object.values(vnode.attrs.components).map((c) => {
+        if (Array.isArray(c)) c = evaluateExpression(c, device || {});
         if (typeof c !== "object") return `${c}`;
-        return m(c["type"], c);
+        const type = evaluateExpression(c["type"], device || {});
+        if (!type) return null;
+        return m(type as string, c);
       });
-      if (vnode.attrs["element"]) return m(vnode.attrs["element"], children);
-      else return children;
+
+      let el = vnode.attrs["element"];
+
+      if (el == null) return children;
+
+      let attrs: Attributes;
+      if (Array.isArray(el)) {
+        el = evaluateExpression(el, device || {});
+      } else if (typeof el === "object") {
+        if (el["attributes"] != null) {
+          attrs = evaluateAttributes(
+            el["attributes"],
+            device || {},
+            getTimestamp(),
+          );
+        }
+
+        el = evaluateExpression(el["tag"], device || {});
+      }
+
+      return m(el, attrs, children);
     },
   };
 };