1.2.9

Fixes occassional lock expired errors after updating presets, etc.

CHANGELOG.md

@@ -1,5 +1,83 @@
 # Change Log
 
+## 1.2.9 (2022-08-22)
+
+- New config option `cwmp.skipRootGpn` to enable a workaround for some
+  problematic CPEs that reject GPN requests on data model root.
+- Stream query results and CSV downloads as data becomes available instead of
+  buffering the entire response.
+- Log HTTP/HTTPS client errors in debug log.
+- Fix occasional lock expired errors after updating presets, etc.
+- Fix bug where queries containing `<>` operator may return incorrect results.
+- Fix a performance hit caused by DB calls containing the entire CPE data model
+  rather then just the updated parameters.
+- Fix bug where tags containing special characters are saved in their encoded
+  form when set via a Provision script.
+- Fix issue causing invalid `xsd:dateTime` values to be saved in DB as `NaN`
+  rather than maintain the original string value.
+- Fix bug using `$regex` operator with a numeric or a datetime string.
+- Fix ping not working on certain platforms.
+- Ping requests are now authenticated.
+- Fix error when the `FORWARDED_HEADER` environment variable contains IPv4 CIDR
+  while the listening interface is IPv6 and vice versa.
+- Fix false warning "unexpected parameter in response" showing in
+  `genieacs-cwmp` logs.
+- Other minor fixes and stability improvements.
+
+## 1.2.8 (2021-10-27)
+
+- Fix a remote code execution security vulnerability in genieacs-ui.
+- All UI components can now be configured using fully dynamic expressions.
+  Previously some component types only accept fixed string values as properties.
+- The `container` UI component can now be configured with an `element` property
+  that's either a string or a nested pair of `tag` and `attributes` properties.
+  The various attributes under the `attributes` property can now make use of a
+  new function `ENCODEURICOMPONENT()` to facilitate creating custom hyperlinks
+  in the UI.
+- Improve sorting buttons' behavior in the device listing page and other pages.
+  Sorting by multiple columns should now feel more intuitive.
+- Support the modulo (%) operator in expressions.
+- Fix a regression in the previous release where the config option 'ui.pageSize'
+  no longer works.
+- Fix process crash when a CPE sends an unsupported ACS method.
+
+## 1.2.7 (2021-09-18)
+
+- Fix regression causing frequent invalid session errors.
+- Fix regression breaking digest authentication in connection requests.
+
+## 1.2.6 (2021-09-16)
+
+- New config option `cwmp.downloadSuccessOnTimeout` to enable a workaround for
+  CPEs that neglect to send a TransferComplete message after a successful
+  download.
+- Display a progress bar when uploading new files.
+- Default to dual stack interface binding (i.e. `::` instead of `0.0.0.0`).
+  Unless the binding interface is explicitly set, this will cause IPv4 addresses
+  in the logs to be displayed as IPv4-mapped IPv6 addresses (e.g.
+  `::ffff:127.0.0.1` instead of `127.0.0.1`).
+- Detect and correct for client side (browser) clock skew that would otherwise
+  alter the numbers in the pie charts.
+- Various improvements relating to dealing with buggy TR-069 client
+  implementations.
+- Fix a bug causing "lock expired" exceptions when a CWMP session remains open
+  for a very long time due to slow clients.
+- Fix metadata of uploaded files going missing due to nginx stripping away what
+  it considers to be invalid headers. The nginx directive
+  `ignore_invalid_headers` is no longer required.
+- Fix crash when a CPE is assigned a tag containing a dot.
+- Fix bug preventing the user from closing the preset pop-up after being
+  presented with unsupported preset message.
+- Fix exceptions raised from ext scripts manifesting as timeout faults.
+- Fix download getting triggered repeatedly when the value passed to `Download`
+  parameter is greater than the current timestamp.
+- Fix crash when passing invalid attributes to `declare`.
+- Fix crash in NBI when pushing tasks to non-existing devices.
+- Fix crash in NBI when passing invalid JSON to various API endpoints.
+- Fix crash when the output from `ping` command cannot be parsed in some rare
+  edge cases.
+- A number of other fixes and stability improvements.
+
 ## 1.2.5 (2021-03-12)
 
 - Support specifying custom types when uploading files.

README.md

@@ -72,5 +72,5 @@ For commercial support options, please visit
 
 ## License
 
-Copyright 2013-2020 GenieACS Inc. GenieACS is released under the
+Copyright 2013-2021 GenieACS Inc. GenieACS is released under the
 [AGPLv3 license terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

bin/genieacs-cwmp.ts

@@ -83,6 +83,7 @@ if (!cluster.worker) {
     host: SERVICE_ADDRESS,
     ssl: key && cert ? { key, cert } : null,
     onConnection: cwmp.onConnection,
+    onClientError: cwmp.onClientError,
     timeout: 30000,
     keepAliveTimeout: 0,
   };

build/build.ts

@@ -28,8 +28,9 @@ import commonjs from "@rollup/plugin-commonjs";
 import postcss from "postcss";
 import postcssImport from "postcss-import";
 import postcssPresetEnv from "postcss-preset-env";
+import postcssColorMod from "postcss-color-mod-function";
 import cssnano from "cssnano";
-import { optimize, extendDefaultPlugins } from "svgo";
+import { optimize } from "svgo";
 import * as xmlParser from "../lib/xml-parser";
 
 const MODE = process.env["NODE_ENV"] || "production";
@@ -74,6 +75,7 @@ function rmDirSync(dirPath): void {
   fs.rmdirSync(dirPath);
 }
 
+// For lockfileVersion = 1
 function stripDevDeps(deps): void {
   if (!deps["dependencies"]) return;
   for (const [k, v] of Object.entries(deps["dependencies"])) {
@@ -83,6 +85,15 @@ function stripDevDeps(deps): void {
   if (!Object.keys(deps["dependencies"]).length) delete deps["dependencies"];
 }
 
+// For lockfileVersion = 2
+function stripDevDeps2(deps): void {
+  if (!deps["packages"]) return;
+  for (const [k, v] of Object.entries(deps["packages"])) {
+    delete v["devDependencies"];
+    if (v["dev"]) delete deps["packages"][k];
+  }
+}
+
 function xmlTostring(xml): string {
   const children = [];
   for (const c of xml.children || []) children.push(xmlTostring(c));
@@ -140,6 +151,7 @@ async function init(): Promise<string[]> {
   );
   npmShrinkwrapJson["version"] = packageJson["version"];
   stripDevDeps(npmShrinkwrapJson);
+  stripDevDeps2(npmShrinkwrapJson);
   fs.writeFileSync(
     path.resolve(OUTPUT_DIR, "npm-shrinkwrap.json"),
     JSON.stringify(npmShrinkwrapJson, null, 2)
@@ -174,10 +186,10 @@ async function generateCss(): Promise<void> {
       stage: 3,
       features: {
         "nesting-rules": true,
-        "color-mod-function": true,
       },
     }),
-    cssnano,
+    postcssColorMod,
+    ...(MODE === "production" ? [cssnano] : []),
   ]).process(cssIn, { from: cssInPath, to: cssOutPath });
   fs.writeFileSync(cssOutPath, cssOut.css);
 }
@@ -299,12 +311,16 @@ async function generateIconsSprite(): Promise<void> {
     const id = path.parse(file).name;
     const filePath = path.join(iconsDir, file);
     const { data } = await optimize(fs.readFileSync(filePath).toString(), {
-      plugins: extendDefaultPlugins([
+      plugins: [
         {
-          name: "removeViewBox",
+          name: "preset-default",
-          active: false,
+          params: {
+            overrides: {
+              removeViewBox: false,
+            },
+          },
         },
-      ]),
+      ],
     });
     symbols.push(generateSymbol(id, data));
   }

build/spellcheck-dict.pws

@@ -165,3 +165,12 @@ Brotli
 CSV
 hostInfo
 IPv
+downloadSuccessOnTimeout
+TransferComplete
+nginx
+ENCODEURICOMPONENT
+pageSize
+skipRootGpn
+GPN
+NaN
+CIDR

docs/environment-variables.rst

@@ -64,7 +64,7 @@ CWMP_PORT
 CWMP_INTERFACE
   The network interface that genieacs-cwmp binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 CWMP_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
@@ -102,7 +102,7 @@ NBI_PORT
 NBI_INTERFACE
   The network interface that genieacs-nbi binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 NBI_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
@@ -140,7 +140,7 @@ FS_PORT
 FS_INTERFACE
   The network interface that genieacs-fs binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 FS_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
@@ -186,7 +186,7 @@ UI_PORT
 UI_INTERFACE
   The network interface that genieacs-ui binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 UI_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.

docs/installation-guide.rst

@@ -19,7 +19,7 @@ Prerequisites
 
 .. topic:: Node.js
 
-  GenieACS requires Node.js 10.x and up. Refer to https://nodejs.org/ for
+  GenieACS requires Node.js 12.13 and up. Refer to https://nodejs.org/ for
   instructions.
 
 .. topic:: MongoDB

lib/api-functions.ts

@@ -17,7 +17,6 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { parse } from "url";
 import * as db from "./db";
 import * as common from "./common";
 import * as cache from "./cache";
@@ -80,7 +79,7 @@ export async function connectionRequest(
     ).value || [""])[0];
   }
 
-  const remoteAddress = parse(connectionRequestUrl).host;
+  const remoteAddress = new URL(connectionRequestUrl).hostname;
 
   const evalCallback = (exp): Expression => {
     if (!Array.isArray(exp)) return exp;
@@ -95,7 +94,7 @@ export async function connectionRequest(
       else if (name === "password") return password;
 
       const p = device[name];
-      if (p && p.value) return p.value[0];
+      if (p?.value) return p.value[0];
     } else if (exp[0] === "FUNC") {
       if (exp[1] === "REMOTE_ADDRESS") return remoteAddress;
       else if (exp[1] === "USERNAME") return username;
@@ -145,15 +144,24 @@ export async function connectionRequest(
 
   const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
 
-  let udpProm;
+  let udpProm = Promise.resolve(false);
   if (udpConnectionRequestAddress) {
-    udpProm = udpConnectionRequest(
+    try {
-      udpConnectionRequestAddress,
+      const u = new URL("udp://" + udpConnectionRequestAddress);
-      authExp,
+      udpProm = udpConnectionRequest(
-      UDP_CONNECTION_REQUEST_PORT,
+        u.hostname,
-      debug,
+        parseInt(u.port || "80"),
-      deviceId
+        authExp,
-    );
+        UDP_CONNECTION_REQUEST_PORT,
+        debug,
+        deviceId
+      ).then(
+        () => true,
+        () => false
+      );
+    } catch (err) {
+      // Ignore invalid address
+    }
   }
 
   const status = await httpConnectionRequest(
@@ -165,10 +173,7 @@ export async function connectionRequest(
     deviceId
   );
 
-  if (udpProm) {
+  if (await udpProm) return "";
-    await udpProm;
-    return "";
-  }
 
   return status;
 }
@@ -312,7 +317,7 @@ function sanitizeTask(task): void {
 
 export async function insertTasks(tasks: any[]): Promise<Task[]> {
   if (tasks && !Array.isArray(tasks)) tasks = [tasks];
-  else if (!tasks || tasks.length === 0) return tasks || [];
+  else if (!tasks?.length) return tasks || [];
 
   for (const task of tasks) {
     sanitizeTask(task);

lib/auth.ts

@@ -52,9 +52,9 @@ function parseHeaderFeilds(str: string): Record<string, string> {
   return res;
 }
 
-export function parseAuthorizationHeader(
+export function parseAuthorizationHeader(authHeader: string): {
-  authHeader: string
+  method: string;
-): { method: string } {
+} {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };

lib/cache.ts

@@ -20,11 +20,12 @@
 import { Collection } from "mongodb";
 import { onConnect } from "./db";
 import * as config from "./config";
+import { Cache } from "./mongodb-types";
 
 const CLOCK_SKEW_TOLERANCE = 30000;
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let cacheCollection: Collection;
+let cacheCollection: Collection<Cache>;
 
 onConnect(async (db) => {
   cacheCollection = db.collection("cache");
@@ -33,8 +34,7 @@ onConnect(async (db) => {
 
 export async function get(key: string): Promise<string> {
   const res = await cacheCollection.findOne({ _id: key });
-  if (res) return res["value"];
+  return res?.value;
-  return null;
 }
 
 export async function del(key: string): Promise<void> {
@@ -52,15 +52,14 @@ export async function set(
   );
   await cacheCollection.replaceOne(
     { _id: key },
-    { _id: key, value, expire, timestamp },
+    { value, expire, timestamp },
     { upsert: true }
   );
 }
 
-export async function pop(key: string): Promise<any> {
+export async function pop(key: string): Promise<string> {
   const res = await cacheCollection.findOneAndDelete({ _id: key });
-  if (res && res["value"]) return res["value"]["value"];
+  return res.value?.value;
-  return null;
 }
 
 export async function acquireLock(
@@ -72,35 +71,23 @@ export async function acquireLock(
   try {
     const now = Date.now();
     const r = await cacheCollection.findOneAndUpdate(
-      { _id: lockName, token },
+      { _id: lockName, value: token },
       {
         $set: {
-          value: token,
           expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
         },
         $currentDate: { timestamp: true },
       },
-      { upsert: true, returnOriginal: false }
+      { upsert: true, returnDocument: "after" }
     );
-    const v = r.value;
+    if (Math.abs(r.value.timestamp.getTime() - now) > CLOCK_SKEW_TOLERANCE)
-    if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
       throw new Error("Database clock skew too great");
   } catch (err) {
-    if (err.code === 11000) {
+    if (err.code !== 11000) throw err;
-      if (timeout > 0) {
+    if (!(timeout > 0)) return null;
-        return new Promise((resolve, reject) => {
+    const w = 50 + Math.random() * 50;
-          const w = 50 + Math.random() * 50;
+    await new Promise((resolve) => setTimeout(resolve, w));
-          setTimeout(() => {
+    return acquireLock(lockName, ttl, timeout - w, token);
-            acquireLock(lockName, ttl, timeout - w, token).then(
-              resolve,
-              reject
-            );
-          }, w);
-        });
-      }
-      throw new Error("Failed to acquire lock");
-    }
-    throw err;
   }
 
   return token;
@@ -114,5 +101,5 @@ export async function releaseLock(
     _id: lockName,
     value: token,
   });
-  if (res["result"]["n"] !== 1) throw new Error("Lock expired");
+  if (res.deletedCount !== 1) throw new Error("Lock expired");
 }

lib/cluster.ts

@@ -17,14 +17,14 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import cluster from "cluster";
+import cluster, { Worker } from "cluster";
 import { cpus } from "os";
 import * as logger from "./logger";
 
 let respawnTimestamp = 0;
 let crashes: number[] = [];
 
-function fork(): cluster.Worker {
+function fork(): Worker {
   const w = cluster.fork();
   w.on("error", (err: NodeJS.ErrnoException) => {
     // Avoid exception when attempting to kill the process just as it's exiting

lib/common.ts

@@ -51,7 +51,7 @@ export function escapeRegExp(str: string): string {
 export function encodeTag(tag: string): string {
   return encodeURIComponent(tag)
     .replace(
-      /[!~*'()]/g,
+      /[!~*'().]/g,
       (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
     )
     .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")

lib/common/authorizer.ts

@@ -21,13 +21,13 @@ import { PermissionSet, Expression } from "../types";
 import { evaluate, or } from "./expression";
 
 export default class Authorizer {
-  private permissionSets: PermissionSet[];
+  private declare permissionSets: PermissionSet[];
-  private validatorCache: WeakMap<
+  private declare validatorCache: WeakMap<
     any,
     (mutationType, mutation, any) => boolean
   >;
-  private hasAccessCache: Map<string, boolean>;
+  private declare hasAccessCache: Map<string, boolean>;
-  private getFilterCache: Map<string, Expression>;
+  private declare getFilterCache: Map<string, Expression>;
 
   public constructor(permissionSets: PermissionSet[]) {
     this.permissionSets = permissionSets;

lib/common/boolean-expression.ts

@@ -29,8 +29,8 @@ const TWO = BI.BigInt(2);
 const NEGATIVE_ONE = BI.BigInt(-1);
 
 class Indeterminates {
-  public map: Map<string, number>;
+  public declare map: Map<string, number>;
-  public sortedKeys: string[];
+  public declare sortedKeys: string[];
 
   public constructor(str?: string) {
     this.map = new Map();
@@ -118,7 +118,7 @@ function findGcd(a: BI.bigint, b: BI.bigint): BI.bigint {
 }
 
 class Polynomial {
-  public terms: Term[];
+  public declare terms: Term[];
 
   public constructor(terms: Term[]) {
     this.terms = terms;
@@ -298,12 +298,11 @@ class Polynomial {
 }
 
 type Minterm = number[];
-type Context = Map<string, number>;
 
 abstract class BoolExprSynth {
-  public abstract true(context: Context): Minterm[];
+  public abstract true(context: SynthContext): Minterm[];
-  public abstract false(context: Context): Minterm[];
+  public abstract false(context: SynthContext): Minterm[];
-  public abstract null(context: Context): Minterm[];
+  public abstract null(context: SynthContext): Minterm[];
 }
 
 class TrueSynth extends BoolExprSynth {
@@ -343,8 +342,8 @@ class NullSynth extends BoolExprSynth {
 }
 
 class VarSynth extends BoolExprSynth {
-  private expStr: string;
+  private declare exp: Expression;
-  private negate: boolean;
+  private declare negate: boolean;
   public constructor(exp: Expression) {
     super();
     this.negate = false;
@@ -368,52 +367,49 @@ class VarSynth extends BoolExprSynth {
         this.negate = true;
       }
     }
-    this.expStr = JSON.stringify(exp);
+    this.exp = exp;
   }
-  public true(context: Context): Minterm[] {
+  public true(context: SynthContext): Minterm[] {
-    let v = context.get(this.expStr);
+    const v = context.getVariable(this.exp);
-    if (v == null) context.set(this.expStr, (v = context.size));
     return [[(v << 2) ^ (this.negate ? 1 : 3)]];
   }
-  public false(context: Context): Minterm[] {
+  public false(context: SynthContext): Minterm[] {
-    let v = context.get(this.expStr);
+    const v = context.getVariable(this.exp);
-    if (v == null) context.set(this.expStr, (v = context.size));
     return [[(v << 2) ^ (this.negate ? 3 : 1)]];
   }
-  public null(context: Context): Minterm[] {
+  public null(context: SynthContext): Minterm[] {
-    let v = context.get(this.expStr);
+    const v = context.getVariable(this.exp);
-    if (v == null) context.set(this.expStr, (v = context.size));
     return [[v << 2, (v << 2) ^ 2]];
   }
 }
 
 class NotSynth extends BoolExprSynth {
-  private exprSynth: BoolExprSynth;
+  private declare exprSynth: BoolExprSynth;
   public constructor(e: BoolExprSynth) {
     super();
     this.exprSynth = e;
   }
-  public true(context: Context): Minterm[] {
+  public true(context: SynthContext): Minterm[] {
     return this.exprSynth.false(context);
   }
-  public false(context: Context): Minterm[] {
+  public false(context: SynthContext): Minterm[] {
     return this.exprSynth.true(context);
   }
-  public null(context: Context): Minterm[] {
+  public null(context: SynthContext): Minterm[] {
     return this.exprSynth.null(context);
   }
 }
 
 class IsNullSynth extends BoolExprSynth {
-  private exprSynth: BoolExprSynth;
+  private declare exprSynth: BoolExprSynth;
   public constructor(e: BoolExprSynth) {
     super();
     this.exprSynth = e;
   }
-  public true(context: Context): Minterm[] {
+  public true(context: SynthContext): Minterm[] {
     return this.exprSynth.null(context);
   }
-  public false(context: Context): Minterm[] {
+  public false(context: SynthContext): Minterm[] {
     return [...this.exprSynth.true(context), ...this.exprSynth.false(context)];
   }
   public null(): Minterm[] {
@@ -422,7 +418,7 @@ class IsNullSynth extends BoolExprSynth {
 }
 
 class OrSynth extends BoolExprSynth {
-  private exprSynths: BoolExprSynth[];
+  private declare exprSynths: BoolExprSynth[];
   public constructor(...e: BoolExprSynth[]) {
     super();
     this.exprSynths = e.filter((ee) => !(ee instanceof FalseSynth));
@@ -436,13 +432,13 @@ class OrSynth extends BoolExprSynth {
     });
     this.exprSynths.push(...unpacked);
   }
-  public true(context: Context): Minterm[] {
+  public true(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].true(context);
     if (this.exprSynths.some((e) => e instanceof TrueSynth)) return [[]];
     return this.exprSynths.map((e) => e.true(context)).flat();
   }
-  public false(context: Context): Minterm[] {
+  public false(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [[]];
     if (this.exprSynths.length === 1) return this.exprSynths[0].false(context);
     if (
@@ -455,7 +451,7 @@ class OrSynth extends BoolExprSynth {
       this.exprSynths.map((e) => complement(e.false(context))).flat()
     );
   }
-  public null(context: Context): Minterm[] {
+  public null(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].null(context);
     const n = this.exprSynths.map((e) => e.null(context)).flat();
@@ -465,7 +461,7 @@ class OrSynth extends BoolExprSynth {
 }
 
 class AndSynth extends BoolExprSynth {
-  private exprSynths: BoolExprSynth[];
+  private declare exprSynths: BoolExprSynth[];
   public constructor(...e: BoolExprSynth[]) {
     super();
     this.exprSynths = e.filter((ee) => !(ee instanceof TrueSynth));
@@ -479,7 +475,7 @@ class AndSynth extends BoolExprSynth {
     });
     this.exprSynths.push(...unpacked);
   }
-  public true(context: Context): Minterm[] {
+  public true(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [[]];
     if (this.exprSynths.length === 1) return this.exprSynths[0].true(context);
     if (
@@ -492,13 +488,13 @@ class AndSynth extends BoolExprSynth {
       this.exprSynths.map((e) => complement(e.true(context))).flat()
     );
   }
-  public false(context: Context): Minterm[] {
+  public false(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].false(context);
     if (this.exprSynths.some((e) => e instanceof FalseSynth)) return [[]];
     return this.exprSynths.map((e) => e.false(context)).flat();
   }
-  public null(context: Context): Minterm[] {
+  public null(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].null(context);
     const n = this.exprSynths.map((e) => e.null(context)).flat();
@@ -508,12 +504,12 @@ class AndSynth extends BoolExprSynth {
 }
 
 class CaseSynth extends BoolExprSynth {
-  private exprSynths: BoolExprSynth[];
+  private declare exprSynths: BoolExprSynth[];
   public constructor(e: BoolExprSynth[]) {
     super();
     this.exprSynths = e;
   }
-  public true(context: Context): Minterm[] {
+  public true(context: SynthContext): Minterm[] {
     const minterms: Minterm[] = [];
     const cumulative: Minterm[] = [];
     for (let i = 0; i < this.exprSynths.length; i += 2) {
@@ -533,7 +529,7 @@ class CaseSynth extends BoolExprSynth {
     }
     return minterms;
   }
-  public false(context: Context): Minterm[] {
+  public false(context: SynthContext): Minterm[] {
     const minterms: Minterm[] = [];
     const cumulative: Minterm[] = [];
     for (let i = 0; i < this.exprSynths.length; i += 2) {
@@ -553,7 +549,7 @@ class CaseSynth extends BoolExprSynth {
     }
     return minterms;
   }
-  public null(context: Context): Minterm[] {
+  public null(context: SynthContext): Minterm[] {
     const minterms: Minterm[] = [];
     const cumulative: Minterm[] = [];
     for (let i = 0; i < this.exprSynths.length; i += 2) {
@@ -666,10 +662,10 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return (args.reduce(
+    return args.reduce(
       (previousValue, currentValue) => previousValue.add(currentValue),
       ADDITIVE_IDENTITY
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (op === "*") {
     const args: Polynomial[] = [];
     for (let i = 1; i < exp.length; ++i) {
@@ -677,10 +673,10 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return (args.reduce(
+    return args.reduce(
       (previousValue, currentValue) => previousValue.multiply(currentValue),
       MULTIPLICATIVE_IDENTITY
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (op === "-") {
     const args: Polynomial[] = [];
     for (let i = 1; i < exp.length; ++i) {
@@ -688,9 +684,9 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return (args.reduce((previousValue, currentValue) =>
+    return args.reduce((previousValue, currentValue) =>
       previousValue.subtract(currentValue)
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (op === "/") {
     const args: Polynomial[] = [];
     for (let i = 1; i < exp.length; ++i) {
@@ -698,9 +694,9 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return (args.reduce((previousValue, currentValue) =>
+    return args.reduce((previousValue, currentValue) =>
       previousValue.divide(currentValue)
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (["=", "<>", ">", ">=", "<", "<="].includes(op)) {
     if (exp[1] == null || exp[2] == null) return null;
     let lhs: Polynomial, rhs: Polynomial;
@@ -798,17 +794,14 @@ function toBoolExprSynth(e: Expression | BoolExprSynth): BoolExprSynth {
   return new FalseSynth();
 }
 
-function sopToExpression(
+function sopToExpression(sop: number[][], context: SynthContext): Expression {
-  sop: number[][],
-  expressions: Map<number, Expression>
-): Expression {
   if (!sop.length) return false;
   const res: Expression[] = [];
   for (const s of sop) {
     if (!s.length) return true;
     const conjs: Expression[] = [];
     for (const i of s) {
-      let expr = expressions.get(i >>> 2);
+      let expr = context.getExpression(i >>> 2);
       if (!(i & 1) !== !(i & 2)) expr = ["NOT", expr];
       if (Array.isArray(expr) && expr[0] === "NOT" && Array.isArray(expr[1])) {
         const e: Expression[] = expr[1];
@@ -832,163 +825,196 @@ function sopToExpression(
   return res[0];
 }
 
-function generateDcSetAndIsNull(
+function findIsNullDeps(exp: Expression): Expression[] {
-  variables: Map<number, Expression>
+  if (!Array.isArray(exp)) return [];
-): { dcSet: number[][]; isNull: Map<number, number> } {
+  const op = exp[0];
-  const relations: Map<
+  if (op === "IS NULL" || op === "IS NOT NULL") return [];
+
+  if (op === "FUNC") {
+    if (exp[1] === "NOW") {
+      return [];
+    } else if (exp[1] === "LOWER" || exp[1] === "UPPER") {
+      return findIsNullDeps(exp[2]);
+    } else if (exp[1] === "ROUND") {
+      return exp
+        .slice(2, 4)
+        .map((e) => findIsNullDeps(e))
+        .flat();
+    }
+  } else if (op !== "PARAM") {
+    return exp
+      .slice(1)
+      .map((e) => findIsNullDeps(e))
+      .flat();
+  }
+  return [exp];
+}
+
+class SynthContext {
+  private declare _dcSet: number[][];
+  private declare _variables: Map<string, number>;
+  private declare _expressions: Map<number, Expression>;
+  private declare _isNullRelations: Map<number, number[]>;
+  private declare _isNullVars: Set<number>;
+  private declare _comparisons: Map<
     string,
-    { op: string; rhs: boolean | number | string; var: number }[]
+    Map<string, { eq: number; gt: number; lt: number }>
-  > = new Map();
+  >;
 
-  for (const [n, e] of variables) {
+  constructor() {
-    const op = e[0] as string;
+    this._dcSet = [];
-    if (["=", ">", "<"].includes(op) && !Array.isArray(e[2])) {
+    this._variables = new Map();
-      const lhs = JSON.stringify(e[1]);
+    this._expressions = new Map();
-      let r = relations.get(lhs);
+    this._isNullRelations = new Map();
-      if (!r) relations.set(lhs, (r = []));
+    this._isNullVars = new Set();
-      r.push({ op: op, rhs: e[2], var: n });
+    this._comparisons = new Map();
-    } else if (op === "LIKE" && !Array.isArray(e[2]) && !Array.isArray(e[3])) {
+  }
-      const lhs = JSON.stringify(e[1]);
+
-      let r = relations.get(lhs);
+  getVariable(exp: Expression): number {
-      if (!r) relations.set(lhs, (r = []));
+    const expStr = JSON.stringify(exp);
-      r.push({ op: "", rhs: null, var: n });
+    if (this._variables.has(expStr)) return this._variables.get(expStr);
-    } else {
+    const v = this._variables.size;
-      const lhs = JSON.stringify(e);
+    this._variables.set(expStr, v);
-      let r = relations.get(lhs);
+    this._expressions.set(v, exp);
-      if (!r) relations.set(lhs, (r = []));
+    this.generateIsNullDc(v);
-      r.push({ op: "", rhs: null, var: n });
+    this.generateComparisionDc(v);
+
+    if (
+      expStr === '["IS NULL",["PARAM","DeviceID.ID"]]' ||
+      expStr === '["IS NULL",["PARAM","_id"]]'
+    )
+      this._dcSet.push([(v << 2) ^ 3]);
+
+    return v;
+  }
+
+  private generateIsNullDc(v: number): void {
+    this._isNullRelations.set(v, []);
+    const exp = this._expressions.get(v);
+
+    const isNull = new Set(
+      findIsNullDeps(exp).map((e) => this.getVariable(["IS NULL", e]))
+    );
+
+    if (!isNull.size) return;
+
+    for (const n of isNull) {
+      this._isNullVars.add(n);
+      this._isNullRelations.get(n).push(v);
+      this._isNullRelations.get(v).push(n);
+      this._dcSet.push([(n << 2) ^ 3, (v << 2) ^ 1]);
+      this._dcSet.push([(n << 2) ^ 3, (v << 2) ^ 3]);
+    }
+    this._dcSet.push(
+      [...isNull].map((n) => (n << 2) ^ 2).concat([(v << 2) ^ 0, (v << 2) ^ 2])
+    );
+  }
+
+  private generateComparisionDc(v: number): void {
+    const exp = this._expressions.get(v);
+    if (!Array.isArray(exp)) return;
+    const op = exp[0];
+    if (![">", "<", "="].includes(op)) return;
+    const rhs = exp[2];
+    const rhsStr = JSON.stringify(rhs);
+    const lhsStr = JSON.stringify(exp[1]);
+    let allComps = this._comparisons.get(lhsStr);
+    if (!allComps) this._comparisons.set(lhsStr, (allComps = new Map()));
+    if (allComps.has(rhsStr)) return;
+
+    const comp = {
+      eq: -1,
+      gt: -1,
+      lt: -1,
+    };
+
+    allComps.set(rhsStr, comp);
+
+    comp.eq = this.getVariable(["=", exp[1], exp[2]]);
+    comp.gt = this.getVariable([">", exp[1], exp[2]]);
+    comp.lt = this.getVariable(["<", exp[1], exp[2]]);
+
+    this._dcSet.push([
+      (comp.eq << 2) ^ 1,
+      (comp.gt << 2) ^ 1,
+      (comp.lt << 2) ^ 1,
+    ]);
+    this._dcSet.push([(comp.eq << 2) ^ 3, (comp.gt << 2) ^ 3]);
+    this._dcSet.push([(comp.eq << 2) ^ 3, (comp.lt << 2) ^ 3]);
+    this._dcSet.push([(comp.gt << 2) ^ 3, (comp.lt << 2) ^ 3]);
+
+    const type1 = typeof exp[2];
+    if (!["boolean", "number", "string"].includes(type1)) return;
+
+    for (const [rhs2Str, comp2] of allComps) {
+      if (comp2 === comp) continue;
+      const rhs2 = JSON.parse(rhs2Str);
+      const type2 = typeof rhs2;
+
+      if (!["boolean", "number", "string"].includes(type2)) continue;
+      let cmp = 0;
+      if (type1 === type2) cmp = rhs > rhs2 ? 1 : -1;
+      else if (type1 === "string") cmp = 1;
+      else if (type2 === "string") cmp = -1;
+      else cmp = +rhs - +rhs2;
+
+      const c1 = cmp > 0 ? comp : comp2;
+      const c2 = cmp > 0 ? comp2 : comp;
+
+      // This is the minimum clauses required if all relavent vars
+      // were included in the DC set.
+      // this._dcSet.push([(c1.eq << 2) ^ 3, (c2.eq << 2) ^ 3]);
+      // this._dcSet.push([(c1.lt << 2) ^ 1, (c2.gt << 2) ^ 1]);
+
+      // But we use non-minimal set because intermediate vars
+      // between any two may not be present in the DC set.
+      this._dcSet.push([(c1.lt << 2) ^ 1, (c2.gt << 2) ^ 1]);
+      this._dcSet.push([(c1.lt << 2) ^ 1, (c2.eq << 2) ^ 3]);
+      this._dcSet.push([(c1.lt << 2) ^ 1, (c2.lt << 2) ^ 3]);
+      this._dcSet.push([(c1.eq << 2) ^ 3, (c2.gt << 2) ^ 1]);
+      this._dcSet.push([(c1.eq << 2) ^ 3, (c2.eq << 2) ^ 3]);
+      this._dcSet.push([(c1.eq << 2) ^ 3, (c2.lt << 2) ^ 3]);
+      this._dcSet.push([(c1.gt << 2) ^ 3, (c2.gt << 2) ^ 1]);
+      this._dcSet.push([(c1.gt << 2) ^ 3, (c2.eq << 2) ^ 3]);
+      this._dcSet.push([(c1.gt << 2) ^ 3, (c2.lt << 2) ^ 3]);
     }
   }
 
-  const dcSet: number[][] = [];
+  getExpression(v: number): Expression {
-  const isNull: Map<number, number> = new Map();
+    return this._expressions.get(v);
-
-  for (const [key, rels] of relations) {
-    const n = variables.size;
-    variables.set(n, ["IS NULL", JSON.parse(key)]);
-
-    if (key === '["PARAM","DeviceID.ID"]' || key === '["PARAM","_id"]')
-      dcSet.push([(n << 2) ^ 3]);
-
-    const existingOps = new Map() as Map<string, Map<string, number>>;
-    for (const rel of rels) {
-      if (!rel.op) continue;
-      const rhsStr = JSON.stringify(rel.rhs);
-      const s = existingOps.get(rhsStr);
-      if (s) s.set(rel.op, rel.var);
-      else existingOps.set(rhsStr, new Map([[rel.op, rel.var]]));
-    }
-
-    for (const [k, v] of existingOps) {
-      if (v.size > 1) {
-        if (!v.has("=")) {
-          const va = variables.size;
-          variables.set(va, ["=", JSON.parse(key), JSON.parse(k)]);
-          rels.push({ op: "=", rhs: JSON.parse(k), var: va });
-          v.set("=", va);
-        }
-
-        if (!v.has(">")) {
-          const va = variables.size;
-          variables.set(va, [">", JSON.parse(key), JSON.parse(k)]);
-          rels.push({ op: ">", rhs: JSON.parse(k), var: va });
-          v.set(">", va);
-        }
-
-        if (!v.has("<")) {
-          const va = variables.size;
-          variables.set(va, ["<", JSON.parse(key), JSON.parse(k)]);
-          rels.push({ op: "<", rhs: JSON.parse(k), var: va });
-          v.set("<", va);
-        }
-
-        dcSet.push([
-          (v.get("=") << 2) ^ 1,
-          (v.get(">") << 2) ^ 1,
-          (v.get("<") << 2) ^ 1,
-        ]);
-      }
-    }
-
-    for (let i = 0; i < rels.length; ++i) {
-      const rel1 = rels[i];
-      isNull.set(rel1.var, n);
-
-      dcSet.push([(n << 2) ^ 3, (rel1.var << 2) ^ 1]);
-      dcSet.push([(n << 2) ^ 3, (rel1.var << 2) ^ 3]);
-      dcSet.push([(rel1.var << 2) ^ 0, (rel1.var << 2) ^ 2, (n << 2) ^ 2]);
-
-      if (!rel1.op) continue;
-      for (let j = i + 1; j < rels.length; ++j) {
-        const rel2 = rels[j];
-        if (!rel2.op) continue;
-        let comp = 0;
-        if (Array.isArray(rel1.rhs) || Array.isArray(rel2.rhs)) {
-          if (JSON.stringify(rel1.rhs) !== JSON.stringify(rel2.rhs)) continue;
-        } else if (typeof rel1.rhs === "string") {
-          if (typeof rel2.rhs === "string") {
-            if (rel1.rhs > rel2.rhs) comp = 1;
-            else if (rel1.rhs < rel2.rhs) comp = -1;
-          } else {
-            comp = 1;
-          }
-        } else if (typeof rel2.rhs === "string") {
-          comp = -1;
-        } else {
-          comp = +rel1.rhs - +rel2.rhs;
-        }
-
-        if (rel1.op === rel2.op) {
-          if (comp === 0)
-            dcSet.push([(rel1.var << 2) ^ 1, (rel2.var << 2) ^ 1]);
-          else if (rel1.op === "=")
-            dcSet.push([(rel1.var << 2) ^ 3, (rel2.var << 2) ^ 3]);
-          else if (
-            (rel1.op === ">" && comp > 0) ||
-            (rel1.op === "<" && comp < 0)
-          )
-            dcSet.push([(rel1.var << 2) ^ 3, (rel2.var << 2) ^ 1]);
-          else if (
-            (rel1.op === "<" && comp > 0) ||
-            (rel1.op === ">" && comp < 0)
-          )
-            dcSet.push([(rel1.var << 2) ^ 1, (rel2.var << 2) ^ 3]);
-        } else if (rel1.op === "=" || rel2.op === "=") {
-          let r1 = rel1,
-            r2 = rel2,
-            cmp = comp;
-          if (rel1.op !== "=") {
-            r2 = rel1;
-            r1 = rel2;
-            cmp = -comp;
-          }
-          if (r2.op === ">" || r2.op === "<") {
-            if (cmp === 0) {
-              dcSet.push([(r1.var << 2) ^ 3, (r2.var << 2) ^ 3]);
-            } else {
-              if ((cmp > 0 && r2.op === ">") || (cmp < 0 && r2.op === "<"))
-                dcSet.push([(r1.var << 2) ^ 3, (r2.var << 2) ^ 1]);
-              else dcSet.push([(r1.var << 2) ^ 3, (r2.var << 2) ^ 3]);
-            }
-          }
-        } else if (
-          [">", "<"].includes(rel1.op) &&
-          [">", "<"].includes(rel2.op)
-        ) {
-          if ((rel1.op === ">" && comp < 0) || (rel1.op === "<" && comp > 0))
-            dcSet.push([(rel1.var << 2) ^ 1, (rel2.var << 2) ^ 1]);
-          else dcSet.push([(rel1.var << 2) ^ 3, (rel2.var << 2) ^ 3]);
-        }
-      }
-    }
   }
 
-  return { dcSet, isNull };
+  getIsNullRelations(v: number): number[] {
+    return this._isNullRelations.get(v);
+  }
+
+  isIsNull(v: number): boolean {
+    return this._isNullVars.has(v);
+  }
+
+  getDcSet(minterms?: Minterm[]): number[][] {
+    if (!minterms) return this._dcSet;
+    const vars = new Set(minterms.flat().map((v) => v >> 2));
+
+    for (const comps of this._comparisons.values()) {
+      for (const comp of comps.values()) {
+        const c = Object.values(comp).filter((v) => !vars.has(v));
+        if (c.length === 1) vars.add(c[0]);
+      }
+    }
+
+    for (const n of this._isNullVars)
+      if (this._isNullRelations.get(n).some((v) => vars.has(v))) vars.add(n);
+
+    const dcSet = this._dcSet.filter((m) => m.every((v) => vars.has(v >> 2)));
+
+    return dcSet;
+  }
 }
 
 function sanitizeMinterms(
   minterms: number[][],
-  isNull: Map<number, number>
+  context: SynthContext
 ): number[][] {
   const res = [] as number[][];
   loop: for (const m of minterms) {
@@ -999,13 +1025,20 @@ function sanitizeMinterms(
     const perms: number[][] = [];
     for (const [k, v] of merged) {
       if (v === 0b1010) continue loop;
-      const n = isNull.get(k) << 2;
+      const isNullVars = context.getIsNullRelations(k);
       const t = k << 2;
-      if (v === 0b0101) minterm.push(n ^ 3);
+      if (v === 0b0101) {
-      else if (v === 0b0001) perms.push([n ^ 3, t ^ 3]);
+        if (isNullVars.length === 1) minterm.push((isNullVars[0] << 2) ^ 3);
-      else if (v === 0b0100) perms.push([n ^ 3, t ^ 1]);
+        else perms.push(isNullVars.map((n) => (n << 2) ^ 3));
-      else if (v & 0b1000) minterm.push(t ^ 3);
+      } else if (v === 0b0001) {
-      else if (v & 0b0010) minterm.push(t ^ 1);
+        perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 3]);
+      } else if (v === 0b0100) {
+        perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 1]);
+      } else if (v & 0b1000) {
+        minterm.push(t ^ 3);
+      } else if (v & 0b0010) {
+        minterm.push(t ^ 1);
+      }
     }
     let ms = [minterm];
     while (perms.length) {
@@ -1020,17 +1053,13 @@ function sanitizeMinterms(
 }
 
 function boolExprSynthToExpression(boolExpr: BoolExprSynth): Expression {
-  const context: Map<string, number> = new Map();
+  const context = new SynthContext();
   let minterms = boolExpr.true(context);
-  const variables = new Map(
+  minterms = sanitizeMinterms(minterms, context);
-    Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
+  const canRaise = getCanRaiseCallback(context);
-  );
+  const dcSet = context.getDcSet(minterms);
-  const { dcSet, isNull } = generateDcSetAndIsNull(variables);
-  minterms = sanitizeMinterms(minterms, isNull);
-
-  const canRaise = getCanRaiseCallback(isNull);
   minterms = espresso(minterms, dcSet, { canRaise });
-  return sopToExpression(minterms, variables);
+  return sopToExpression(minterms, context);
 }
 
 function mapCallback(exp: Expression | BoolExprSynth): Expression {
@@ -1043,22 +1072,22 @@ function mapCallback(exp: Expression | BoolExprSynth): Expression {
 
   const op = exp[0];
   if (op === "IS NULL")
-    return (new IsNullSynth(toBoolExprSynth(exp[1])) as unknown) as Expression;
+    return new IsNullSynth(toBoolExprSynth(exp[1])) as unknown as Expression;
 
   if (op === "IS NOT NULL") {
-    return (new NotSynth(
+    return new NotSynth(
       new IsNullSynth(toBoolExprSynth(exp[1]))
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (op === "NOT") {
-    return (new NotSynth(toBoolExprSynth(exp[1])) as unknown) as Expression;
+    return new NotSynth(toBoolExprSynth(exp[1])) as unknown as Expression;
   } else if (op === "OR") {
-    return (new OrSynth(
+    return new OrSynth(
       ...exp.slice(1).map((a) => toBoolExprSynth(a))
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (op === "AND") {
-    return (new AndSynth(
+    return new AndSynth(
       ...exp.slice(1).map((a) => toBoolExprSynth(a))
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   }
   for (let i = 1; i < exp.length; ++i) {
     if (exp[i] instanceof BoolExprSynth) {
@@ -1074,19 +1103,14 @@ function mapCallback(exp: Expression | BoolExprSynth): Expression {
 }
 
 function getCanRaiseCallback(
-  isNull: Map<number, number>
+  context: SynthContext
 ): (idx: number, set: Set<number>) => boolean {
   return (idx: number, set: Set<number>): boolean => {
     const i = idx >> 2;
-    const n = isNull.get(i);
+    const vars = context.getIsNullRelations(i);
-    if (n != null) {
+    if (!context.isIsNull(i)) return !(idx & 1) || !set.has(idx ^ 3);
-      if (!(idx & 1)) return true;
+
-      if (set.has((n << 2) ^ 2)) return true;
+    for (const k of vars) {
-      if (set.has((n << 2) ^ 3)) return true;
-      return !set.has(idx ^ 3);
-    }
-    for (const [k, v] of isNull) {
-      if (v !== i) continue;
       if (set.has((k << 2) ^ 1)) continue;
       if (set.has((k << 2) ^ 3)) continue;
       if (set.has((k << 2) ^ 0)) return false;
@@ -1101,33 +1125,31 @@ export function minimize(expr: Expression, boolean = false): Expression {
   expr = map(expr, mapCallback);
   if (Array.isArray(expr) && expr[0] === "CASE") {
     if (!boolean) {
-      const context: Map<string, number> = new Map();
+      const context = new SynthContext();
       const whens = expr.filter((e, i) => i % 2).map((e) => e.true(context));
-      const variables = new Map(
+      const caseDcSet: Minterm[] = [];
-        Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
+      const canRaise = getCanRaiseCallback(context);
-      );
-      const { dcSet, isNull } = generateDcSetAndIsNull(variables);
-      const canRaise = getCanRaiseCallback(isNull);
       const res: Expression = ["CASE"];
       for (let i = 1; i < expr.length; i += 2) {
-        let minterms = sanitizeMinterms(whens[(i - 1) / 2], isNull);
+        let minterms = sanitizeMinterms(whens[(i - 1) / 2], context);
+        const dcSet = context.getDcSet(minterms.concat(caseDcSet));
         minterms = espresso(minterms, dcSet, { canRaise });
         if (!minterms.length) continue;
-        const w = sopToExpression(minterms, variables);
+        const w = sopToExpression(minterms, context);
         let t = expr[i + 1];
         if (t instanceof BoolExprSynth) t = boolExprSynthToExpression(t);
         res.push(w, t);
         if (w === true) break;
-        dcSet.push(...minterms);
+        caseDcSet.push(...minterms);
       }
       while (res[res.length - 1] == null) res.splice(-2);
       if (res.length < 3) return null;
       return res;
     }
-    expr = (toBoolExprSynth(expr) as unknown) as Expression;
+    expr = toBoolExprSynth(expr) as unknown as Expression;
   }
 
-  if (boolean) expr = (toBoolExprSynth(expr) as unknown) as Expression;
+  if (boolean) expr = toBoolExprSynth(expr) as unknown as Expression;
 
   if (expr instanceof BoolExprSynth) expr = boolExprSynthToExpression(expr);
   return expr;
@@ -1145,7 +1167,7 @@ export function unionDiff(
 
   if (!expr1) {
     if (Array.isArray(expr2) && expr2[0] === "CASE")
-      expr2 = (toBoolExprSynth(expr2) as unknown) as Expression;
+      expr2 = toBoolExprSynth(expr2) as unknown as Expression;
     if (expr2 instanceof BoolExprSynth)
       expr2 = boolExprSynthToExpression(expr2);
     return [expr2, expr2];
@@ -1156,35 +1178,37 @@ export function unionDiff(
   expr1 = map(expr1, mapCallback);
   const b1 = toBoolExprSynth(expr1);
 
-  const context: Map<string, number> = new Map();
+  const context = new SynthContext();
 
   const expr2Minterms = b2.true(context);
   const expr1Minterms = b1.true(context);
   const expr1NullMinterms = b1.null(context);
   const expr1FalseMinterms = b1.false(context);
-  const variables = new Map(
-    Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
-  );
-  const { dcSet, isNull } = generateDcSetAndIsNull(variables);
 
   const unionMinterms = sanitizeMinterms(
     [...expr1Minterms, ...expr2Minterms],
-    isNull
+    context
   );
+
   const diffMinterms = sanitizeMinterms(
     complement([
       ...complement([...expr1NullMinterms, ...expr1FalseMinterms]),
       ...complement(expr2Minterms),
     ]),
-    isNull
+    context
   );
 
-  const canRaise = getCanRaiseCallback(isNull);
+  const canRaise = getCanRaiseCallback(context);
 
-  const union = espresso(unionMinterms, dcSet, { canRaise });
+  const union = espresso(unionMinterms, context.getDcSet(unionMinterms), {
-  const diff = espresso(diffMinterms, dcSet, { canRaise });
+    canRaise,
+  });
 
-  return [sopToExpression(union, variables), sopToExpression(diff, variables)];
+  const diff = espresso(diffMinterms, context.getDcSet(diffMinterms), {
+    canRaise,
+  });
+
+  return [sopToExpression(union, context), sopToExpression(diff, context)];
 }
 
 export function covers(expr1: Expression, expr2: Expression): boolean {
@@ -1198,13 +1222,13 @@ export function covers(expr1: Expression, expr2: Expression): boolean {
   expr2 = map(expr2, mapCallback);
   const b2 = toBoolExprSynth(expr2);
 
-  const context: Map<string, number> = new Map();
+  const context = new SynthContext();
   const expr1Minterms = b1.true(context);
   const expr2Minterms = b2.true(context);
-  const variables = new Map(
-    Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
-  );
-  const { dcSet } = generateDcSetAndIsNull(variables);
 
-  return tautology([...complement(expr2Minterms), ...dcSet, ...expr1Minterms]);
+  return tautology([
+    ...complement(expr2Minterms),
+    ...context.getDcSet(),
+    ...expr1Minterms,
+  ]);
 }

lib/common/debounce.ts

@@ -0,0 +1,16 @@
+export default function debounce<T>(
+  func: (args: T[]) => void,
+  timeout: number
+): (arg: T) => void {
+  let timer: ReturnType<typeof setTimeout>;
+  let args: T[] = [];
+  return (arg: T) => {
+    args.push(arg);
+    clearTimeout(timer);
+    timer = setTimeout(() => {
+      const argscopy = args;
+      args = [];
+      func(argscopy);
+    }, timeout);
+  };
+}

lib/common/expression-parser.ts

@@ -29,7 +29,7 @@ function interpretEscapes(str): string {
     r: "\r",
     t: "\t",
   };
-  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
+  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/g, (_, escape) => {
     const type = escape.charAt(0);
     const hex = escape.slice(1);
     if (type === "u") return String.fromCharCode(parseInt(hex, 16));
@@ -87,8 +87,6 @@ function binaryLeft(
       rest.reduce((acc, ch) => {
         const [op, another] = ch;
         if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
-        if (Array.isArray(another) && op === another[0])
-          return [op, acc].concat(another.slice(1));
         return [op, acc, another];
       }, first)
   );
@@ -287,7 +285,11 @@ const lang = parsimmon.createLanguage({
           .skip(parsimmon.optWhitespace),
         binaryLeft(
           parsimmon
-            .alt(parsimmon.string("*"), parsimmon.string("/"))
+            .alt(
+              parsimmon.string("*"),
+              parsimmon.string("/"),
+              parsimmon.string("%")
+            )
             .skip(parsimmon.optWhitespace),
           parsimmon.alt(
             r.Value,
@@ -386,6 +388,7 @@ export function stringify(exp: Expression, level = 0): string {
     "-": 31,
     "*": 32,
     "/": 32,
+    "%": 32,
   };
 
   const op = exp[0].toUpperCase();

lib/common/expression.ts

@@ -178,11 +178,11 @@ export function evaluateCallback(exp: Expression): Expression {
   } else if (exp[0] === "IS NULL") {
     if (isArray(exp[1])) return exp;
     else if (exp[1] == null) return true;
-    else return null;
+    else return false;
   } else if (exp[0] === "IS NOT NULL") {
     if (isArray(exp[1])) return exp;
     else if (exp[1] != null) return true;
-    else return null;
+    else return false;
   } else if (exp[0] === "LIKE") {
     if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
     else if (
@@ -244,9 +244,12 @@ export function evaluateCallback(exp: Expression): Expression {
   } else if (exp[0] === "/") {
     return reduce(exp, (a, b, i) => {
       if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b))
+      if (isArray(a) || isArray(b)) return REDUCE_SKIP;
-        return i === 0 ? toNumber(a) / toNumber(b) : toNumber(a) * toNumber(b);
+      const n1 = toNumber(a);
-      return REDUCE_SKIP;
+      const n2 = toNumber(b);
+      if (i !== 0) return n1 * n2;
+      if (n2 === 0) return null;
+      return n1 / n2;
     });
   } else if (exp[0] === "+") {
     return reduce(exp, (a, b) => {
@@ -261,6 +264,15 @@ export function evaluateCallback(exp: Expression): Expression {
         return i === 0 ? toNumber(a) - toNumber(b) : toNumber(a) + toNumber(b);
       return REDUCE_SKIP;
     });
+  } else if (exp[0] === "%") {
+    return reduce(exp, (a, b, i) => {
+      if (a == null || b == null) return null;
+      if (isArray(a) || isArray(b) || i !== 0) return REDUCE_SKIP;
+      const n1 = toNumber(a);
+      const n2 = Math.trunc(toNumber(b));
+      if (n2 === 0) return null;
+      return n1 % n2;
+    });
   } else if (exp[0] === "||") {
     return reduce(exp, (a, b) => {
       if (a == null || b == null) return null;
@@ -334,15 +346,7 @@ export async function evaluateAsync(
     if (!isArray(e)) return e;
 
     if (e[0] === "FUNC") {
-      if (e[1] === "NOW") {
+      if (e[1] === "NOW") if (now) return now;
-        if (now) return now;
-      } else if (e[1] === "UPPER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
-      } else if (e[1] === "LOWER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
-      }
     } else if (e[0] === "PARAM") {
       if (e[1] == null) return null;
       if (obj && !isArray(e[1])) {

lib/common/path-set.ts

@@ -20,9 +20,9 @@
 import Path from "./path";
 
 export default class PathSet {
-  private lengthIndex: Set<Path>[];
+  private declare lengthIndex: Set<Path>[];
-  private fragmentIndex: Map<string, Set<Path>>[];
+  private declare fragmentIndex: Map<string, Set<Path>>[];
-  private stringIndex: Map<string, Path>;
+  private declare stringIndex: Map<string, Path>;
 
   public constructor() {
     this.lengthIndex = [];

lib/common/path.ts

@@ -23,44 +23,102 @@ type Alias = [Path, string][];
 let cache1 = new Map<string, Path>();
 let cache2 = new Map<string, Path>();
 
+const CHAR_SPACE = 32;
+const CHAR_BACKSLASH = 92;
+const CHAR_COLON = 58;
+const CHAR_QUOTE = 34;
+const CHAR_OPEN_BRACKET = 91;
+const CHAR_CLOSE_BRACKET = 93;
+const CHAR_ASTERISK = 42;
+const CHAR_COMMA = 44;
+const CHAR_DOT = 46;
+
+function charCodeAt(str: string, index: number): number {
+  if (index >= str.length) return 0;
+  return str.charCodeAt(index);
+}
+
+function legalChar(c: number): boolean {
+  return (
+    (c >= 97 && c <= 122) ||
+    (c >= 65 && c <= 90) ||
+    (c >= 48 && c <= 57) ||
+    c === 95 ||
+    c === 45
+  );
+}
+
 export default class Path {
-  public readonly segments: Segments;
+  public declare readonly segments: Segments;
-  public readonly wildcard: number;
+  public declare readonly wildcard: number;
-  public readonly alias: number;
+  public declare readonly alias: number;
-  protected _string: string;
+  protected declare _string: string;
-  protected _stringIndex: number[];
+  protected declare _stringIndex: number[];
+
+  protected static parseAliasValue(
+    pattern: string,
+    index: number
+  ): { index: number; value: string } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) === CHAR_QUOTE) {
+      for (let j = i + 1; j < pattern.length; ++j) {
+        if (
+          pattern.charCodeAt(j) === CHAR_QUOTE &&
+          pattern.charCodeAt(j - 1) !== CHAR_BACKSLASH
+        ) {
+          try {
+            ++j;
+            const v = JSON.parse(pattern.slice(i, j));
+            return { index: j, value: v };
+          } catch {
+            return { index, value: null };
+          }
+        }
+      }
+      return { index, value: null };
+    }
+
+    for (; i < pattern.length; ++i) {
+      const c = pattern.charCodeAt(i);
+      if (c === CHAR_CLOSE_BRACKET || c === CHAR_COMMA) break;
+    }
+
+    return { index: i, value: pattern.slice(index, i).trim() };
+  }
+
+  protected static parseAliasPath(
+    pattern: string,
+    index: number
+  ): { index: number; path: Path } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    const { index: idx, segments } = Path.parsePath(pattern, i);
+    if (idx === i) return { index, path: null };
+    i = idx;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) !== CHAR_COLON) return { index, path: null };
+    return { index: i + 1, path: new Path(segments) };
+  }
 
   protected static parseAlias(
     pattern: string,
     index: number
   ): { index: number; alias: Alias } {
     const alias: Alias = [];
-    while (index < pattern.length && pattern[index] !== "]") {
+    let i = index;
-      const { index: idx, segments } = Path.parsePath(pattern, index);
+    for (;;) {
-      let j = (index = idx + 1);
+      let path: Path;
-      while (pattern[j] !== "]" && pattern[j] !== ",") {
+      let value: string;
-        if (pattern[j] === '"' && index === j) {
+      ({ index: i, path } = Path.parseAliasPath(pattern, i));
-          ++j;
+      if (!path) break;
-          while (pattern[j] !== '"' || pattern[j - 1] === "\\") {
+      ({ index: i, value } = Path.parseAliasValue(pattern, i));
-            if (++j >= pattern.length)
+      if (value == null) break;
-              throw new Error("Invalid alias expression");
+      alias.push([path, value]);
-          }
+      index = i;
-        }
-        if (++j >= pattern.length) throw new Error("Invalid alias expression");
-      }
 
-      let value = pattern.slice(index, j).trim();
+      if (charCodeAt(pattern, index) !== CHAR_COMMA) break;
-      index = j;
+      ++i;
-      if (value[0] === '"') {
-        try {
-          value = JSON.parse(value);
-        } catch (error) {
-          throw new Error("Invalid alias expression");
-        }
-      }
-
-      alias.push([new Path(segments), value]);
-      if (pattern[index] === ",") ++index;
     }
 
     // Ensure identical expressions have idential string representation
@@ -81,30 +139,26 @@ export default class Path {
     index: number
   ): { index: number; segments: Segments } {
     const segments = [];
-    // Colon separator is needed for parseAlias
-    if (index < pattern.length && pattern[index] !== ":") {
-      for (;;) {
-        if (pattern[index] === "[") {
-          const { index: idx, alias } = Path.parseAlias(pattern, index + 1);
-          index = idx + 1;
-          segments.push(alias);
-        } else {
-          const j = index;
-          while (
-            index < pattern.length &&
-            pattern[index] !== ":" &&
-            pattern[index] !== "."
-          )
-            ++index;
-          const s = pattern.slice(j, index).trim();
-          if (!s) throw new Error("Invalid parameter path");
-          segments.push(s);
-        }
 
-        if (index >= pattern.length || pattern[index] === ":") break;
+    let segStart = index;
-        else if (pattern[index] !== ".")
+    let wildcard = -1;
-          throw new Error("Invalid alias expression");
+    for (let i = index; i <= pattern.length; ++i) {
-        ++index;
+      const c = charCodeAt(pattern, i);
+      if (c === CHAR_OPEN_BRACKET && i === segStart) {
+        const { index: idx, alias } = Path.parseAlias(pattern, i + 1);
+        if (charCodeAt(pattern, idx) !== CHAR_CLOSE_BRACKET) break;
+        segments.push(alias);
+        index = i = idx + 1;
+        segStart = i + 1;
+      } else if (c === CHAR_ASTERISK && i === segStart) {
+        wildcard = i + 1;
+      } else if (!legalChar(c) || i === wildcard) {
+        if (i === segStart) break;
+        const s = pattern.slice(segStart, i);
+        segments.push(s);
+        index = i;
+        if (c !== CHAR_DOT) break;
+        segStart = i + 1;
       }
     }
 
@@ -143,7 +197,8 @@ export default class Path {
     if (!path) {
       path = cache2.get(str);
       if (!path) {
-        const { segments } = Path.parsePath(str, 0);
+        const { index, segments } = Path.parsePath(str, 0);
+        if (index < str.length) throw new Error("Invalid parameter path");
         path = new Path(segments);
         if (path.toString() !== str) cache1.set(path.toString(), path);
       }

lib/config.ts

@@ -43,7 +43,7 @@ const options = {
 
   CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
   CWMP_PORT: { type: "int", default: 7547 },
-  CWMP_INTERFACE: { type: "string", default: "0.0.0.0" },
+  CWMP_INTERFACE: { type: "string", default: "::" },
   CWMP_SSL_CERT: { type: "string", default: "" },
   CWMP_SSL_KEY: { type: "string", default: "" },
   CWMP_LOG_FILE: { type: "path", default: "" },
@@ -51,7 +51,7 @@ const options = {
 
   NBI_WORKER_PROCESSES: { type: "int", default: 0 },
   NBI_PORT: { type: "int", default: 7557 },
-  NBI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  NBI_INTERFACE: { type: "string", default: "::" },
   NBI_SSL_CERT: { type: "string", default: "" },
   NBI_SSL_KEY: { type: "string", default: "" },
   NBI_LOG_FILE: { type: "path", default: "" },
@@ -59,7 +59,7 @@ const options = {
 
   FS_WORKER_PROCESSES: { type: "int", default: 0 },
   FS_PORT: { type: "int", default: 7567 },
-  FS_INTERFACE: { type: "string", default: "0.0.0.0" },
+  FS_INTERFACE: { type: "string", default: "::" },
   FS_SSL_CERT: { type: "string", default: "" },
   FS_SSL_KEY: { type: "string", default: "" },
   FS_URL_PREFIX: { type: "string", default: "" },
@@ -68,7 +68,7 @@ const options = {
 
   UI_WORKER_PROCESSES: { type: "int", default: 0 },
   UI_PORT: { type: "int", default: 3000 },
-  UI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  UI_INTERFACE: { type: "string", default: "::" },
   UI_SSL_CERT: { type: "string", default: "" },
   UI_SSL_KEY: { type: "string", default: "" },
   UI_LOG_FILE: { type: "path", default: "" },

lib/connection-request.ts

@@ -19,7 +19,7 @@
 
 import * as crypto from "crypto";
 import * as dgram from "dgram";
-import { parse } from "url";
+import { URL } from "url";
 import * as http from "http";
 import { evaluateAsync } from "./common/expression";
 import { Expression } from "./types";
@@ -61,24 +61,26 @@ async function extractAuth(
 }
 
 function httpGet(
+  url: URL,
   options: http.RequestOptions,
   _debug: boolean,
   deviceId: string
 ): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
   return new Promise((resolve, reject) => {
     const req = http
-      .get(options, (res) => {
+      .get(url, options, (res) => {
         res.resume();
         resolve({ statusCode: res.statusCode, headers: res.headers });
         if (_debug) {
-          debug.outgoingHttpRequest(req, deviceId, options, null);
+          debug.outgoingHttpRequest(req, deviceId, "GET", url, null);
           debug.incomingHttpResponse(res, deviceId, null);
         }
       })
       .on("error", (err) => {
         req.destroy();
         reject(err);
-        if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
+        if (_debug)
+          debug.outgoingHttpRequestError(req, deviceId, "GET", url, err);
       })
       .on("timeout", () => {
         req.destroy();
@@ -94,15 +96,13 @@ export async function httpConnectionRequest(
   _debug: boolean,
   deviceId: string
 ): Promise<string> {
-  const options: http.RequestOptions = parse(address);
+  const url = new URL(address);
-  if (options.protocol !== "http:")
+  if (url.protocol !== "http:")
     return "Invalid connection request URL or protocol";
 
-  options.agent = new http.Agent({
+  const options: http.RequestOptions = {
-    maxSockets: 1,
+    agent: new http.Agent({ maxSockets: 1, keepAlive: true, timeout: timeout }),
-    keepAlive: true,
+  };
-    timeout: timeout,
-  });
 
   let authHeader: Record<string, string>;
   let username: string;
@@ -129,7 +129,7 @@ export async function httpConnectionRequest(
               Authorization: auth.solveDigest(
                 username,
                 password,
-                options.path,
+                url.pathname + url.search,
                 "GET",
                 null,
                 authHeader
@@ -145,12 +145,12 @@ export async function httpConnectionRequest(
 
     let res: { statusCode: number; headers: http.IncomingHttpHeaders };
     try {
-      res = await httpGet(opts, _debug, deviceId);
+      res = await httpGet(url, opts, _debug, deviceId);
     } catch (err) {
       // Workaround for some devices unexpectedly closing the connection
       if (authHeader) {
         try {
-          res = await httpGet(opts, _debug, deviceId);
+          res = await httpGet(url, opts, _debug, deviceId);
         } catch (err) {
           return `Connection request error: ${err.message}`;
         }
@@ -169,9 +169,13 @@ export async function httpConnectionRequest(
     if (res.statusCode === 503) return "Device is offline";
 
     if (res.statusCode === 401 && res.headers["www-authenticate"]) {
-      authHeader = auth.parseWwwAuthenticateHeader(
+      try {
-        res.headers["www-authenticate"]
+        authHeader = auth.parseWwwAuthenticateHeader(
-      );
+          res.headers["www-authenticate"]
+        );
+      } catch (err) {
+        return "Connection request error: Error parsing www-authenticate header";
+      }
       [username, password, authExp] = await extractAuth(authExp, false);
     } else {
       return `Connection request error: Unexpected status code ${res.statusCode}`;
@@ -182,14 +186,13 @@ export async function httpConnectionRequest(
 }
 
 export async function udpConnectionRequest(
-  address: string,
+  host: string,
+  port: number,
   authExp: Expression,
   sourcePort = 0,
   _debug: boolean,
   deviceId: string
 ): Promise<void> {
-  const [host, portStr] = address.split(":", 2);
-  const port = portStr ? parseInt(portStr) : 80;
   const now = Date.now();
 
   const client = dgram.createSocket({ type: "udp4", reuseAddr: true });
@@ -216,8 +219,8 @@ export async function udpConnectionRequest(
       .createHmac("sha1", password)
       .update(`${ts}${id}${username}${cn}`)
       .digest("hex");
-    const uri = `http://${address}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
+    const uri = `http://${host}:${port}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
-    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${address}\r\n\r\n`;
+    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${host}:${port}\r\n\r\n`;
     const message = Buffer.from(msg);
 
     for (let i = 0; i < 3; ++i) {

lib/cwmp.ts

@@ -45,6 +45,7 @@ import {
   InformRequest,
   Preset,
   GetRPCMethodsResponse,
+  CpeFault,
 } from "./types";
 import { IncomingMessage, ServerResponse } from "http";
 import { Readable } from "stream";
@@ -53,6 +54,7 @@ import { decode, encodingExists } from "iconv-lite";
 import { parseXmlDeclaration } from "./xml-parser";
 import * as debug from "./debug";
 import { getRequestOrigin } from "./forwarded";
+import { getSocketEndpoints } from "./server";
 
 const gzipPromisified = promisify(zlib.gzip);
 const deflatePromisified = promisify(zlib.deflate);
@@ -84,12 +86,16 @@ async function authenticate(
   let authentication;
 
   if (sessionContext.httpRequest.headers["authorization"]) {
-    authentication = auth.parseAuthorizationHeader(
+    try {
-      sessionContext.httpRequest.headers["authorization"]
+      authentication = auth.parseAuthorizationHeader(
-    );
+        sessionContext.httpRequest.headers["authorization"]
+      );
+    } catch (err) {
+      return false;
+    }
   }
 
-  if (authentication && authentication.method === "Digest") {
+  if (authentication?.method === "Digest") {
     const sessionNonce = sessionsNonces.get(sessionContext.httpRequest.socket);
 
     if (
@@ -183,7 +189,8 @@ async function writeResponse(
   }
 
   const httpResponse = sessionContext.httpResponse;
-  const connection = httpResponse.socket;
+  // Don't use httpResponse.socket as it may be null, even before end() is called
+  const connection = sessionContext.httpRequest.socket;
 
   httpResponse.setHeader("Content-Length", Buffer.byteLength(data));
   httpResponse.writeHead(res.code, res.headers);
@@ -206,12 +213,13 @@ async function writeResponse(
     currentSessions.set(connection, sessionContext);
     if (now >= sessionContext.extendLock) {
       sessionContext.extendLock = now + 10000;
-      await cache.acquireLock(
+      const lockToken = await cache.acquireLock(
         `cwmp_session_${sessionContext.deviceId}`,
-        sessionContext.timeout + 15000,
+        sessionContext.timeout * 1000 + 15000,
         0,
         sessionContext.sessionId
       );
+      if (!lockToken) throw new Error("Failed to extend lock");
     }
   }
 }
@@ -430,7 +438,7 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
   const deviceEvents = {};
   for (const p of deviceData.paths.find(Path.parse("Events.*"), false, true)) {
     const attrs = deviceData.attributes.get(p);
-    if (attrs && attrs.value && attrs.value[1][0] >= sessionContext.timestamp)
+    if (attrs?.value && attrs.value[1][0] >= sessionContext.timestamp)
       deviceEvents[p.segments[1] as string] = true;
   }
 
@@ -454,7 +462,7 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
 
     if (!eventsMatch) continue;
 
-    if (preset.schedule && preset.schedule.schedule) {
+    if (preset.schedule?.schedule) {
       const r = scheduling.cron(
         sessionContext.timestamp,
         preset.schedule.schedule
@@ -493,10 +501,11 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
     defer: true,
   }));
 
-  const { fault: flt, rpcId: reqId, rpc: acsReq } = await session.rpcRequest(
+  const {
-    sessionContext,
+    fault: flt,
-    declarations
+    rpcId: reqId,
-  );
+    rpc: acsReq,
+  } = await session.rpcRequest(sessionContext, declarations);
 
   if (flt) {
     recordFault(sessionContext, flt);
@@ -527,7 +536,7 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
               session.configContextCallback(sessionContext, e)
             )
           ),
-      ]);
+      ]) as [string, ...Expression[]][];
       if (blackList[p.channel] === 2) {
         appendProvisionsToFaults[p.channel] = (
           appendProvisionsToFaults[p.channel] || []
@@ -568,10 +577,11 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
 
   deviceData.timestamps.dirty = 0;
   deviceData.attributes.dirty = 0;
-  const { fault: fault, rpcId: id, rpc: acsRequest } = await session.rpcRequest(
+  const {
-    sessionContext,
+    fault: fault,
-    null
+    rpcId: id,
-  );
+    rpc: acsRequest,
+  } = await session.rpcRequest(sessionContext, null);
 
   if (fault) {
     recordFault(sessionContext, fault);
@@ -601,10 +611,11 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
 }
 
 async function nextRpc(sessionContext: SessionContext): Promise<void> {
-  const { fault: fault, rpcId: id, rpc: acsRequest } = await session.rpcRequest(
+  const {
-    sessionContext,
+    fault: fault,
-    null
+    rpcId: id,
-  );
+    rpc: acsRequest,
+  } = await session.rpcRequest(sessionContext, null);
 
   if (fault) {
     recordFault(sessionContext, fault);
@@ -782,7 +793,7 @@ async function endSession(sessionContext: SessionContext): Promise<void> {
     }
   }
 
-  if (sessionContext.doneTasks && sessionContext.doneTasks.length) {
+  if (sessionContext.doneTasks?.length) {
     saveCache = true;
     promises.push(
       db.clearTasks(sessionContext.deviceId, sessionContext.doneTasks)
@@ -874,37 +885,8 @@ async function sendAcsRequest(
   return writeResponse(sessionContext, res);
 }
 
-async function getSession(
-  connection: Socket,
-  sessionId: string
-): Promise<SessionContext> {
-  let sessionContext = currentSessions.get(connection);
-  if (sessionContext) {
-    currentSessions.delete(connection);
-    return sessionContext;
-  }
-
-  if (!sessionId) return null;
-
-  await new Promise((resolve) => setTimeout(resolve, 100));
-
-  const sessionContextString = await cache.pop(`session_${sessionId}`);
-  if (!sessionContextString) return null;
-  sessionContext = await session.deserialize(sessionContextString);
-  connection.setTimeout(sessionContext.timeout);
-  return sessionContext;
-}
-
-// Only needed to prevent tree shaking from removing the remoteAddress
-// workaround in onConnection function.
-const remoteAddressWorkaround = new WeakMap<Socket, string>();
-
 // When socket closes, store active sessions in cache
 export function onConnection(socket: Socket): void {
-  // The property remoteAddress may be undefined after the connection is
-  // closed, unless we read it at least once (caching?)
-  remoteAddressWorkaround.set(socket, socket.remoteAddress);
-
   socket.on("close", async () => {
     const sessionContext = currentSessions.get(socket);
     if (!sessionContext) return;
@@ -959,6 +941,32 @@ export function onConnection(socket: Socket): void {
   });
 }
 
+export function onClientError(err: Error, socket: Socket): void {
+  const remoteAddress = getSocketEndpoints(socket).remoteAddress;
+  localCache
+    .getCurrentSnapshot()
+    .then((cacheSnapshot) => {
+      const debugEnabled = !!localCache.getConfig(
+        cacheSnapshot,
+        "cwmp.debug",
+        {
+          remoteAddress: remoteAddress,
+        },
+        Date.now(),
+        (e) => {
+          if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
+            return remoteAddress;
+          return e;
+        }
+      );
+
+      if (debugEnabled) debug.clientError(remoteAddress, err);
+    })
+    .catch((err) => {
+      throw err;
+    });
+}
+
 setInterval(() => {
   if (stats.droppedRequests) {
     logger.warn({
@@ -1041,13 +1049,13 @@ async function reportBadState(sessionContext: SessionContext): Promise<void> {
     sessionContext: sessionContext,
   });
   const httpResponse = sessionContext.httpResponse;
-  currentSessions.delete(httpResponse.socket);
   const body = "Bad session state";
   httpResponse.setHeader("Content-Length", Buffer.byteLength(body));
   httpResponse.writeHead(400, { Connection: "close" });
   if (sessionContext.debug)
     debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, body);
   httpResponse.end(body);
+  if (sessionContext.state) return endSession(sessionContext);
 }
 
 async function responseUnauthorized(
@@ -1099,7 +1107,7 @@ async function processRequest(
   }
 
   if (sessionContext.state === 0) {
-    if (!rpc.cpeRequest || rpc.cpeRequest.name !== "Inform")
+    if (rpc.cpeRequest?.name !== "Inform")
       return reportBadState(sessionContext);
 
     const res = await inform(sessionContext, rpc);
@@ -1142,15 +1150,15 @@ async function processRequest(
       }
     }
 
-    try {
+    sessionContext.extendLock = sessionContext.timestamp + 10000;
-      sessionContext.extendLock = sessionContext.timestamp + 10000;
+    const lockToken = await cache.acquireLock(
-      await cache.acquireLock(
+      `cwmp_session_${sessionContext.deviceId}`,
-        `cwmp_session_${sessionContext.deviceId}`,
+      sessionContext.timeout * 1000 + 15000,
-        sessionContext.timeout + 15000,
+      0,
-        0,
+      sessionContext.sessionId
-        sessionContext.sessionId
+    );
-      );
+
-    } catch (err) {
+    if (!lockToken) {
       logger.accessError({
         message: "CPE already in session",
         sessionContext: sessionContext,
@@ -1269,6 +1277,41 @@ async function processRequest(
       session.clearProvisions(sessionContext);
     }
     return nextRpc(sessionContext);
+  } else if (rpc.unknownMethod) {
+    if (sessionContext.state === 1) {
+      logger.accessWarn({
+        sessionContext: sessionContext,
+        message: "Method not supported",
+        method: rpc.unknownMethod,
+      });
+
+      const f: CpeFault = {
+        faultCode: "Server",
+        faultString: "CWMP fault",
+        detail: {
+          faultCode: "8000",
+          faultString: "Method not supported",
+        },
+      };
+
+      const res = soap.response({
+        id: rpc.id,
+        acsFault: f,
+        cwmpVersion: sessionContext.cwmpVersion,
+      });
+
+      return writeResponse(sessionContext, res);
+    } else if (sessionContext.state === 2) {
+      const fault = {
+        code: "invalid_response",
+        message: "Response name does not match request name",
+      };
+      recordFault(sessionContext, fault);
+      session.clearProvisions(sessionContext);
+      return nextRpc(sessionContext);
+    } else {
+      return reportBadState(sessionContext);
+    }
   } else {
     // CPE sent empty response
     if (sessionContext.state !== 1) return reportBadState(sessionContext);
@@ -1304,13 +1347,13 @@ export function listener(
       stats.concurrentRequests -= 1;
     })
     .catch((err) => {
-      currentSessions.delete(httpResponse.socket);
+      currentSessions.delete(httpRequest.socket);
       stats.concurrentRequests -= 1;
       setTimeout(() => {
         throw err;
       });
       try {
-        httpResponse.socket.unref();
+        httpRequest.socket.unref();
         httpResponse.writeHead(500, { Connection: "close" });
         httpResponse.end(`${err.name}: ${err.message}`);
       } catch (err) {
@@ -1319,6 +1362,48 @@ export function listener(
     });
 }
 
+async function clientError(
+  httpRequest: IncomingMessage,
+  httpResponse: ServerResponse,
+  sessionContext: SessionContext,
+  body: string,
+  msg: string
+): Promise<void> {
+  let debugEnabled = false;
+  let deviceId: string = null;
+
+  if (sessionContext) {
+    debugEnabled = sessionContext.debug;
+    deviceId = sessionContext.deviceId;
+  } else {
+    const cacheSnapshot = await localCache.getCurrentSnapshot();
+    debugEnabled = !!localCache.getConfig(
+      cacheSnapshot,
+      "cwmp.debug",
+      {
+        remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
+      },
+      Date.now(),
+      (e) => {
+        if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
+          return getRequestOrigin(httpRequest).remoteAddress;
+        return e;
+      }
+    );
+  }
+
+  httpResponse.setHeader("Content-Length", Buffer.byteLength(msg));
+  httpResponse.writeHead(400, { Connection: "close" });
+
+  if (debugEnabled) {
+    debug.incomingHttpRequest(httpRequest, deviceId, body);
+    debug.outgoingHttpResponse(httpResponse, deviceId, msg);
+  }
+
+  httpResponse.end(msg);
+  if (sessionContext?.state) await endSession(sessionContext);
+}
+
 function decodeString(buffer: Buffer, charset: string): string {
   try {
     return buffer.toString(charset as BufferEncoding);
@@ -1345,7 +1430,8 @@ async function listenerAsync(
 
   let sessionId;
   // Separation by comma is important as some devices don't comform to standard
-  const COOKIE_REGEX = /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g;
+  const COOKIE_REGEX =
+    /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g;
   let match;
   while ((match = COOKIE_REGEX.exec(httpRequest.headers.cookie)))
     if (match[1] === "session") sessionId = match[2];
@@ -1406,15 +1492,14 @@ async function listenerAsync(
   // Request aborted
   if (!body) return;
 
-  const newConnection = !currentSessions.has(httpRequest.socket);
+  let sessionContext = currentSessions.get(httpRequest.socket);
-
-  const sessionContext = await getSession(httpRequest.socket, sessionId);
 
   if (sessionContext) {
+    currentSessions.delete(httpRequest.socket);
     sessionContext.httpRequest = httpRequest;
     sessionContext.httpResponse = httpResponse;
     if (
-      (newConnection && sessionContext.sessionId !== sessionId) ||
+      (sessionContext.sessionId !== sessionId && sessionContext.state) ||
       sessionContext.lastActivity + sessionContext.timeout * 1000 < Date.now()
     ) {
       logger.accessError({
@@ -1422,29 +1507,14 @@ async function listenerAsync(
         sessionContext: sessionContext,
       });
 
-      const _body = "Invalid session";
+      return clientError(
-      httpResponse.setHeader("Content-Length", Buffer.byteLength(_body));
+        httpRequest,
-      httpResponse.writeHead(400, { Connection: "close" });
+        httpResponse,
-      if (sessionContext.debug) {
+        sessionContext,
-        debug.outgoingHttpResponse(
+        body.toString(),
-          httpResponse,
+        "Invalid session"
-          sessionContext.deviceId,
+      );
-          _body
-        );
-      }
-      httpResponse.end(_body);
-      await endSession(sessionContext);
-      return;
     }
-    if (newConnection && sessionContext.authState !== 1)
-      sessionContext.authState = 0;
-  } else if (stats.concurrentRequests > MAX_CONCURRENT_REQUESTS) {
-    // Check again just in case device included old session ID
-    // from the previous session
-    httpResponse.writeHead(503, { "Retry-after": 60, Connection: "close" });
-    httpResponse.end("503 Service Unavailable");
-    stats.droppedRequests += 1;
-    return;
   }
 
   let charset: string;
@@ -1464,6 +1534,16 @@ async function listenerAsync(
   const bodyStr = decodeString(body, charset);
 
   if (bodyStr == null) {
+    if (!sessionContext && sessionId) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      const sessionContextString = await cache.pop(`session_${sessionId}`);
+      if (sessionContextString) {
+        sessionContext = await session.deserialize(sessionContextString);
+        sessionContext.httpRequest = httpRequest;
+        sessionContext.httpResponse = httpResponse;
+      }
+    }
+
     const msg = `Unknown encoding '${charset}'`;
     logger.accessError({
       message: "XML parse error",
@@ -1473,132 +1553,87 @@ async function listenerAsync(
         httpResponse: httpResponse,
       },
     });
-    httpResponse.setHeader("Content-Length", Buffer.byteLength(msg));
+    return clientError(
-    httpResponse.writeHead(400, { Connection: "close" });
+      httpRequest,
-    if (sessionContext) {
+      httpResponse,
-      if (sessionContext.debug) {
+      sessionContext,
-        debug.incomingHttpRequest(
+      body.toString(),
-          httpRequest,
+      msg
-          sessionContext.deviceId,
+    );
-          body.toString()
-        );
-        debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, msg);
-      }
-      await endSession(sessionContext);
-    } else {
-      const cacheSnapshot = await localCache.getCurrentSnapshot();
-      const d = !!localCache.getConfig(
-        cacheSnapshot,
-        "cwmp.debug",
-        {
-          remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
-        },
-        Date.now(),
-        (e) => {
-          if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
-            return getRequestOrigin(httpRequest).remoteAddress;
-          return e;
-        }
-      );
-      if (d) {
-        debug.incomingHttpRequest(httpRequest, null, body.toString());
-        debug.outgoingHttpResponse(httpResponse, null, msg);
-      }
-    }
-    httpResponse.end(msg);
-    return;
   }
 
   const parseWarnings = [];
   let rpc: SoapMessage;
   try {
-    rpc = soap.request(
+    rpc = soap.request(bodyStr, parseWarnings);
-      bodyStr,
+  } catch (err) {
-      sessionContext ? sessionContext.cwmpVersion : null,
+    if (!sessionContext && sessionId) {
-      parseWarnings
+      await new Promise((resolve) => setTimeout(resolve, 100));
-    );
+      const sessionContextString = await cache.pop(`session_${sessionId}`);
-  } catch (error) {
+      if (sessionContextString) {
+        sessionContext = await session.deserialize(sessionContextString);
+        sessionContext.httpRequest = httpRequest;
+        sessionContext.httpResponse = httpResponse;
+      }
+    }
+
     logger.accessError({
       message: "XML parse error",
-      parseError: error.message.trim(),
+      parseError: err.message,
       sessionContext: sessionContext || {
         httpRequest: httpRequest,
         httpResponse: httpResponse,
       },
     });
-    httpResponse.setHeader("Content-Length", Buffer.byteLength(error.message));
+
-    httpResponse.writeHead(400, { Connection: "close" });
+    return clientError(
-    if (sessionContext) {
+      httpRequest,
-      if (sessionContext.debug) {
+      httpResponse,
-        debug.incomingHttpRequest(
+      sessionContext,
-          httpRequest,
+      bodyStr,
-          sessionContext.deviceId,
+      err.message
-          bodyStr
+    );
-        );
+  }
-        debug.outgoingHttpResponse(
+
-          httpResponse,
+  if (!sessionContext && sessionId && rpc.cpeRequest?.name !== "Inform") {
-          sessionContext.deviceId,
+    await new Promise((resolve) => setTimeout(resolve, 100));
-          error.message
+    const sessionContextString = await cache.pop(`session_${sessionId}`);
-        );
+    if (sessionContextString) {
-      }
+      sessionContext = await session.deserialize(sessionContextString);
-      await endSession(sessionContext);
+      sessionContext.httpRequest = httpRequest;
-    } else {
+      sessionContext.httpResponse = httpResponse;
-      const cacheSnapshot = await localCache.getCurrentSnapshot();
+      httpRequest.socket.setTimeout(sessionContext.timeout * 1000);
-      const d = !!localCache.getConfig(
+      if (sessionContext.authState !== 1) sessionContext.authState = 0;
-        cacheSnapshot,
-        "cwmp.debug",
-        {
-          remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
-        },
-        Date.now(),
-        (e) => {
-          if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
-            return getRequestOrigin(httpRequest).remoteAddress;
-          return e;
-        }
-      );
-      if (d) {
-        debug.incomingHttpRequest(httpRequest, null, bodyStr);
-        debug.outgoingHttpResponse(httpResponse, null, error.message);
-      }
     }
-    httpResponse.end(error.message);
-    return;
   }
 
   if (sessionContext)
     return processRequest(sessionContext, rpc, parseWarnings, bodyStr);
 
-  if (!(rpc.cpeRequest && rpc.cpeRequest.name === "Inform")) {
+  if (rpc.cpeRequest?.name !== "Inform") {
     logger.accessError({
       message: "Invalid session",
-      sessionContext: sessionContext || {
+      sessionContext: {
         httpRequest: httpRequest,
         httpResponse: httpResponse,
       },
     });
-    const _body = "Invalid session";
+
-    httpResponse.setHeader("Content-Length", Buffer.byteLength(_body));
+    return clientError(
-    httpResponse.writeHead(400, { Connection: "close" });
+      httpRequest,
-    const cacheSnapshot = await localCache.getCurrentSnapshot();
+      httpResponse,
-    const d = !!localCache.getConfig(
+      null,
-      cacheSnapshot,
+      bodyStr,
-      "cwmp.debug",
+      "Invalid session"
-      {
-        remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
-      },
-      Date.now(),
-      (e) => {
-        if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
-          return getRequestOrigin(httpRequest).remoteAddress;
-        return e;
-      }
     );
-    if (d) {
+  }
-      debug.incomingHttpRequest(httpRequest, null, bodyStr);
+
-      debug.outgoingHttpResponse(httpResponse, null, _body);
+  if (stats.concurrentRequests > MAX_CONCURRENT_REQUESTS) {
-    }
+    // Check again just in case device included old session ID
-    httpResponse.end(_body);
+    // from the previous session
+    httpResponse.writeHead(503, { "Retry-after": 60, Connection: "close" });
+    httpResponse.end("503 Service Unavailable");
+    stats.droppedRequests += 1;
     return;
   }
 

lib/db.ts

@@ -17,9 +17,9 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, ObjectID, Collection, Db } from "mongodb";
+import { MongoClient, ObjectId, Collection, Db } from "mongodb";
 import { get } from "./config";
-import { encodeTag, escapeRegExp } from "./common";
+import { decodeTag, encodeTag, escapeRegExp } from "./common";
 import { parse } from "./common/expression-parser";
 import {
   DeviceData,
@@ -30,22 +30,25 @@ import {
   Expression,
 } from "./types";
 import Path from "./common/path";
+import * as MongoTypes from "./mongodb-types";
 
-export let tasksCollection: Collection,
+export let tasksCollection: Collection<MongoTypes.Task>,
   devicesCollection: Collection,
   presetsCollection: Collection,
   objectsCollection: Collection,
   provisionsCollection: Collection,
   virtualParametersCollection: Collection,
-  faultsCollection: Collection,
+  faultsCollection: Collection<MongoTypes.Fault>,
   filesCollection: Collection,
-  operationsCollection: Collection,
+  operationsCollection: Collection<MongoTypes.Operation>,
   permissionsCollection: Collection,
   usersCollection: Collection,
-  configCollection: Collection;
+  configCollection: Collection<MongoTypes.Config>;
 
 let clientPromise: Promise<MongoClient>;
 
+const INVALID_PATH_SUFFIX = "__invalid";
+
 function compareAccessLists(list1: string[], list2: string[]): boolean {
   if (list1.length !== list2.length) return false;
   for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
@@ -59,10 +62,7 @@ export function onConnect(callback: (db: Db) => Promise<void>): void {
 }
 
 export async function connect(): Promise<void> {
-  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"));
-    useNewUrlParser: true,
-    useUnifiedTopology: true,
-  });
 
   const client = await clientPromise;
   const db = client.db();
@@ -92,7 +92,9 @@ export async function disconnect(): Promise<void> {
 
 // Optimize projection by removing overlaps
 // This can modify the object
-function optimizeProjection(obj: { [path: string]: 1 }): { [path: string]: 1 } {
+export function optimizeProjection(obj: { [path: string]: 1 }): {
+  [path: string]: 1;
+} {
   if (obj[""]) return { "": obj[""] };
 
   const keys = Object.keys(obj).sort();
@@ -147,7 +149,7 @@ export async function fetchDevice(
 
     if (obj["_value"] != null) {
       attrs.value = [obj["_timestamp"] || 1, [obj["_value"], obj["_type"]]];
-      if (obj["_type"] === "xsd:dateTime")
+      if (obj["_type"] === "xsd:dateTime" && obj["_value"] instanceof Date)
         attrs.value[1][0] = +attrs.value[1][0];
 
       obj["_object"] = false;
@@ -167,7 +169,19 @@ export async function fetchDevice(
     if (obj["_accessList"] != null)
       attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
 
-    res.push([Path.parse(path), t, attrs]);
+    try {
+      res.push([Path.parse(path), t, attrs]);
+    } catch (err) {
+      // The path parser is now more strict so we might be in a situation where
+      // the database contains invalid paths from before this change So here we
+      // encode the invalid characters.
+      const splits = path.split(".");
+      splits[splits.length - 1] =
+        encodeTag(splits[splits.length - 1]) + INVALID_PATH_SUFFIX;
+      path = splits.join(".");
+      res.push([Path.parse(path), t, attrs]);
+      return;
+    }
 
     for (const [k, v] of Object.entries(obj)) {
       if (!k.startsWith("_")) {
@@ -435,21 +449,34 @@ export async function saveDevice(
           if (value2 != null) {
             if (!update["$addToSet"]["_tags"])
               update["$addToSet"]["_tags"] = { $each: [] };
-            update["$addToSet"]["_tags"]["$each"].push(path.segments[1]);
+            update["$addToSet"]["_tags"]["$each"].push(
+              decodeTag(path.segments[1] as string)
+            );
           } else {
             if (!update["$pull"]["_tags"]) {
               update["$pull"]["_tags"] = {
                 $in: [],
               };
             }
-            update["$pull"]["_tags"]["$in"].push(path.segments[1]);
+            update["$pull"]["_tags"]["$in"].push(
+              decodeTag(path.segments[1] as string)
+            );
           }
         }
 
         break;
       default:
         if (!diff[2]) {
-          update["$unset"][path.toString()] = 1;
+          let pathStr = path.toString();
+          // Paths with that suffix are encoded and need to be decoded
+          if (pathStr.endsWith(INVALID_PATH_SUFFIX)) {
+            const splits = pathStr.split(".");
+            splits[splits.length - 1] = decodeTag(
+              splits[splits.length - 1].slice(0, 0 - INVALID_PATH_SUFFIX.length)
+            );
+            pathStr = splits.join(".");
+          }
+          update["$unset"][pathStr] = 1;
           continue;
         }
 
@@ -485,7 +512,7 @@ export async function saveDevice(
 
                 break;
               case "object":
-                if (!diff[1] || !diff[1].object || object2 !== object1) {
+                if (!diff[1]?.object || object2 !== object1) {
                   update["$set"][
                     path.length ? path.toString() + "._object" : "_object"
                   ] = !!object2;
@@ -493,7 +520,7 @@ export async function saveDevice(
 
                 break;
               case "writable":
-                if (!diff[1] || !diff[1].writable || writable2 !== writable1) {
+                if (!diff[1]?.writable || writable2 !== writable1) {
                   update["$set"][
                     path.length ? path.toString() + "._writable" : "_writable"
                   ] = !!writable2;
@@ -514,9 +541,8 @@ export async function saveDevice(
                 }
 
                 if (attributesTimestamp2 !== attributesTimestamp1) {
-                  update["$set"][
+                  update["$set"][path.toString() + "._attributesTimestamp"] =
-                    path.toString() + "._attributesTimestamp"
+                    new Date(attributesTimestamp2);
-                  ] = new Date(attributesTimestamp2);
                 }
 
                 break;
@@ -534,9 +560,8 @@ export async function saveDevice(
                 }
 
                 if (attributesTimestamp2 !== attributesTimestamp1) {
-                  update["$set"][
+                  update["$set"][path.toString() + "._attributesTimestamp"] =
-                    path.toString() + "._attributesTimestamp"
+                    new Date(attributesTimestamp2);
-                  ] = new Date(attributesTimestamp2);
                 }
             }
           }
@@ -546,7 +571,7 @@ export async function saveDevice(
           for (const attrName of Object.keys(diff[1])) {
             if (
               diff[1][attrName][1] != null &&
-              (!diff[2] || !diff[2][attrName] || diff[2][attrName][1] == null)
+              diff[2]?.[attrName]?.[1] == null
             ) {
               const p = path.length ? path.toString() + "." : "";
               update["$unset"][`${p}_${attrName}`] = 1;
@@ -600,7 +625,7 @@ export async function saveDevice(
     upsert: isNew,
   });
 
-  if (result.result.n !== 1)
+  if (!result.matchedCount && !result.upsertedCount)
     throw new Error(`Device ${deviceId} not found in database`);
 
   if (update2) {
@@ -619,12 +644,16 @@ export async function getFaults(
   const faults: { [channel: string]: SessionFault } = {};
   for (const r of res) {
     const channel = r._id.slice(deviceId.length + 1);
-    delete r._id;
+    const fault: SessionFault = {
-    delete r.channel;
+      code: r.code,
-    delete r.device;
+      message: r.message,
-    r.timestamp = +r.timestamp;
+      ...(r.detail && { detail: r.detail }),
-    r.provisions = JSON.parse(r.provisions);
+      timestamp: +r.timestamp,
-    faults[channel] = r;
+      provisions: JSON.parse(r.provisions),
+      retries: r.retries,
+      ...(r.expiry && { expiry: +r.expiry }),
+    };
+    faults[channel] = fault;
   }
 
   return faults;
@@ -636,12 +665,18 @@ export async function saveFault(
   fault: SessionFault
 ): Promise<void> {
   const id = `${deviceId}:${channel}`;
-  const f = Object.assign({}, fault) as Record<string, any>;
+  const f: MongoTypes.Fault = {
-  f["_id"] = id;
+    _id: id,
-  f["device"] = deviceId;
+    device: deviceId,
-  f["channel"] = channel;
+    channel: channel,
-  f["timestamp"] = new Date(fault.timestamp);
+    timestamp: new Date(fault.timestamp),
-  f["provisions"] = JSON.stringify(fault.provisions);
+    code: fault.code,
+    message: fault.message,
+    ...(fault.detail && { detail: fault.detail }),
+    retries: fault.retries,
+    ...(fault.expiry && { expiry: new Date(fault.expiry) }),
+    provisions: JSON.stringify(fault.provisions),
+  };
   await faultsCollection.replaceOne({ _id: id }, f, { upsert: true });
 }
 
@@ -659,20 +694,47 @@ export async function getDueTasks(
   const cur = tasksCollection.find({ device: deviceId }).sort({ timestamp: 1 });
   const tasks = [] as Task[];
 
-  for await (const task of cur) {
+  for await (const t of cur) {
-    if (task.timestamp) task.timestamp = +task.timestamp;
+    if (+t.timestamp >= timestamp) return [tasks, +t.timestamp];
-    if (task.expiry) task.expiry = +task.expiry;
+    const task: Task = {
-    if (task.timestamp >= timestamp) return [tasks, +task.timestamp];
+      _id: t._id.toString(),
-    task._id = String(task._id);
+      name: t.name,
+      ...(t.timestamp && { timestamp: +t.timestamp }),
+      ...(t.expiry && { expiry: +t.expiry }),
+      ...(t.name === "getParameterValues" && {
+        parameterNames: t.parameterNames,
+      }),
+      ...(t.name === "setParameterValues" && {
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "refreshObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "download" && {
+        fileType: t.fileType,
+        fileName: t.fileName,
+        targetFileName: t.targetFileName,
+      }),
+      ...(t.name === "addObject" && {
+        objectName: t.objectName,
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "deleteObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "provisions" && {
+        provisions: t.provisions,
+      }),
+    };
 
     tasks.push(task);
 
     // For API compatibility
-    if (task.name === "download" && task.file) {
+    if (task.name === "download" && t["file"]) {
       let q;
-      if (ObjectID.isValid(task.file))
+      if (ObjectId.isValid(t["file"]))
-        q = { _id: { $in: [task.file, new ObjectID(task.file)] } };
+        q = { _id: { $in: [t["file"], new ObjectId(t["file"])] } };
-      else q = { _id: task.file };
+      else q = { _id: t["file"] };
 
       const res = await filesCollection.find(q).toArray();
 
@@ -692,7 +754,7 @@ export async function clearTasks(
   taskIds: string[]
 ): Promise<void> {
   await tasksCollection.deleteMany({
-    _id: { $in: taskIds.map((id) => new ObjectID(id)) },
+    _id: { $in: taskIds.map((id) => new ObjectId(id)) },
   });
 }
 
@@ -706,17 +768,22 @@ export async function getOperations(
   const operations: { [commandKey: string]: Operation } = {};
   for (const r of res) {
     const commandKey = r._id.slice(deviceId.length + 1);
-    delete r._id;
     // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
     if (typeof r.provisions !== "string") {
-      operations[commandKey] = r;
+      delete r._id;
+      operations[commandKey] = r as unknown as Operation;
       continue;
     }
-    r.timestamp = +r.timestamp;
+    const operation: Operation = {
-    if (r.args) r.args = JSON.parse(r.args);
+      name: r.name,
-    r.provisions = JSON.parse(r.provisions);
+      timestamp: +r.timestamp,
-    r.retries = JSON.parse(r.retries);
+      channels:
-    operations[commandKey] = r;
+        typeof r.channels === "string" ? JSON.parse(r.channels) : r.channels,
+      retries: JSON.parse(r.retries),
+      provisions: JSON.parse(r.provisions),
+      ...(r.args && { args: JSON.parse(r.args) }),
+    };
+    operations[commandKey] = operation;
   }
   return operations;
 }
@@ -727,12 +794,15 @@ export async function saveOperation(
   operation: Operation
 ): Promise<void> {
   const id = `${deviceId}:${commandKey}`;
-  const o = Object.assign({}, operation) as Record<string, any>;
+  const o: MongoTypes.Operation = {
-  o["_id"] = id;
+    _id: id,
-  o["timestamp"] = new Date(operation.timestamp);
+    name: operation.name,
-  o["provisions"] = JSON.stringify(operation.provisions);
+    timestamp: new Date(operation.timestamp),
-  o["retries"] = JSON.stringify(operation.retries);
+    channels: JSON.stringify(operation.channels),
-  o["args"] = JSON.stringify(operation.args);
+    provisions: JSON.stringify(operation.provisions),
+    retries: JSON.stringify(operation.retries),
+    args: JSON.stringify(operation.args),
+  };
   await operationsCollection.replaceOne({ _id: id }, o, {
     upsert: true,
   });
@@ -784,7 +854,7 @@ interface Permission {
 }
 
 export async function getPermissions(): Promise<Permission[]> {
-  return permissionsCollection.find().toArray();
+  return permissionsCollection.find().toArray() as unknown as Permission[];
 }
 
 interface User {
@@ -795,5 +865,5 @@ interface User {
 }
 
 export async function getUsers(): Promise<User[]> {
-  return usersCollection.find().toArray();
+  return usersCollection.find().toArray() as unknown as User[];
 }

lib/debug.ts

@@ -17,16 +17,12 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import {
+import { IncomingMessage, ServerResponse, ClientRequest } from "http";
-  IncomingMessage,
-  ServerResponse,
-  ClientRequest,
-  RequestOptions,
-} from "http";
 import { Socket } from "net";
 import { appendFileSync } from "fs";
 import { stringify } from "./common/yaml";
 import * as config from "./config";
+import { getSocketEndpoints } from "./server";
 
 const DEBUG_FILE = "" + config.get("DEBUG_FILE");
 const DEBUG_FORMAT = "" + config.get("DEBUG_FORMAT");
@@ -50,13 +46,14 @@ export function incomingHttpRequest(
   if (!DEBUG_FILE) return;
   const now = new Date();
   const con = httpRequest.socket;
+  const socketEndpoints = getSocketEndpoints(con);
   const msg = {
     event: "incoming HTTP request",
     timestamp: now,
-    remoteAddress: con.remoteAddress,
+    remoteAddress: socketEndpoints.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
-    localPort: con.localPort,
+    localPort: socketEndpoints.localPort,
     method: httpRequest.method,
     url: httpRequest.url,
     headers: httpRequest.headers,
@@ -78,10 +75,11 @@ export function outgoingHttpResponse(
   if (!DEBUG_FILE) return;
   const now = new Date();
   const con = httpResponse.socket;
+  const socketEndpoints = getSocketEndpoints(con);
   const msg = {
     event: "outgoing HTTP response",
     timestamp: now,
-    remoteAddress: con.remoteAddress,
+    remoteAddress: socketEndpoints.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
     statusCode: httpResponse.statusCode,
@@ -99,7 +97,8 @@ export function outgoingHttpResponse(
 export function outgoingHttpRequest(
   httpRequest: ClientRequest,
   deviceId: string,
-  options: RequestOptions,
+  method: "GET" | "PUT" | "POST" | "DELETE",
+  url: URL,
   body: string
 ): void {
   if (!DEBUG_FILE) return;
@@ -111,9 +110,9 @@ export function outgoingHttpRequest(
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
-    remotePort: options.port,
+    remotePort: url.port,
-    method: options.method || "GET",
+    method: method,
-    url: options.path,
+    url: url.pathname + url.search,
     headers: httpRequest.getHeaders(),
     body: body,
   };
@@ -128,7 +127,8 @@ export function outgoingHttpRequest(
 export function outgoingHttpRequestError(
   httpRequest: ClientRequest,
   deviceId: string,
-  options: RequestOptions,
+  method: "GET" | "PUT" | "POST" | "DELETE",
+  url: URL,
   err: Error
 ): void {
   if (!DEBUG_FILE) return;
@@ -136,12 +136,12 @@ export function outgoingHttpRequestError(
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
-    remoteAddress: options.hostname,
+    remoteAddress: url.hostname,
     deviceId: deviceId,
     connection: null,
-    remotePort: options.port,
+    remotePort: url.port,
-    method: options.method,
+    method: method,
-    url: options.path,
+    url: url.pathname + url.search,
     headers: httpRequest.getHeaders(),
     error: err.message,
   };
@@ -202,3 +202,20 @@ export function outgoingUdpMessage(
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
 }
+
+export function clientError(remoteAddress: string, err: Error): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "client error",
+    timestamp: now,
+    remoteAddress: remoteAddress,
+    error: err.message,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}

lib/device.ts

@@ -54,13 +54,6 @@ export function sanitizeParameterValue(
           else parameterValue[0] = b;
         }
         break;
-      case "xsd:string":
-      case "xsd:base64":
-      case "xsd:hexBinary":
-        if (typeof parameterValue[0] !== "string")
-          parameterValue[0] = "" + parameterValue[0];
-
-        break;
       case "xsd:int":
       case "xsd:unsignedInt":
         if (typeof parameterValue[0] !== "number") {
@@ -83,7 +76,7 @@ export function sanitizeParameterValue(
         }
         break;
       default:
-        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
+        parameterValue[0] = "" + parameterValue[0];
         break;
     }
   }
@@ -208,7 +201,7 @@ export function clear(
   timestamp = timestamp || 0;
 
   let descendantsTimestamp = timestamp;
-  if (attributes && attributes.object) {
+  if (attributes?.object) {
     if (attributes.object > descendantsTimestamp)
       descendantsTimestamp = attributes.object;
     if (!(attributes.object <= attributes.value))

lib/extensions.ts

@@ -30,14 +30,6 @@ const TIMEOUT = +config.get("EXT_TIMEOUT");
 const processes: { [script: string]: ChildProcess } = {};
 const jobs = new Map();
 
-function messageHandler(message): void {
-  const func = jobs.get(message[0]);
-  if (func) {
-    jobs.delete(message[0]);
-    func({ fault: message[1], value: message[2] });
-  }
-}
-
 export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
   return new Promise((resolve) => {
     const scriptName = args[0];
@@ -70,7 +62,16 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
         if (processes[scriptName] === p) delete processes[scriptName];
       });
 
-      p.on("message", messageHandler);
+      p.on("message", (message) => {
+        const func = jobs.get(message[0]);
+        if (func) {
+          jobs.delete(message[0]);
+          // Wait for any disconnect even to fire
+          setTimeout(() => {
+            func({ fault: message[1], value: message[2] });
+          });
+        }
+      });
 
       const rlstdout = readline.createInterface(p.stdout);
       rlstdout.on("line", (line) => {

lib/forwarded.ts

@@ -21,6 +21,7 @@ import { IncomingMessage } from "http";
 import { TLSSocket } from "tls";
 import { parseCIDR, parse, IPv6, IPv4 } from "ipaddr.js";
 import * as config from "./config";
+import { getSocketEndpoints } from "./server";
 
 interface RequestOrigin {
   localAddress: string;
@@ -98,19 +99,22 @@ export function getRequestOrigin(request: IncomingMessage): RequestOrigin {
   let origin = cache.get(request);
   if (!origin) {
     const soc = request.socket;
+    const socketEndpoints = getSocketEndpoints(soc);
     origin = {
-      localAddress: soc.localAddress,
+      localAddress: socketEndpoints.localAddress,
-      localPort: soc.localPort,
+      localPort: socketEndpoints.localPort,
-      remoteAddress: soc.remoteAddress,
+      remoteAddress: socketEndpoints.remoteAddress,
-      remotePort: soc.remotePort,
+      remotePort: socketEndpoints.remotePort,
       host: request.headers["host"],
       encrypted: !!(request.socket as TLSSocket).encrypted,
     };
 
     const header = request.headers["forwarded"];
     if (header) {
-      const ip = parse(soc.remoteAddress) as IPv4;
+      const ip = parse(socketEndpoints.remoteAddress);
-      if (cidrs.some((cidr) => ip.match(cidr as [IPv4, number]))) {
+      if (
+        cidrs.some((cidr) => ip.kind() === cidr[0].kind() && ip.match(cidr))
+      ) {
         const parsed = parseForwardedHeader(header);
 
         if (parsed["proto"] === "https") {

lib/fs.ts

@@ -18,7 +18,6 @@
  */
 
 import * as url from "url";
-import * as querystring from "querystring";
 import { IncomingMessage, ServerResponse } from "http";
 import { Collection, GridFSBucket } from "mongodb";
 import { onConnect } from "./db";
@@ -35,7 +34,7 @@ onConnect(async (db) => {
 });
 
 const getFile = memoize(
-  (md5: string, size: number, filename: string): Promise<Buffer> => {
+  (uploadDate: number, size: number, filename: string): Promise<Buffer> => {
     return new Promise((resolve, reject) => {
       const buffer = Buffer.allocUnsafe(size);
       let i = 0;
@@ -64,7 +63,7 @@ export async function listener(
   }
 
   const urlParts = url.parse(request.url, true);
-  const filename = querystring.unescape(urlParts.pathname.substring(1));
+  const filename = decodeURIComponent(urlParts.pathname.substring(1));
 
   const log = {
     message: "Fetch file",
@@ -82,10 +81,14 @@ export async function listener(
     return;
   }
 
-  const buffer = await getFile(file.md5, file.length, filename);
+  const buffer = await getFile(
+    file["uploadDate"].getTime(),
+    file.length,
+    filename
+  );
 
   response.writeHead(200, {
-    "Content-Type": file.contentType || "application/octet-stream",
+    "Content-Type": "application/octet-stream",
     "Content-Length": file.length,
   });
 

lib/init.ts

@@ -59,7 +59,7 @@ const INFORM_SCRIPT = `
 // Device ID as user name
 const username = declare("DeviceID.ID", {value: 1}).value[0]
 
-// Password will be fixed a given device because Math.random() is seeded with devcie ID by default.
+// Password will be fixed for a given device because Math.random() is seeded with device ID by default.
 const password = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER).toString(36);
 
 const informInterval = 300;
@@ -257,7 +257,7 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
           "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
       },
       { _id: "ui.device.3.type", value: "'parameter-table'" },
-      { _id: "ui.device.3.label", value: "'WLAN hosts'" },
+      { _id: "ui.device.3.label", value: "'LAN hosts'" },
       {
         _id: "ui.device.3.parameter",
         value: "InternetGatewayDevice.LANDevice.1.Hosts.Host",

lib/instance-set.ts

@@ -22,7 +22,7 @@ interface Instance {
 }
 
 export default class InstanceSet {
-  private set: Set<Instance>;
+  private declare set: Set<Instance>;
 
   public constructor() {
     this.set = new Set();

lib/local-cache.ts

@@ -131,7 +131,7 @@ async function fetchPresets(): Promise<Preset[]> {
 
     // If no keys are defined, consider all parameters as keys to keep the
     // same behavior from v1.0
-    if (!obj["_keys"] || !obj["_keys"].length)
+    if (!obj["_keys"]?.length)
       obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
 
     return obj;
@@ -266,9 +266,7 @@ async function fetchProvisions(): Promise<Provisions> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    provisions[
+    provisions[r["_id"]].script = new vm.Script(
-      r["_id"]
-    ].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -287,9 +285,7 @@ async function fetchVirtualParameters(): Promise<VirtualParameters> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    virtualParameters[
+    virtualParameters[r["_id"]].script = new vm.Script(
-      r["_id"]
-    ].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -325,8 +321,6 @@ async function fetchFiles(): Promise<Files> {
     const id = r["filename"] || r["_id"].toString();
     files[id] = {};
     files[id].length = r["length"];
-    files[id].md5 = r["md5"];
-    files[id].contentType = r["contentType"];
   }
 
   return files;
@@ -357,6 +351,7 @@ async function fetchConfig(): Promise<[Config, UiConfig]> {
     device: {},
     index: {},
     overview: {},
+    pageSize: null,
   };
   const _config = {};
 
@@ -366,12 +361,13 @@ async function fetchConfig(): Promise<[Config, UiConfig]> {
     _config[c.id] = val;
     if (c.id.startsWith("ui.")) {
       const keys = c.id.split(".");
+      if (!(keys[1] in ui)) continue;
       // remove the first key(ui)
       keys.shift();
       let ref = ui;
       while (keys.length > 1) {
         const k = keys.shift();
-        if (typeof ref[k] !== "object") ref[k] = {};
+        if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
         ref = ref[k];
       }
       ref[keys[0]] = val;
@@ -384,13 +380,15 @@ async function fetchConfig(): Promise<[Config, UiConfig]> {
 async function refresh(): Promise<void> {
   if (!nextRefresh) {
     await new Promise((resolve) => setTimeout(resolve, 20));
-    await refresh();
+    return refresh();
-    return;
   }
 
-  nextRefresh = 0;
   const now = Date.now();
 
+  if (now < nextRefresh) return;
+
+  nextRefresh = 0;
+
   const dbHash = await cache.get("presets_hash");
 
   if (currentSnapshot && dbHash === currentSnapshot) {
@@ -398,7 +396,7 @@ async function refresh(): Promise<void> {
     return;
   }
 
-  const lockToken = await cache.acquireLock("presets_hash_lock", 5000, 5000);
+  const lockToken = await cache.acquireLock("presets_hash_lock", 5000);
 
   const res = await Promise.all([
     fetchPresets(),
@@ -431,8 +429,12 @@ async function refresh(): Promise<void> {
 
   currentSnapshot = computeHash(snapshot);
   snapshots.set(currentSnapshot, snapshot);
-  await cache.set("presets_hash", currentSnapshot, 300);
+
-  await cache.releaseLock("presets_hash_lock", lockToken);
+  if (lockToken) {
+    if (currentSnapshot !== dbHash)
+      await cache.set("presets_hash", currentSnapshot, 300);
+    await cache.releaseLock("presets_hash_lock", lockToken);
+  }
 
   nextRefresh = now + (REFRESH - (now % REFRESH));
 }
@@ -501,7 +503,7 @@ export function getConfig(
   if (!(key in snapshot.config)) {
     if (key in oldOpts) {
       let id;
-      if (context && context["id"]) {
+      if (context?.["id"]) {
         id = context["id"];
       } else if (cb) {
         id = cb(["PARAM", "DeviceID.ID"]);

lib/logger.ts

@@ -292,11 +292,9 @@ function formatSimple(
     return `${severity}${remote}${details.message}${meta}${os.EOL}`;
   }
 
-  return `${
+  return `${details.timestamp} [${(
-    details.timestamp
+    details.severity as string
-  } [${(details.severity as string).toUpperCase()}] ${remote}${
+  ).toUpperCase()}] ${remote}${details.message}${meta}${os.EOL}`;
-    details.message
-  }${meta}${os.EOL}`;
 }
 
 function log(details: Record<string, unknown>): void {

lib/mongodb-functions.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ObjectID } from "mongodb";
+import { ObjectId } from "mongodb";
 import { map, parse, stringify, parseList } from "./common/expression-parser";
 import { likePatternToRegExp } from "./common/expression";
 import { Expression, Fault, Task } from "./types";
@@ -77,7 +77,8 @@ export function processDeviceFilter(filter: Expression): Expression {
       if (typeof e[2] === "number") {
         const alt = (e as any[]).slice();
         alt[2] = new Date(e[2]);
-        e = ["OR", e, alt];
+        if (exp[0] === "<>" || exp[0] === "NOT LIKE") e = ["AND", e, alt];
+        else e = ["OR", e, alt];
       }
       return e;
     }
@@ -90,7 +91,7 @@ export function processTasksFilter(filter: Expression): Expression {
     if (!isArray(exp)) return exp;
     if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
       const e = exp.slice();
-      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectID(e[2]);
+      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectId(e[2]);
       else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
         e[2] = new Date(e[2]);
       else if (e[1][0] === "PARAM" && e[1][1] === "expiry")

lib/mongodb-types.ts

@@ -0,0 +1,107 @@
+import { ObjectId } from "mongodb";
+import { Expression, FaultStruct } from "./types";
+
+export interface Fault {
+  _id: string;
+  device: string;
+  channel: string;
+  timestamp: Date;
+  provisions: string;
+  retries: number;
+  code: string;
+  message: string;
+  detail?:
+    | FaultStruct
+    | {
+        name: string;
+        message: string;
+        stack?: string;
+      };
+  expiry?: Date;
+}
+
+interface TaskBase {
+  _id: ObjectId;
+  timestamp?: Date;
+  expiry?: Date;
+  name: string;
+}
+
+interface TaskGetParameterValues extends TaskBase {
+  name: "getParameterValues";
+  parameterNames: string[];
+}
+
+interface TaskSetParameterValues extends TaskBase {
+  name: "setParameterValues";
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskRefreshObject extends TaskBase {
+  name: "refreshObject";
+  objectName: string;
+}
+
+interface TaskReboot extends TaskBase {
+  name: "reboot";
+}
+
+interface TaskFactoryReset extends TaskBase {
+  name: "factoryReset";
+}
+
+interface TaskDownload extends TaskBase {
+  name: "download";
+  fileType: string;
+  fileName: string;
+  targetFileName?: string;
+}
+
+interface TaskAddObject extends TaskBase {
+  name: "addObject";
+  objectName: string;
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskDeleteObject extends TaskBase {
+  name: "deleteObject";
+  objectName: string;
+}
+
+interface TaskProvisions extends TaskBase {
+  name: "provisions";
+  provisions?: [string, ...Expression[]][];
+}
+
+export type Task =
+  | TaskGetParameterValues
+  | TaskSetParameterValues
+  | TaskRefreshObject
+  | TaskReboot
+  | TaskFactoryReset
+  | TaskDownload
+  | TaskAddObject
+  | TaskDeleteObject
+  | TaskProvisions;
+
+export interface Operation {
+  _id: string;
+  name: string;
+  timestamp: Date;
+  channels: string;
+  retries: string;
+  provisions: string;
+  args: string;
+}
+
+export interface Config {
+  _id: string;
+  value: string;
+}
+
+export interface Cache {
+  _id: string;
+  value: string;
+  timestamp: Date;
+  expire: Date;
+}

lib/nbi.ts

@@ -19,10 +19,9 @@
 
 import * as url from "url";
 import { Collection, GridFSBucket, ObjectId } from "mongodb";
-import * as querystring from "querystring";
 import * as vm from "vm";
 import * as config from "./config";
-import { onConnect } from "./db";
+import { onConnect, optimizeProjection } from "./db";
 import * as query from "./query";
 import * as apiFunctions from "./api-functions";
 import { IncomingMessage, ServerResponse } from "http";
@@ -30,10 +29,13 @@ import * as cache from "./cache";
 import { version as VERSION } from "../package.json";
 import { ping } from "./ping";
 import * as logger from "./logger";
+import { flattenDevice } from "./mongodb-functions";
+import { getSocketEndpoints } from "./server";
 
 const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
 const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
-const TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tags\/([a-zA-Z0-9\-_%]+)\/?$/;
+const TAGS_REGEX =
+  /^\/devices\/([a-zA-Z0-9\-_%]+)\/tags\/([a-zA-Z0-9\-_%]+)\/?$/;
 const PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-_%]+)\/?$/;
 const OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FILES_REGEX = /^\/files\/([a-zA-Z0-9%!*'();:@&=+$,?#[\]\-_.~]+)\/?$/;
@@ -41,10 +43,11 @@ const PING_REGEX = /^\/ping\/([a-zA-Z0-9\-_.:]+)\/?$/;
 const QUERY_REGEX = /^\/([a-zA-Z0-9_]+)\/?$/;
 const DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/?$/;
 const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
-const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
+const VIRTUAL_PARAMETERS_REGEX =
+  /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
 
-const collections = {
+const collections: Record<string, Collection> = {
   tasks: null as Collection,
   devices: null as Collection,
   presets: null as Collection,
@@ -99,20 +102,28 @@ export function listener(
   request.addListener("end", () => {
     const body = getBody();
     const urlParts = url.parse(request.url, true);
+    const socketEndpoints = getSocketEndpoints(request.socket);
 
     logger.accessInfo(
       Object.assign({}, urlParts.query, {
-        remoteAddress: request.connection.remoteAddress,
+        remoteAddress: socketEndpoints.remoteAddress,
         message: `${request.method} ${urlParts.pathname}`,
       })
     );
 
     if (PRESETS_REGEX.test(urlParts.pathname)) {
-      const presetName = querystring.unescape(
+      const presetName = decodeURIComponent(
         PRESETS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
-        const preset = JSON.parse(body.toString());
+        let preset;
+        try {
+          preset = JSON.parse(body.toString());
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
         preset._id = presetName;
         collections.presets.replaceOne(
           { _id: presetName },
@@ -155,11 +166,18 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (OBJECTS_REGEX.test(urlParts.pathname)) {
-      const objectName = querystring.unescape(
+      const objectName = decodeURIComponent(
         OBJECTS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
-        const object = JSON.parse(body.toString());
+        let object;
+        try {
+          object = JSON.parse(body.toString());
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
         object._id = objectName;
         collections.objects.replaceOne(
           { _id: objectName },
@@ -202,7 +220,7 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (PROVISIONS_REGEX.test(urlParts.pathname)) {
-      const provisionName = querystring.unescape(
+      const provisionName = decodeURIComponent(
         PROVISIONS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
@@ -260,7 +278,7 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (VIRTUAL_PARAMETERS_REGEX.test(urlParts.pathname)) {
-      const virtualParameterName = querystring.unescape(
+      const virtualParameterName = decodeURIComponent(
         VIRTUAL_PARAMETERS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
@@ -322,8 +340,8 @@ export function listener(
       }
     } else if (TAGS_REGEX.test(urlParts.pathname)) {
       const r = TAGS_REGEX.exec(urlParts.pathname);
-      const deviceId = querystring.unescape(r[1]);
+      const deviceId = decodeURIComponent(r[1]);
-      const tag = querystring.unescape(r[2]);
+      const tag = decodeURIComponent(r[2]);
       if (request.method === "POST") {
         collections.devices.updateOne(
           { _id: deviceId },
@@ -351,7 +369,7 @@ export function listener(
       }
     } else if (FAULTS_REGEX.test(urlParts.pathname)) {
       if (request.method === "DELETE") {
-        const faultId = querystring.unescape(
+        const faultId = decodeURIComponent(
           FAULTS_REGEX.exec(urlParts.pathname)[1]
         );
         const deviceId = faultId.split(":", 1)[0];
@@ -396,15 +414,31 @@ export function listener(
       }
     } else if (DEVICE_TASKS_REGEX.test(urlParts.pathname)) {
       if (request.method === "POST") {
-        const deviceId = querystring.unescape(
+        const deviceId = decodeURIComponent(
           DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1]
         );
         if (body.length) {
-          const task = JSON.parse(body.toString());
+          let task;
+          try {
+            task = JSON.parse(body.toString());
+          } catch (err) {
+            response.writeHead(400);
+            response.end(`${err.name}: ${err.message}`);
+            return;
+          }
           task.device = deviceId;
-          apiFunctions
+          collections.devices
-            .insertTasks(task)
+            .findOne({ _id: deviceId })
-            .then(async () => {
+            .then(async (dev) => {
+              if (!dev) {
+                response.writeHead(404);
+                response.end("No such device");
+                return;
+              }
+
+              const device = flattenDevice(dev);
+              await apiFunctions.insertTasks(task);
+
               const lastInform = Date.now();
 
               const socketTimeout: number = request.socket["timeout"];
@@ -435,7 +469,10 @@ export function listener(
                 return;
               }
 
-              const status = await apiFunctions.connectionRequest(deviceId);
+              const status = await apiFunctions.connectionRequest(
+                deviceId,
+                device
+              );
 
               if (status) {
                 if (socketTimeout) request.socket.setTimeout(socketTimeout);
@@ -518,15 +555,28 @@ export function listener(
             });
         } else if (urlParts.query.connection_request != null) {
           // No task, send connection request only
-          apiFunctions
+          collections.devices
-            .connectionRequest(deviceId)
+            .findOne({ _id: deviceId })
-            .then(() => {
+            .then(async (dev) => {
-              response.writeHead(200);
+              if (!dev) {
-              response.end();
+                response.writeHead(404);
+                response.end("No such device");
+                return;
+              }
+              return apiFunctions.connectionRequest(deviceId).then((status) => {
+                if (status) {
+                  response.writeHead(504, status);
+                  response.end(status);
+                  return;
+                }
+                response.writeHead(200);
+                response.end();
+              });
             })
             .catch((err) => {
-              response.writeHead(504);
+              setTimeout(() => {
-              response.end(`${err.name}: ${err.message}`);
+                throwError(err, response);
+              });
             });
         } else {
           response.writeHead(400);
@@ -538,7 +588,7 @@ export function listener(
       }
     } else if (TASKS_REGEX.test(urlParts.pathname)) {
       const r = TASKS_REGEX.exec(urlParts.pathname);
-      const taskId = querystring.unescape(r[1]);
+      const taskId = decodeURIComponent(r[1]);
       const action = r[2];
       if (!action || action === "/") {
         if (request.method === "DELETE") {
@@ -624,7 +674,7 @@ export function listener(
         response.end();
       }
     } else if (FILES_REGEX.test(urlParts.pathname)) {
-      const filename = querystring.unescape(
+      const filename = decodeURIComponent(
         FILES_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
@@ -634,9 +684,9 @@ export function listener(
           productClass: request.headers.productclass,
           version: request.headers.version,
         };
-        filesBucket.delete((filename as unknown) as ObjectId, () => {
+        filesBucket.delete(filename as unknown as ObjectId, () => {
           const uploadStream = filesBucket.openUploadStreamWithId(
-            filename,
+            filename as unknown as ObjectId,
             filename,
             {
               metadata: metadata,
@@ -653,7 +703,7 @@ export function listener(
           });
         });
       } else if (request.method === "DELETE") {
-        filesBucket.delete((filename as unknown) as ObjectId, (err) => {
+        filesBucket.delete(filename as unknown as ObjectId, (err) => {
           if (err) {
             if (err.message.startsWith("FileNotFound")) {
               response.writeHead(404);
@@ -671,7 +721,7 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (PING_REGEX.test(urlParts.pathname)) {
-      const host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1]);
+      const host = decodeURIComponent(PING_REGEX.exec(urlParts.pathname)[1]);
       ping(host, (err, res, stdout) => {
         if (err) {
           if (!res) {
@@ -697,7 +747,7 @@ export function listener(
         return;
       }
 
-      const deviceId = querystring.unescape(
+      const deviceId = decodeURIComponent(
         DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1]
       );
       apiFunctions
@@ -771,12 +821,20 @@ export function listener(
         projection = {};
         for (const p of (urlParts.query.projection as string).split(","))
           projection[p.trim()] = 1;
+        projection = optimizeProjection(projection);
       }
 
       const cur = collection.find(q, { projection: projection });
 
       if (urlParts.query.sort) {
-        const s = JSON.parse(urlParts.query.sort as string);
+        let s;
+        try {
+          s = JSON.parse(urlParts.query.sort as string);
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
         const sort = {};
         for (const [k, v] of Object.entries(s)) {
           if (k[k.lastIndexOf(".") + 1] !== "_" && collectionName === "devices")
@@ -787,13 +845,7 @@ export function listener(
         cur.sort(sort);
       }
 
-      if (urlParts.query.skip)
+      collection.countDocuments(q, (err, total) => {
-        cur.skip(parseInt(urlParts.query.skip as string));
-
-      if (urlParts.query.limit)
-        cur.limit(parseInt(urlParts.query.limit as string));
-
-      cur.count(false, (err, total) => {
         if (err) return void throwError(err);
 
         response.writeHead(200, {
@@ -806,6 +858,12 @@ export function listener(
           return;
         }
 
+        if (urlParts.query.skip)
+          cur.skip(parseInt(urlParts.query.skip as string));
+
+        if (urlParts.query.limit)
+          cur.limit(parseInt(urlParts.query.limit as string));
+
         response.write("[\n");
         i = 0;
         cur.forEach(

lib/ping.ts

@@ -19,6 +19,7 @@
 
 import { platform } from "os";
 import { exec } from "child_process";
+import { domainToASCII } from "url";
 
 export interface PingResult {
   packetsTransmitted: number;
@@ -30,61 +31,87 @@ export interface PingResult {
   mdev: number;
 }
 
+function isValidHost(host: string): boolean {
+  // Valid chars in IPv4, IPv6, domain names
+  if (/^[a-zA-Z0-9\-.:[\]-]+$/.test(host)) return true;
+
+  // Check if input is an IDN convert to Punycode
+  // Can't merge with above because domainToASCII doesn't accept IP addresses
+  return /^[a-zA-Z0-9\-.:[\]-]+$/.test(domainToASCII(host));
+}
+
+export function parsePing(osPlatform: string, stdout: string): PingResult {
+  let parseRegExp1: RegExp, parseRegExp2: RegExp, parsed: PingResult;
+  switch (osPlatform) {
+    case "linux":
+      parseRegExp1 =
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss[^]*= ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+      parseRegExp2 =
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss/;
+      break;
+
+    case "freebsd":
+      parseRegExp1 =
+        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
+      parseRegExp2 =
+        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
+      break;
+  }
+
+  const m1 = stdout.match(parseRegExp1);
+  if (m1) {
+    parsed = {
+      packetsTransmitted: +m1[1],
+      packetsReceived: +m1[2],
+      packetLoss: +m1[3],
+      min: +m1[4],
+      avg: +m1[5],
+      max: +m1[6],
+      mdev: +m1[7],
+    };
+  } else {
+    const m2 = stdout.match(parseRegExp2);
+    if (m2) {
+      parsed = {
+        packetsTransmitted: +m2[1],
+        packetsReceived: +m2[2],
+        packetLoss: +m2[3],
+        min: null,
+        avg: null,
+        max: null,
+        mdev: null,
+      };
+    }
+  }
+  return parsed;
+}
+
 export function ping(
   host: string,
   callback: (err: Error, res?: PingResult, stdout?: string) => void
 ): void {
-  let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
+  // Validate input to prevent possible remote code execution
+  // Credit to Alex Hordijk for reporting this vulnerability
+  if (!isValidHost(host)) return callback(new Error("Invalid host"));
   host = host.replace("[", "").replace("]", "");
+  let cmd: string;
+
   switch (platform()) {
     case "linux":
       cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
-      parseRegExp1 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
-      parseRegExp2 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
       break;
-
     case "freebsd":
       // Send a single packet because on FreeBSD only superuser can send
       // packets that are only 200 ms apart.
       cmd = `ping -t 1 -c 3 ${host}`;
-      parseRegExp1 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
-      parseRegExp2 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
       break;
-
     default:
       return callback(new Error("Platform not supported"));
   }
 
   exec(cmd, (err, stdout) => {
-    let parsed: PingResult;
+    if (err) return callback(err);
-    if (stdout) {
+    const parsed: PingResult = parsePing(platform(), stdout);
-      const m1 = stdout.match(parseRegExp1);
+    return callback(err, parsed, stdout);
-      if (m1) {
-        parsed = {
-          packetsTransmitted: +m1[1],
-          packetsReceived: +m1[2],
-          packetLoss: +m1[3],
-          min: +m1[4],
-          avg: +m1[5],
-          max: +m1[6],
-          mdev: +m1[7],
-        };
-      } else {
-        const m2 = stdout.match(parseRegExp2);
-        if (m2) {
-          parsed = {
-            packetsTransmitted: +m2[1],
-            packetsReceived: +m2[2],
-            packetLoss: +m2[3],
-            min: null,
-            avg: null,
-            max: null,
-            mdev: null,
-          };
-        }
-      }
-    }
-
-    callback(err, parsed, stdout);
   });
 }

lib/query.ts

@@ -54,6 +54,17 @@ function normalize(input): any {
   return input;
 }
 
+const EXPAND_OPS = new Set([
+  "$eq",
+  "$gt",
+  "$gte",
+  "$in",
+  "$lt",
+  "$lte",
+  "$ne",
+  "$nin",
+]);
+
 function expandValue(value: unknown): unknown[] {
   if (Array.isArray(value)) {
     let a = [];
@@ -71,7 +82,8 @@ function expandValue(value: unknown): unknown[] {
   const values = [];
   for (const [k, v] of Object.entries(value)) {
     keys.push(k);
-    values.push(expandValue(v));
+    if (EXPAND_OPS.has(k)) values.push(expandValue(v));
+    else values.push([v]);
     indices.push(0);
   }
 

lib/sandbox.ts

@@ -255,6 +255,7 @@ function declare(
   }
 
   for (const [attrName, attrTimestamp] of Object.entries(timestamps)) {
+    if (!(attrTimestamp >= 1)) continue;
     if (attrName === "path") {
       declaration.pathGet = attrTimestamp;
     } else {

lib/scheduling.ts

@@ -18,7 +18,7 @@
  */
 
 import * as crypto from "crypto";
-import * as later from "later";
+import * as later from "@breejs/later";
 
 function md532(str): number {
   const digest = crypto.createHash("md5").update(str).digest();

lib/server.ts

@@ -59,8 +59,20 @@ interface ServerOptions {
   timeout?: number;
   keepAliveTimeout?: number;
   onConnection?: (socket: Socket) => void;
+  onClientError?: (err: Error, socket: Socket) => void;
 }
 
+interface SocketEndpoint {
+  localAddress: string;
+  localPort: number;
+  remoteAddress: string;
+  remotePort: number;
+  remoteFamily: "IPv4" | "IPv6";
+}
+
+// Save this info as they're not accessible after a socket has been closed
+const socketEndpoints: WeakMap<Socket, SocketEndpoint> = new WeakMap();
+
 export function start(
   options: ServerOptions,
   _listener: http.RequestListener
@@ -85,6 +97,27 @@ export function start(
     if (options.onConnection) server.on("connection", options.onConnection);
   }
 
+  server.on("connection", (socket: Socket) => {
+    socketEndpoints.set(socket, {
+      localAddress: socket.localAddress,
+      localPort: socket.localPort,
+      remoteAddress: socket.remoteAddress,
+      remotePort: socket.remotePort,
+      remoteFamily: socket.remoteFamily as "IPv4" | "IPv6",
+    });
+  });
+
+  if (options.onClientError) {
+    server.on("clientError", (err, socket: Socket) => {
+      if (err["code"] !== "ECONNRESET" && socket.writable)
+        socket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n");
+
+      // As per Node docs: This event is guaranteed to be passed an instance
+      // of the <net.Socket> class
+      options.onClientError(err, socket as Socket);
+    });
+  }
+
   server.timeout = options.timeout || 0;
   if (options.keepAliveTimeout != null)
     server.keepAliveTimeout = options.keepAliveTimeout;
@@ -99,3 +132,8 @@ export function stop(): Promise<void> {
     closeServer(20000, resolve);
   });
 }
+
+export function getSocketEndpoints(socket: Socket): SocketEndpoint {
+  // TLSSocket keeps a reference to the raw TCP socket in _parent
+  return socketEndpoints.get(socket["_parent"] ?? socket);
+}

lib/session.ts

@@ -59,6 +59,7 @@ import {
   GetParameterValuesResponse,
 } from "./types";
 import { getRequestOrigin } from "./forwarded";
+import * as logger from "./logger";
 
 const VALID_PARAM_TYPES = new Set([
   "xsd:int",
@@ -135,7 +136,7 @@ export function configContextCallback(
     const path = paths.get(Path.parse(name));
     if (path) {
       const attrs = deviceData.attributes.get(path, 1);
-      if (attrs && attrs.value && attrs.value[1]) return attrs.value[1][0];
+      if (attrs?.value?.[1]) return attrs.value[1][0];
     }
   } else if (exp[0] === "FUNC") {
     if (exp[1] === "REMOTE_ADDRESS")
@@ -298,7 +299,7 @@ export async function transferComplete(
   if (!sessionContext.operationsTouched) sessionContext.operationsTouched = {};
   sessionContext.operationsTouched[commandKey] = 1;
 
-  if (rpcReq.faultStruct && rpcReq.faultStruct.faultCode !== "0") {
+  if (rpcReq.faultStruct?.faultCode !== "0") {
     revertDownloadParameters(sessionContext, operation.args.instance);
 
     const fault: Fault = {
@@ -417,13 +418,7 @@ function revertDownloadParameters(
   );
 
   const toClear = device.set(sessionContext.deviceData, p, timestamp, {
-    value: [
+    value: [timestamp, [lastDownload?.value[1]?.[0] || 0, "xsd:dateTime"]],
-      timestamp,
-      [
-        lastDownload && lastDownload.value[1] ? lastDownload.value[1][0] : 0,
-        "xsd:dateTime",
-      ],
-    ],
   });
 
   if (toClear) {
@@ -457,22 +452,50 @@ export async function timeoutOperations(
         (e) => configContextCallback(sessionContext, e)
       ) * 1000;
 
-    if (sessionContext.timestamp > operation.timestamp + DOWNLOAD_TIMEOUT) {
+    if (sessionContext.timestamp < operation.timestamp + DOWNLOAD_TIMEOUT)
-      delete sessionContext.operations[commandKey];
+      continue;
-      if (!sessionContext.operationsTouched)
-        sessionContext.operationsTouched = {};
-      sessionContext.operationsTouched[commandKey] = 1;
 
-      faults.push({
+    logger.accessWarn({
-        code: "timeout",
+      sessionContext: sessionContext,
-        message: "Download operation timed out",
+      message: "Download operation timed out",
-        timestamp: operation.timestamp,
+      commandKey: commandKey,
-      });
+    });
 
-      operations.push(operation);
+    const SUCCESS_ON_TIMEOUT = +localCache.getConfig(
+      sessionContext.cacheSnapshot,
+      "cwmp.downloadSuccessOnTimeout",
+      {},
+      sessionContext.timestamp,
+      (e) => configContextCallback(sessionContext, e)
+    );
 
-      revertDownloadParameters(sessionContext, operation.args.instance);
+    if (SUCCESS_ON_TIMEOUT) {
+      const r = {
+        name: "TransferComplete" as const,
+        commandKey: commandKey,
+        startTime: 0,
+        completeTime: 0,
+      };
+
+      // Call transferComplete code and ignore the response
+      await transferComplete(sessionContext, r);
+      continue;
     }
+
+    delete sessionContext.operations[commandKey];
+    if (!sessionContext.operationsTouched)
+      sessionContext.operationsTouched = {};
+    sessionContext.operationsTouched[commandKey] = 1;
+
+    faults.push({
+      code: "timeout",
+      message: "Download operation timed out",
+      timestamp: operation.timestamp,
+    });
+
+    operations.push(operation);
+
+    revertDownloadParameters(sessionContext, operation.args.instance);
   }
 
   return { faults, operations };
@@ -481,7 +504,7 @@ export async function timeoutOperations(
 export function addProvisions(
   sessionContext: SessionContext,
   channel: string,
-  provisions: (string | number | boolean)[][]
+  provisions: [string, ...Expression[]][]
 ): void {
   // Multiply by two because every iteration is two
   // phases: read and update
@@ -1012,7 +1035,7 @@ export async function rpcRequest(
   if (
     !sessionContext.virtualParameters.length &&
     !sessionContext.declarations.length &&
-    !(_declarations && _declarations.length) &&
+    !_declarations?.length &&
     !sessionContext.provisions.length
   )
     return { fault: null, rpcId: null, rpc: null };
@@ -1087,12 +1110,11 @@ export async function rpcRequest(
     return rpcRequest(sessionContext, _declarations);
   }
 
-  if (_declarations && _declarations.length) {
+  if (_declarations?.length) {
     delete sessionContext.syncState;
     if (!sessionContext.declarations[0]) sessionContext.declarations[0] = [];
-    sessionContext.declarations[0] = sessionContext.declarations[0].concat(
+    sessionContext.declarations[0] =
-      _declarations
+      sessionContext.declarations[0].concat(_declarations);
-    );
     return rpcRequest(sessionContext, null);
   }
 
@@ -1423,7 +1445,7 @@ export async function rpcRequest(
       for (const [p, v] of sessionContext.syncState.downloadsValues) {
         const attrs = sessionContext.deviceData.attributes.get(p);
         if (attrs) {
-          if (attrs.writable && attrs.writable[1] && attrs.value) {
+          if (attrs.writable?.[1] && attrs.value) {
             const val = device.sanitizeParameterValue([v, attrs.value[1][1]]);
             if (val[0] !== attrs.value[1][0]) {
               toClear = device.set(
@@ -1602,6 +1624,20 @@ function generateGetRpcRequest(
       (a, b) => b.length - a.length
     );
     let path = paths.pop();
+
+    // Skip root GPN workaround
+    if (path && !path.length) {
+      const SKIP_ROOT_GPN = !!localCache.getConfig(
+        sessionContext.cacheSnapshot,
+        "cwmp.skipRootGpn",
+        {},
+        sessionContext.timestamp,
+        (e) => configContextCallback(sessionContext, e)
+      );
+
+      if (SKIP_ROOT_GPN) path = paths.pop();
+    }
+
     while (
       path &&
       path.length &&
@@ -1664,7 +1700,7 @@ function generateGetRpcRequest(
       syncState.refreshAttributes.value.delete(path);
       // Need to check in case param is deleted or changed to object
       const attrs = sessionContext.deviceData.attributes.get(path);
-      if (attrs && attrs.object && attrs.object[1] === 0) {
+      if (attrs?.object?.[1] === 0) {
         parameterNames.push(path.toString());
         if (parameterNames.length >= GPV_BATCH_SIZE) break;
       }
@@ -1817,7 +1853,7 @@ function generateSetRpcRequest(
   for (const [k, v] of syncState.spv) {
     syncState.spv.delete(k);
     const attrs = sessionContext.deviceData.attributes.get(k);
-    const curVal = attrs.value ? attrs.value[1] : null;
+    const curVal = attrs.value?.[1];
     if (curVal && canWrite(attrs)) {
       const val = v.slice() as [string | number | boolean, string];
       if (!val[1]) val[1] = curVal[1];
@@ -1881,8 +1917,9 @@ function generateSetRpcRequest(
 
   // Downloads
   for (const [p, t] of syncState.downloadsDownload) {
+    if (!(t > 0 && t <= sessionContext.timestamp)) continue;
     const attrs = deviceData.attributes.get(p);
-    if (!(attrs && attrs.value && t <= attrs.value[1][0])) {
+    if (!(t <= attrs?.value?.[1]?.[0])) {
       const fileTypeAttrs = deviceData.attributes.get(
         deviceData.paths.get(p.slice(0, -1).concat(Path.parse("FileType")))
       );
@@ -1899,42 +1936,31 @@ function generateSetRpcRequest(
         name: "Download",
         commandKey: generateRpcId(sessionContext),
         instance: p.segments[1] as string,
-        fileType: fileTypeAttrs
+        fileType: fileTypeAttrs?.value?.[1][0] as string,
-          ? fileTypeAttrs.value
+        fileName: fileNameAttrs?.value?.[1][0] as string,
-            ? (fileTypeAttrs.value[1][0] as string)
+        targetFileName: targetFileNameAttrs?.value?.[1][0] as string,
-            : null
-          : null,
-        fileName: fileNameAttrs
-          ? fileNameAttrs.value
-            ? (fileNameAttrs.value[1][0] as string)
-            : null
-          : null,
-        targetFileName: targetFileNameAttrs
-          ? targetFileNameAttrs.value
-            ? (targetFileNameAttrs.value[1][0] as string)
-            : null
-          : null,
       };
     }
   }
 
   // Reboot
-  if (syncState.reboot) {
+  if (syncState.reboot > 0 && syncState.reboot <= sessionContext.timestamp) {
     const p = sessionContext.deviceData.paths.get(Path.parse("Reboot"));
     const attrs = p ? sessionContext.deviceData.attributes.get(p) : null;
-    if (!(attrs && attrs.value && attrs.value[1][0] >= syncState.reboot)) {
+    if (!(attrs?.value?.[1][0] >= syncState.reboot)) {
       delete syncState.reboot;
       return { name: "Reboot" };
     }
   }
 
   // Factory reset
-  if (syncState.factoryReset) {
+  if (
+    syncState.factoryReset > 0 &&
+    syncState.factoryReset <= sessionContext.timestamp
+  ) {
     const p = sessionContext.deviceData.paths.get(Path.parse("FactoryReset"));
     const attrs = p ? sessionContext.deviceData.attributes.get(p) : null;
-    if (
+    if (!(attrs?.value?.[1][0] >= syncState.factoryReset)) {
-      !(attrs && attrs.value && attrs.value[1][0] >= syncState.factoryReset)
-    ) {
       delete syncState.factoryReset;
       return { name: "FactoryReset" };
     }
@@ -2002,7 +2028,7 @@ function generateSetVirtualParameterProvisions(
   let provisions;
   if (virtualParameterDeclarations) {
     for (const declaration of virtualParameterDeclarations) {
-      if (declaration[2] && declaration[2].value != null) {
+      if (declaration[2]?.value != null) {
         const attrs = sessionContext.deviceData.attributes.get(declaration[0]);
         if (
           attrs &&
@@ -2131,9 +2157,7 @@ function processDeclarations(
 
     if (currentAttributes) {
       leafParam = currentPath;
-      leafIsObject = currentAttributes.object
+      leafIsObject = currentAttributes.object?.[1];
-        ? currentAttributes.object[1]
-        : null;
       // Possible V8 bug causes null === 0
       if (leafIsObject != null && leafIsObject === 0)
         leafTimestamp = Math.max(leafTimestamp, currentAttributes.object[0]);
@@ -2148,13 +2172,13 @@ function processDeclarations(
     ) {
       case "Reboot":
         if (currentPath.length === 1) {
-          if (declareAttributeValues && declareAttributeValues.value)
+          if (declareAttributeValues?.value)
             syncState.reboot = +new Date(declareAttributeValues.value[0]);
         }
         break;
       case "FactoryReset":
         if (currentPath.length === 1) {
-          if (declareAttributeValues && declareAttributeValues.value)
+          if (declareAttributeValues?.value)
             syncState.factoryReset = +new Date(declareAttributeValues.value[0]);
         }
         break;
@@ -2290,7 +2314,7 @@ function processDeclarations(
                     syncState.refreshAttributes.object.add(currentPath);
                   else if (currentAttributes.object[1] === 0)
                     syncState.refreshAttributes.value.add(currentPath);
-                } else {
+                } else if (attrName in syncState.refreshAttributes) {
                   syncState.refreshAttributes[attrName].add(currentPath);
                 }
               }
@@ -2420,8 +2444,15 @@ export async function rpcResponse(
   id: string,
   _rpcRes: CpeResponse
 ): Promise<Fault> {
+  function invalidResponse(message: string): Fault {
+    return {
+      code: "invalid_response",
+      message: message,
+    };
+  }
+
   if (id !== generateRpcId(sessionContext))
-    throw new Error("Request ID not recognized");
+    return invalidResponse("Request ID not recognized");
 
   ++sessionContext.rpcCount;
 
@@ -2512,47 +2543,99 @@ export async function rpcResponse(
 
   if (rpcRes.name === "GetParameterValuesResponse") {
     if (rpcReq.name !== "GetParameterValues")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
-    if (rpcRes.parameterList.length !== rpcReq.parameterNames.length) {
+    const requested = new Set(rpcReq.parameterNames);
-      return {
-        code: `invalid_response`,
-        message:
-          "The list of parameters returned in the GetParameterValuesResponse message does not match the requested parameters",
-      };
-    }
 
-    for (const p of rpcRes.parameterList) {
+    for (const [path, value, type] of rpcRes.parameterList) {
+      if (!requested.delete(path.toString())) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Unexpected parameter in response",
+          parameter: path.toString(),
+        });
+        continue;
+      }
       toClear = device.set(
         sessionContext.deviceData,
-        p[0],
+        path,
         timestamp,
         {
           object: [timestamp, 0],
-          value: [timestamp, p.slice(1) as [string | number | boolean, string]],
+          value: [timestamp, [value, type]],
         },
         toClear
       );
     }
+
+    if (requested.size) {
+      for (const p of requested) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Missing parameter in response",
+          parameter: p,
+        });
+        toClear = device.set(
+          sessionContext.deviceData,
+          Path.parse(p),
+          timestamp,
+          {
+            object: [timestamp, 0],
+            value: [timestamp, ["", "xsd:string"]],
+          },
+          toClear
+        );
+      }
+    }
   } else if (rpcRes.name === "GetParameterAttributesResponse") {
     if (rpcReq.name !== "GetParameterAttributes")
       throw new Error("Response name does not match request name");
 
-    for (const p of rpcRes.parameterList) {
+    const requested = new Set(rpcReq.parameterNames);
+
+    for (const [path, notification, accessList] of rpcRes.parameterList) {
+      if (!requested.delete(path.toString())) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Unexpected parameter in response",
+          parameter: path.toString(),
+        });
+        continue;
+      }
       toClear = device.set(
         sessionContext.deviceData,
-        p[0],
+        path,
         timestamp,
         {
-          notification: [timestamp, p[1] as number],
+          notification: [timestamp, notification],
-          accessList: [timestamp, p[2] as string[]],
+          accessList: [timestamp, accessList],
         },
         toClear
       );
     }
+
+    if (requested.size) {
+      for (const p of requested) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Missing parameter in response",
+          parameter: p,
+        });
+        toClear = device.set(
+          sessionContext.deviceData,
+          Path.parse(p),
+          timestamp,
+          {
+            notification: [timestamp, 0],
+            accessList: [timestamp, []],
+          },
+          toClear
+        );
+      }
+    }
   } else if (rpcRes.name === "GetParameterNamesResponse") {
     if (rpcReq.name !== "GetParameterNames")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     let root: Path;
     if (rpcReq.parameterPath.endsWith("."))
@@ -2609,6 +2692,17 @@ export async function rpcResponse(
     const wildcardParams: Path[] = [root.concat(wildcardPath)];
 
     for (const [path, object, writable] of rpcRes.parameterList) {
+      if (
+        !path.toString().startsWith(rpcReq.parameterPath) &&
+        !(`${path.toString()}.` === rpcReq.parameterPath && !rpcReq.nextLevel)
+      ) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Unexpected parameter in response",
+          parameter: path.toString(),
+        });
+        continue;
+      }
       if (object && !rpcReq.nextLevel)
         wildcardParams.push(path.concat(wildcardPath));
 
@@ -2635,7 +2729,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "SetParameterValuesResponse") {
     if (rpcReq.name !== "SetParameterValues")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     for (const p of rpcReq.parameterList) {
       toClear = device.set(
@@ -2654,7 +2748,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "SetParameterAttributesResponse") {
     if (rpcReq.name !== "SetParameterAttributes")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     for (const p of rpcReq.parameterList) {
       let attrs;
@@ -2684,7 +2778,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "AddObjectResponse") {
     if (rpcReq.name !== "AddObject")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2695,7 +2789,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "DeleteObjectResponse") {
     if (rpcReq.name !== "DeleteObject")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2706,7 +2800,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "RebootResponse") {
     if (rpcReq.name !== "Reboot")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2717,7 +2811,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "FactoryResetResponse") {
     if (rpcReq.name !== "FactoryReset")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2728,7 +2822,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "DownloadResponse") {
     if (rpcReq.name !== "Download")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2814,7 +2908,7 @@ export async function rpcResponse(
       sessionContext.operationsTouched[rpcReq.commandKey] = 1;
     }
   } else {
-    throw new Error("Response name not recognized");
+    return invalidResponse("Response name not recognized");
   }
 
   if (toClear) {
@@ -2855,13 +2949,9 @@ export async function rpcFault(
         (p) => [Path.parse(p.replace(/\.$/, "")), timestamp] as Clear
       );
     } else if (rpcReq.name === "SetParameterValues") {
-      toClear = (rpcReq.parameterList as [
+      toClear = (
-        string,
+        rpcReq.parameterList as [string, string | number | boolean, string][]
-        string | number | boolean,
+      ).map((p) => [Path.parse(p[0].replace(/\.$/, "")), timestamp] as Clear);
-        string
-      ][]).map(
-        (p) => [Path.parse(p[0].replace(/\.$/, "")), timestamp] as Clear
-      );
     } else if (rpcReq.name === "AddObject") {
       toClear = [[Path.parse(rpcReq.objectName.replace(/\.$/, "")), timestamp]];
     } else if (rpcReq.name === "DeleteObject") {
@@ -2898,10 +2988,10 @@ export async function deserialize(
   const sessionContext = JSON.parse(sessionContextString) as SessionContext;
 
   for (const decs of sessionContext.declarations)
-    for (const d of decs) d.path = Path.parse((d.path as unknown) as string);
+    for (const d of decs) d.path = Path.parse(d.path as unknown as string);
 
   const deviceData = initDeviceData();
-  for (const r of (sessionContext.deviceData as unknown) as any[]) {
+  for (const r of sessionContext.deviceData as unknown as any[]) {
     const path = deviceData.paths.add(Path.parse(r[0]));
 
     if (r[1]) deviceData.trackers.set(path, r[1]);

lib/soap.ts

@@ -109,8 +109,8 @@ function event(xml: Element): string[] {
 
 function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterInfoStruct")
     .map<[Path, boolean, boolean]>((e) => {
+      if (e.localName !== "ParameterInfoStruct") return null;
       let param: string, value: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -134,10 +134,20 @@ function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
         parsed = false;
       }
 
-      if (param && !param.endsWith("."))
+      try {
-        return [Path.parse(param), false, parsed];
+        if (param && !param.endsWith("."))
-      else return [Path.parse(param.slice(0, -1)), true, parsed];
+          return [Path.parse(param), false, parsed];
-    });
+        else return [Path.parse(param.slice(0, -1)), true, parsed];
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 const getValueType = memoize((str: string) => {
@@ -151,8 +161,8 @@ function parameterValueList(
   xml: Element
 ): [Path, string | number | boolean, string][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterValueStruct")
     .map<[Path, string | number | boolean, string]>((e) => {
+      if (e.localName !== "ParameterValueStruct") return null;
       let valueElement: Element, param: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -208,15 +218,24 @@ function parameterValueList(
           parsed = value;
         }
       }
-
+      try {
-      return [Path.parse(param), parsed, valueType];
+        return [Path.parse(param), parsed, valueType];
-    });
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 function parameterAttributeList(xml: Element): [Path, number, string[]][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterAttributeStruct")
     .map<[Path, number, string[]]>((e) => {
+      if (e.localName !== "ParameterAttributeStruct") return null;
       let notificationElement: Element,
         accessListElement: Element,
         param: string;
@@ -248,8 +267,18 @@ function parameterAttributeList(xml: Element): [Path, number, string[]][] {
         .filter((c) => c.localName === "string")
         .map((c) => decodeEntities(c.text));
 
-      return [Path.parse(param), notification, accessList];
+      try {
-    });
+        return [Path.parse(param), notification, accessList];
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 function GetParameterNames(methodRequest): string {
@@ -701,6 +730,18 @@ function RequestDownloadResponse(): string {
   return "<cwmp:RequestDownloadResponse></cwmp:RequestDownloadResponse>";
 }
 
+function AcsFault(f: CpeFault): string {
+  return `<soap-env:Body:Fault><faultcode>${encodeEntities(
+    f.faultCode
+  )}</faultcode><faultstring>${encodeEntities(
+    f.faultString
+  )}</faultstring><detail><cwmp:Fault><FaultCode>${encodeEntities(
+    f.detail.faultCode
+  )}</FaultCode><FaultString>${encodeEntities(
+    f.detail.faultString
+  )}</FaultString></cwmp:Fault></detail></soap-env:Body:Fault>`;
+}
+
 function faultStruct(xml: Element): FaultStruct {
   let faultCode: string,
     faultString: string,
@@ -798,18 +839,18 @@ function fault(xml: Element): CpeFault {
 
 export function request(
   body: string,
-  cwmpVersion: string,
   warn: Record<string, unknown>[]
 ): SoapMessage {
   warnings = warn;
 
   const rpc = {
     id: null,
-    cwmpVersion: cwmpVersion,
+    cwmpVersion: null,
     sessionTimeout: null,
     cpeRequest: null,
     cpeFault: null,
     cpeResponse: null,
+    unknownMethod: null,
   };
 
   if (!body.length) return rpc;
@@ -848,7 +889,7 @@ export function request(
 
   const methodElement = bodyElement.children[0];
 
-  if (!rpc.cwmpVersion && methodElement.localName !== "Fault") {
+  if (methodElement.localName === "Inform") {
     let namespace, namespaceHref;
     for (const e of [methodElement, bodyElement, envelope]) {
       namespace = namespace || e.namespace;
@@ -931,7 +972,8 @@ export function request(
       rpc.cpeFault = fault(methodElement);
       break;
     default:
-      throw new Error(`8000 Method not supported ${methodElement.localName}`);
+      rpc.unknownMethod = methodElement.localName;
+      break;
   }
 
   return rpc;
@@ -959,6 +1001,7 @@ export function response(rpc: {
   id: string;
   acsRequest?: AcsRequest;
   acsResponse?: AcsResponse;
+  acsFault?: CpeFault;
   cwmpVersion?: string;
 }): { code: number; headers: Record<string, string>; data: string } {
   const headers = {
@@ -1027,6 +1070,8 @@ export function response(rpc: {
           `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`
         );
     }
+  } else if (rpc.acsFault) {
+    body = AcsFault(rpc.acsFault);
   }
 
   headers["Content-Type"] = 'text/xml; charset="utf-8"';

lib/types.ts

@@ -180,7 +180,7 @@ export interface Task {
   fileName?: string;
   targetFileName?: string;
   expiry?: number;
-  provisions?: (string | number | boolean)[][];
+  provisions?: [string, ...Expression[]][];
 }
 
 export interface Operation {
@@ -477,7 +477,7 @@ export interface VirtualParameters {
 }
 
 export interface Files {
-  [name: string]: { length: number; md5: string; contentType: string };
+  [name: string]: { length: number };
 }
 
 export interface Users {
@@ -526,6 +526,7 @@ export interface SoapMessage {
   cpeRequest?: CpeRequest;
   cpeFault?: CpeFault;
   cpeResponse?: CpeResponse;
+  unknownMethod?: string;
 }
 
 export interface ScriptResult {

lib/ui.ts

@@ -59,8 +59,10 @@ const getAuthorizer = memoize(
   }
 );
 
-koa.on("error", async (err) => {
+koa.on("error", async (err, ctx) => {
   setTimeout(() => {
+    // Ignored errors resulting from aborted requests
+    if (ctx?.req.aborted) return;
     throw err;
   });
 });
@@ -93,7 +95,7 @@ koa.use(
 koa.use(async (ctx, next) => {
   let roles: string[] = [];
 
-  if (ctx.state.user && ctx.state.user.username) {
+  if (ctx.state.user?.username) {
     let user;
     if (ctx.state.user.authMethod === "local") {
       user = localCache.getUsers(ctx.state.configSnapshot)[
@@ -219,13 +221,14 @@ router.post("/init", async (ctx) => {
 });
 
 router.get("/", async (ctx) => {
-  const permissionSets: PermissionSet[] = ctx.state.authorizer.getPermissionSets();
+  const permissionSets: PermissionSet[] =
+    ctx.state.authorizer.getPermissionSets();
 
   let wizard = "";
   if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
     wizard = '<script>window.location.hash = "#!/wizard";</script>';
 
-  ctx.body = `
+  ctx.body = `<!DOCTYPE html>
   <html>
     <head>
       <title>GenieACS</title>
@@ -260,6 +263,7 @@ koa.use(
       flush: constants.Z_SYNC_FLUSH,
     },
     br: {
+      flush: constants.BROTLI_OPERATION_FLUSH,
       params: {
         [constants.BROTLI_PARAM_QUALITY]: 5,
       },

lib/ui/api-functions.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ObjectID } from "mongodb";
+import { ObjectId } from "mongodb";
 import * as db from "./db";
 import { del } from "../cache";
 import { getUsers } from "../local-cache";
@@ -38,7 +38,7 @@ async function deleteFault(id): Promise<void> {
   await Promise.all([
     db.deleteFault(id),
     channel.startsWith("task_")
-      ? db.deleteTask(new ObjectID(channel.slice(5)))
+      ? db.deleteTask(new ObjectId(channel.slice(5)))
       : null,
   ]);
 
@@ -108,7 +108,7 @@ export async function postTasks(
   const lastInform = Date.now();
   const notInSession = await awaitSessionEnd(deviceId, 30000);
   if (!notInSession) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
     return {
       connectionRequest: "Session took too long to complete",
       tasks: statuses,
@@ -120,7 +120,7 @@ export async function postTasks(
   const status = await connectionRequest(deviceId, device);
 
   if (status) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
     return {
       connectionRequest: status,
       tasks: statuses,
@@ -129,7 +129,7 @@ export async function postTasks(
 
   const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
   if (!sessionStarted) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
     return {
       connectionRequest: "No contact from CPE",
       tasks: statuses,
@@ -138,7 +138,7 @@ export async function postTasks(
 
   const sessionEnded = await awaitSessionEnd(deviceId, 120000);
   if (!sessionEnded) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
     return {
       connectionRequest: "Session took too long to complete",
       tasks: statuses,
@@ -163,7 +163,7 @@ export async function postTasks(
     }
   }
 
-  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
 
   return { connectionRequest: "OK", tasks: statuses };
 }
@@ -200,7 +200,7 @@ export function authLocal(
   return new Promise((resolve, reject) => {
     const users = getUsers(snapshot);
     const user = users[username];
-    if (!user || !user.password) return void resolve(null);
+    if (!user?.password) return void resolve(null);
     hashPassword(password, user.salt)
       .then((hash) => {
         if (hash === user.password) resolve(true);

lib/ui/api.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import * as stream from "stream";
+import { PassThrough } from "stream";
 import Router from "koa-router";
 import * as db from "./db";
 import * as apiFunctions from "./api-functions";
@@ -30,7 +30,6 @@ import { generateSalt, hashPassword } from "../auth";
 import { del } from "../cache";
 import Authorizer from "../common/authorizer";
 import { ping } from "../ping";
-import * as url from "url";
 import { decodeTag } from "../common";
 import { stringify as yamlStringify } from "../common/yaml";
 
@@ -104,7 +103,7 @@ router.get(`/devices/:id.csv`, async (ctx) => {
       .replace(/[:.]/g, "")}.csv`
   );
 
-  ctx.body = new stream.PassThrough();
+  ctx.body = new PassThrough();
   ctx.body.write(
     "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp,Notification,Notification timestamp,Access list,Access list timestamp\n"
   );
@@ -126,7 +125,7 @@ router.get(`/devices/:id.csv`, async (ctx) => {
       new Date(p.objectTimestamp).toJSON(),
       p.writable,
       new Date(p.writableTimestamp).toJSON(),
-      `"${value.toString().replace(/"/g, '""')}"`,
+      `"${String(value).replace(/"/g, '""')}"`,
       type,
       new Date(p.valueTimestamp).toJSON(),
       p.notification,
@@ -217,17 +216,21 @@ for (const [resource, flags] of Object.entries(resources)) {
       ]);
     }
 
-    ctx.body = new stream.PassThrough();
+    ctx.body = new PassThrough();
     ctx.type = "application/json";
 
     let c = 0;
     ctx.body.write("[\n");
-    await db.query(resource, filter, options, (obj) => {
+    db.query(resource, filter, options, (obj) => {
       ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
-    });
+    })
-    ctx.body.end("]");
+      .then(() => {
-
+        ctx.body.end("]");
-    logger.accessInfo(log);
+        logger.accessInfo(log);
+      })
+      .catch((err) => {
+        ctx.body.emit("error", err);
+      });
   });
 
   // CSV download
@@ -277,7 +280,7 @@ for (const [resource, flags] of Object.entries(resources)) {
       ]);
     }
 
-    ctx.body = new stream.PassThrough();
+    ctx.body = new PassThrough();
     ctx.type = "text/csv";
     ctx.attachment(
       `${resource}-${new Date(now).toISOString().replace(/[:.]/g, "")}.csv`
@@ -286,7 +289,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     ctx.body.write(
       Object.keys(columns).map((k) => `"${k.replace(/"/, '""')}"`) + "\n"
     );
-    await db.query(resource, filter, options, (obj) => {
+    db.query(resource, filter, options, (obj) => {
       const arr = Object.values(columns).map((exp) => {
         const v = evaluate(exp, obj, null, (e) => {
           if (Array.isArray(e)) {
@@ -328,10 +331,14 @@ for (const [resource, flags] of Object.entries(resources)) {
         return v;
       });
       ctx.body.write(arr.join(",") + "\n");
-    });
+    })
-    ctx.body.end();
+      .then(() => {
-
+        ctx.body.end();
-    logger.accessInfo(log);
+        logger.accessInfo(log);
+      })
+      .catch((err) => {
+        ctx.body.emit("error", err);
+      });
   });
 
   router.head(`/${resource}/:id`, async (ctx) => {
@@ -509,11 +516,11 @@ router.put("/files/:id", async (ctx) => {
   }
 
   const metadata = {
-    fileType: singleParam(ctx.request.headers["metadata.filetype"]) || "",
+    fileType: singleParam(ctx.request.headers["metadata-filetype"]) || "",
-    oui: singleParam(ctx.request.headers["metadata.oui"]) || "",
+    oui: singleParam(ctx.request.headers["metadata-oui"]) || "",
     productClass:
-      singleParam(ctx.request.headers["metadata.productclass"]) || "",
+      singleParam(ctx.request.headers["metadata-productclass"]) || "",
-    version: singleParam(ctx.request.headers["metadata.version"]) || "",
+    version: singleParam(ctx.request.headers["metadata-version"]) || "",
   };
 
   const validate = authorizer.getValidator(resource, metadata);
@@ -574,12 +581,12 @@ router.post("/devices/:id/tasks", async (ctx) => {
       if (!Array.isArray(exp)) return exp;
       if (exp[0] === "PARAM") {
         const p = device[exp[1]];
-        if (p && p.value) return p.value[0];
+        if (p?.value) return p.value[0];
       } else if (exp[0] === "FUNC") {
         if (exp[1] === "REMOTE_ADDRESS") {
           for (const root of ["InternetGatewayDevice", "Device"]) {
             const p = device[`${root}.ManagementServer.ConnectionRequestURL`];
-            if (p && p.value) return url.parse(p.value[0]).host;
+            if (p?.value) return new URL(p.value[0]).hostname;
           }
           return null;
         }
@@ -653,13 +660,14 @@ router.post("/devices/:id/tags", async (ctx) => {
 });
 
 router.get("/ping/:host", async (ctx) => {
+  if (!ctx.state.user) return void (ctx.status = 401);
   return new Promise<void>((resolve) => {
     ping(ctx.params.host, (err, parsed) => {
       if (parsed) {
         ctx.body = parsed;
       } else {
         ctx.status = 500;
-        ctx.body = `${err.name}: ${err.message}`;
+        ctx.body = err ? `${err.name}: ${err.message}` : "Unknown error";
       }
       resolve();
     });

lib/ui/db.ts

@@ -17,13 +17,13 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Db, GridFSBucket, ObjectID } from "mongodb";
+import { Db, GridFSBucket, ObjectId } from "mongodb";
 import { Script } from "vm";
-import { onConnect } from "../db";
+import { onConnect, optimizeProjection } from "../db";
 import * as mongodbFunctions from "../mongodb-functions";
 import * as expression from "../common/expression";
 import { QueryOptions, Expression } from "../types";
-import { Readable } from "stream";
+import { Readable, Writable } from "stream";
 import { minimize } from "../common/boolean-expression";
 
 const RESOURCE_COLLECTION = {
@@ -89,6 +89,7 @@ export function query(
       }
 
       if (resource === "presets") projection.configurations = 1;
+      projection = optimizeProjection(projection);
       cursor.project(projection);
     }
 
@@ -114,30 +115,31 @@ export function query(
       cursor.toArray((err, docs) => {
         if (err) return reject(err);
         if (resource === "devices")
-          docs = docs.map((d) => mongodbFunctions.flattenDevice(d));
+          return resolve(docs.map((d) => mongodbFunctions.flattenDevice(d)));
         else if (resource === "faults")
-          docs = docs.map((d) => mongodbFunctions.flattenFault(d));
+          return resolve(docs.map((d) => mongodbFunctions.flattenFault(d)));
         else if (resource === "tasks")
-          docs = docs.map((d) => mongodbFunctions.flattenTask(d));
+          return resolve(docs.map((d) => mongodbFunctions.flattenTask(d)));
         else if (resource === "presets")
-          docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
+          return resolve(docs.map((d) => mongodbFunctions.flattenPreset(d)));
         else if (resource === "files")
-          docs = docs.map((d) => mongodbFunctions.flattenFile(d));
+          return resolve(docs.map((d) => mongodbFunctions.flattenFile(d)));
         return resolve(docs);
       });
     } else {
       cursor.forEach(
         (doc) => {
-          if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc);
+          if (resource === "devices")
+            callback(mongodbFunctions.flattenDevice(doc));
           else if (resource === "faults")
-            doc = mongodbFunctions.flattenFault(doc);
+            callback(mongodbFunctions.flattenFault(doc));
           else if (resource === "tasks")
-            doc = mongodbFunctions.flattenTask(doc);
+            callback(mongodbFunctions.flattenTask(doc));
           else if (resource === "presets")
-            doc = mongodbFunctions.flattenPreset(doc);
+            callback(mongodbFunctions.flattenPreset(doc));
           else if (resource === "files")
-            doc = mongodbFunctions.flattenFile(doc);
+            callback(mongodbFunctions.flattenFile(doc));
-          callback(doc);
+          else callback(doc);
         },
         (err) => {
           if (err) reject(err);
@@ -149,7 +151,8 @@ export function query(
 }
 
 export function count(resource: string, filter: Expression): Promise<number> {
-  let q;
+  const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
+  let q: Parameters<typeof collection.countDocuments>[0];
   filter = expression.evaluate(filter, null, Date.now());
   filter = minimize(filter, true);
 
@@ -165,13 +168,7 @@ export function count(resource: string, filter: Expression): Promise<number> {
     return Promise.resolve(0);
   }
 
-  return new Promise((resolve, reject) => {
+  return collection.countDocuments(q);
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-    collection.find(q).count((err, c) => {
-      if (err) reject(err);
-      else resolve(c);
-    });
-  });
 }
 
 export async function updateDeviceTags(
@@ -190,8 +187,8 @@ export async function updateDeviceTags(
   const collection = db.collection("devices");
   const object = {};
 
-  if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
+  if (add?.length) object["$addToSet"] = { _tags: { $each: add } };
-  if (pull && pull.length) object["$pullAll"] = { _tags: pull };
+  if (pull?.length) object["$pullAll"] = { _tags: pull };
 
   await collection.updateOne({ _id: deviceId }, object);
 }
@@ -208,7 +205,7 @@ function putResource(resource, id, object): Promise<void> {
 
 function deleteResource(
   resource: string,
-  id: string | ObjectID
+  id: string | ObjectId
 ): Promise<void> {
   return new Promise((resolve, reject) => {
     const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
@@ -242,7 +239,7 @@ export function putProvision(
       lineOffset: -1,
     });
   } catch (err) {
-    if (err.stack && err.stack.startsWith(`${id}:`)) {
+    if (err.stack?.startsWith(`${id}:`)) {
       return Promise.reject(
         new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`)
       );
@@ -267,7 +264,7 @@ export function putVirtualParameter(
       lineOffset: -1,
     });
   } catch (err) {
-    if (err.stack && err.stack.startsWith(`${id}:`)) {
+    if (err.stack?.startsWith(`${id}:`)) {
       return Promise.reject(
         new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`)
       );
@@ -338,13 +335,17 @@ export function putFile(
 ): Promise<void> {
   return new Promise((resolve, reject) => {
     const bucket = new GridFSBucket(db);
-    const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
+    const uploadStream = bucket.openUploadStreamWithId(
-      metadata: metadata,
+      filename as unknown as ObjectId,
-    });
+      filename,
+      {
+        metadata: metadata,
+      }
+    );
     uploadStream.on("error", reject);
     contentStream.on("error", reject);
     uploadStream.on("finish", resolve);
-    contentStream.pipe(uploadStream);
+    contentStream.pipe(uploadStream as Writable);
   });
 }
 
@@ -362,6 +363,6 @@ export function deleteFault(id: string): Promise<void> {
   return deleteResource("faults", id);
 }
 
-export function deleteTask(id: ObjectID): Promise<void> {
+export function deleteTask(id: ObjectId): Promise<void> {
   return deleteResource("tasks", id);
 }

lib/versioned-map.ts

@@ -26,10 +26,10 @@ interface Revisions<V> {
 }
 
 export default class VersionedMap<K, V> {
-  private _sizeDiff: number[];
+  private declare _sizeDiff: number[];
-  private _revision: number;
+  private declare _revision: number;
-  private map: Map<K, (V | symbol)[]>;
+  private declare map: Map<K, (V | symbol)[]>;
-  public dirty: number;
+  public declare dirty: number;
 
   public constructor() {
     this._sizeDiff = [0];
@@ -168,6 +168,7 @@ export default class VersionedMap<K, V> {
 
   public *diff(): IterableIterator<[K, V, V]> {
     for (const [key, revisions] of this.map) {
+      if (revisions.length === 1) continue;
       let first = revisions[0];
       let last = revisions[revisions.length - 1];
       if (first === NONEXISTENT && last === NONEXISTENT) continue;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.5",
+  "version": "1.2.9",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -19,56 +19,57 @@
     "bin": "bin"
   },
   "dependencies": {
-    "espresso-iisojs": "^1.0.3",
+    "@breejs/later": "^4.1.0",
-    "iconv-lite": "^0.6.2",
+    "espresso-iisojs": "^1.0.4",
-    "ipaddr.js": "^2.0.0",
+    "iconv-lite": "^0.6.3",
+    "ipaddr.js": "^2.0.1",
     "jsonwebtoken": "^8.5.1",
-    "koa": "^2.13.1",
+    "koa": "^2.13.4",
     "koa-bodyparser": "^4.3.0",
-    "koa-compress": "^5.0.1",
+    "koa-compress": "^5.1.0",
-    "koa-jwt": "^4.0.0",
+    "koa-jwt": "^4.0.3",
-    "koa-router": "^10.0.0",
+    "koa-router": "^12.0.0",
     "koa-static": "^5.0.0",
-    "later": "^1.2.0",
+    "mongodb": "^4.9.0",
-    "mongodb": "^3.6.3",
+    "parsimmon": "^1.18.1",
-    "parsimmon": "^1.16.0",
     "seedrandom": "^3.0.5"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^17.1.0",
+    "@rollup/plugin-commonjs": "^22.0.2",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^11.2.0",
+    "@rollup/plugin-node-resolve": "^13.3.0",
-    "@rollup/plugin-typescript": "^8.2.0",
+    "@rollup/plugin-typescript": "^8.3.4",
-    "@types/codemirror": "0.0.108",
+    "@types/codemirror": "^5.60.5",
-    "@types/jsonwebtoken": "^8.5.0",
+    "@types/jsonwebtoken": "^8.5.8",
-    "@types/koa": "^2.13.1",
+    "@types/koa": "^2.13.5",
-    "@types/koa-compress": "^4.0.1",
+    "@types/koa-compress": "^4.0.3",
-    "@types/koa-router": "^7.4.1",
+    "@types/koa-router": "^7.4.4",
-    "@types/koa-static": "^4.0.1",
+    "@types/koa-static": "^4.0.2",
-    "@types/mithril": "^2.0.6",
+    "@types/mithril": "^2.0.11",
-    "@types/mongodb": "^3.6.9",
+    "@types/node": "^18.7.9",
-    "@types/node": "^14.14.34",
     "@types/parsimmon": "^1.10.6",
-    "@types/seedrandom": "^3.0.0",
+    "@types/seedrandom": "^3.0.2",
-    "@typescript-eslint/eslint-plugin": "^4.17.0",
+    "@typescript-eslint/eslint-plugin": "^5.33.1",
-    "@typescript-eslint/parser": "^4.17.0",
+    "@typescript-eslint/parser": "^5.33.1",
     "ava": "^3.15.0",
-    "codemirror": "^5.59.4",
+    "codemirror": "^5.65.8",
-    "cssnano": "^4.1.10",
+    "cssnano": "^5.1.13",
-    "eslint": "^7.22.0",
+    "eslint": "^8.22.0",
-    "eslint-config-prettier": "^8.1.0",
+    "eslint-config-prettier": "^8.5.0",
     "esm": "^3.2.25",
-    "mithril": "^2.0.4",
+    "mithril": "^2.2.2",
-    "postcss": "^8.2.8",
+    "postcss": "^8.4.16",
-    "postcss-import": "^14.0.0",
+    "postcss-color-mod-function": "^3.0.3",
-    "postcss-preset-env": "^6.7.0",
+    "postcss-import": "^14.1.0",
-    "prettier": "^2.2.1",
+    "postcss-preset-env": "^7.8.0",
-    "rollup": "^2.41.2",
+    "prettier": "^2.7.1",
+    "rollup": "^2.78.1",
     "rollup-plugin-terser": "^7.0.2",
-    "svgo": "^2.2.2",
+    "sql.js": "^1.7.0",
-    "ts-node": "^9.1.1",
+    "svgo": "^2.8.0",
-    "typescript": "^4.2.3",
+    "ts-node": "^10.9.1",
-    "yaml": "^1.10.0"
+    "typescript": "^4.7.4",
+    "yaml": "^1.10.2"
   },
   "engines": {
     "node": ">=12.13.0"

test/boolean-expression.ts

@@ -0,0 +1,118 @@
+import ava, { TestInterface } from "ava";
+import { minimize, unionDiff } from "../lib/common/boolean-expression";
+import { parse, stringify } from "../lib/common/expression-parser";
+import initSqlJs from "sql.js/dist/sql-asm.js";
+
+const test = ava as TestInterface<{
+  query: (filter: string) => Promise<Set<number>>;
+}>;
+
+const STRING_VALUES = [null, "", "a", "ab", "ab10", "ab-10"];
+const DECIMAL_VALUES = [null, 0, -10, 10];
+
+test.before(async (t) => {
+  const sql = await initSqlJs();
+  const db = new sql.Database();
+
+  t.context.query = async (filter: string): Promise<Set<number>> => {
+    const res = db.exec(`SELECT id FROM test WHERE ${filter}`);
+    if (!res.length) return new Set();
+    return new Set(res[0].values.flat());
+  };
+
+  db.run(
+    "CREATE TABLE test (id INTEGER PRIMARY KEY, string STRING, decimal DECIMAL(4,2))"
+  );
+
+  const stmt = db.prepare("INSERT INTO test (string, decimal) VALUES (?, ?)");
+  const count = STRING_VALUES.length * DECIMAL_VALUES.length;
+  for (let i = 0; i < count; ++i) {
+    const str = i % STRING_VALUES.length;
+    const dec = Math.trunc(i / STRING_VALUES.length) % DECIMAL_VALUES.length;
+    stmt.run([STRING_VALUES[str], DECIMAL_VALUES[dec]]);
+  }
+  stmt.free();
+});
+
+function setsEqual(set1: Set<number>, set2: Set<number>): boolean {
+  if (set1.size !== set2.size) return false;
+  for (const s of set1) if (!set2.has(s)) return false;
+  return true;
+}
+
+function getPermutations(...arrs: any[][]): any[][] {
+  const count = arrs.reduce((total, arr) => total * arr.length, 1);
+  const res = [];
+  for (let i = 0; i < count; ++i) {
+    let j = i;
+    const row = [];
+    for (const arr of arrs) {
+      const v = arr[j % arr.length];
+      j = Math.trunc(j / arr.length);
+      row.push(v);
+    }
+    res.push(row);
+  }
+  return res;
+}
+
+test("minimize", async (t) => {
+  const cases: string[] = [];
+
+  cases.push("null");
+  cases.push("false");
+  cases.push("true");
+  cases.push("string");
+  cases.push("(string + decimal) IS NULL");
+  cases.push("(string + decimal) = NULL");
+  cases.push("COALESCE(string, decimal) = 0");
+
+  for (const [s1, s2, s3, op1, op2] of getPermutations(
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s),
+    [">", "=", "<"],
+    ["<>", ">="]
+  )) {
+    cases.push(
+      `string ${op1} "${s1}" OR string ='${s2}' OR NOT string ${op2} '${s3}'`
+    );
+  }
+
+  for (const [s1, s2] of getPermutations(
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s)
+  ))
+    cases.push(`string > "${s1}" AND string < '${s2}'`);
+
+  for (const c of cases) {
+    const res1 = await t.context.query(c);
+    const min = stringify(minimize(parse(c), true));
+    const res2 = await t.context.query(min);
+    t.true(setsEqual(res1, res2));
+  }
+});
+
+test("unionDiff", async (t) => {
+  const cases = [
+    "true",
+    "decimal > 0",
+    "decimal > 10",
+    "UPPER(string || decimal) LIKE 'AB10'",
+    "COALESCE(string, decimal) = 0",
+  ];
+
+  for (const [c1, c2] of getPermutations(cases, cases)) {
+    const res1 = await t.context.query(c1);
+    const res2 = await t.context.query(c2);
+    const [union, diff] = unionDiff(parse(c1), parse(c2));
+    const res3 = await t.context.query(stringify(union));
+    const res4 = await t.context.query(stringify(diff));
+
+    const unionSet = new Set([...res1, ...res2]);
+    const diffSet = new Set(Array.from(res2).filter((r) => !res1.has(r)));
+
+    t.true(setsEqual(res3, unionSet));
+    t.true(setsEqual(res4, diffSet));
+  }
+});

test/mongodb-functions.ts

@@ -49,8 +49,16 @@ ava("convertOldPrecondition", (t) => {
   ];
 
   t.plan(tests.length + 2 * shouldFailTests.length);
-  for (const test of tests)
+  for (const test of tests) {
-    t.is(stringify(mongodbFunctions.convertOldPrecondition(test[0])), test[1]);
+    t.is(
+      stringify(
+        mongodbFunctions.convertOldPrecondition(
+          test[0] as Record<string, unknown>
+        )
+      ),
+      test[1]
+    );
+  }
 
   for (const test of shouldFailTests) {
     const func = (): void => {
@@ -72,6 +80,24 @@ ava("filterToMongoQuery", async (t) => {
     ["Tags.tag1 = 123", { "Tags.tag1": 123 }],
     ["Param1 = 'value1'", { "Param1._value": "value1" }],
     ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }],
+    [
+      "Param1 <> 1657844103524",
+      {
+        $and: [
+          { "Param1._value": { $nin: [1657844103524, null] } },
+          { "Param1._value": { $nin: [new Date(1657844103524), null] } },
+        ],
+      },
+    ],
+    [
+      "Param1 = 1657844103524",
+      {
+        $or: [
+          { "Param1._value": 1657844103524 },
+          { "Param1._value": new Date(1657844103524) },
+        ],
+      },
+    ],
     ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
     ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
     ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }],

test/path.ts

@@ -1,6 +1,25 @@
 import ava from "ava";
 import Path from "../lib/common/path";
 
+ava("parse", (t) => {
+  t.throws(() => Path.parse("."));
+  t.throws(() => Path.parse("a "));
+  t.throws(() => Path.parse(".a"));
+  t.throws(() => Path.parse("a."));
+  t.throws(() => Path.parse("a..b"));
+  t.throws(() => Path.parse("b*"));
+  t.throws(() => Path.parse("*b"));
+  t.throws(() => Path.parse("a.b c.d"));
+  t.throws(() => Path.parse("a["));
+  t.throws(() => Path.parse("a[b"));
+  t.throws(() => Path.parse("a[b:"));
+  t.throws(() => Path.parse('a[b:"waef]'));
+  t.notThrows(() => Path.parse("[]"));
+  t.notThrows(() => Path.parse("[a:]"));
+  t.notThrows(() => Path.parse("*"));
+  t.notThrows(() => Path.parse(""));
+});
+
 ava("toString", (t) => {
   const path1 = Path.parse('abc.[ abc : 123 , 123 : abc , 123: " abc "].123');
   const path2 = Path.parse('abc.[123:  " abc ",abc:123,123:abc].123');

test/ping.ts

@@ -0,0 +1,36 @@
+import ava from "ava";
+import { parsePing } from "../lib/ping";
+
+ava("linux Case-1", (t) => {
+  const platform = "linux";
+  const stdout =
+    "PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data\n64 bytes from 8.8.8.8: icmp_seq=1 ttl=118 time=6.66 ms\n64 bytes from 8.8.8.8: icmp_seq=2 ttl=118 time=6.50 ms\n64 bytes from 8.8.8.8: icmp_seq=3 ttl=118 time=6.38 ms\n\n--- 8.8.8.8 ping statistics ---\n3 packets transmitted, 3 received, 0% packet loss, time 402ms\nrtt min/avg/max/mdev = 6.381/6.511/6.656/0.112 ms";
+  const parsedResult = {
+    packetsTransmitted: 3,
+    packetsReceived: 3,
+    packetLoss: 0,
+    min: 6.381,
+    avg: 6.511,
+    max: 6.656,
+    mdev: 0.112,
+  };
+  const parsed = parsePing(platform, stdout);
+  t.deepEqual(parsedResult, parsed);
+});
+
+ava("linux Case-2", (t) => {
+  const platform = "linux";
+  const stdout =
+    "PING 10.251.9.108 (10.251.9.108): 56 data bytes\n64 bytes from 10.251.9.108: icmp_seq=0 ttl=57 time=36.758 ms\n64 bytes from 10.251.9.108: icmp_seq=1 ttl=57 time=69.094 ms\n64 bytes from 10.251.9.108: icmp_seq=2 ttl=57 time=28.868 ms\n--- 10.251.9.108 ping statistics ---3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max/stddev = 28.868/44.907/69.094/17.404 ms";
+  const parsedResult = {
+    packetsTransmitted: 3,
+    packetsReceived: 3,
+    packetLoss: 0,
+    min: 28.868,
+    avg: 44.907,
+    max: 69.094,
+    mdev: 17.404,
+  };
+  const parsed = parsePing(platform, stdout);
+  t.deepEqual(parsedResult, parsed);
+});

tsconfig.json

@@ -1,9 +1,9 @@
 {
   "compilerOptions": {
-    "lib": ["esnext", "dom"],
+    "lib": ["ES2019", "dom"],
-    "module": "esnext",
+    "module": "ES2020",
     "moduleResolution": "node",
-    "target": "esnext",
+    "target": "ES2019",
     "resolveJsonModule": true,
     "allowSyntheticDefaultImports": true,
     "alwaysStrict": true,

ui/app.ts

@@ -69,7 +69,7 @@ function pagify(pageName, page): RouteResolver {
     render: () => {
       const lastRenderTimestamp = Date.now();
       let p;
-      if (state && state.error) p = m(errorPage.component, state);
+      if (state?.error) p = m(errorPage.component, state);
       else p = m(contextifyComponent(page.component), state);
       const attrs = {};
       attrs["page"] = pageName;

ui/autocomplete-compnent.ts

@@ -23,13 +23,13 @@ type AutocompleteCallback = (
 ) => void;
 
 export default class Autocomplete {
-  private callback: AutocompleteCallback;
+  private declare callback: AutocompleteCallback;
-  private element: HTMLInputElement;
+  private declare element: HTMLInputElement;
-  private hideTimeout: NodeJS.Timeout;
+  private declare hideTimeout: NodeJS.Timeout;
-  private visible: boolean;
+  private declare visible: boolean;
-  private default: string;
+  private declare default: string;
-  private selection: number;
+  private declare selection: number;
-  private container: HTMLElement;
+  private declare container: HTMLElement;
 
   public constructor(className: string, callback: AutocompleteCallback) {
     this.callback = callback;
@@ -51,11 +51,8 @@ export default class Autocomplete {
 
     el.addEventListener("focus", () => {
       this.element = el;
-      const domRect = el.getBoundingClientRect();
-      this.container.style.left = `${domRect.left + window.pageXOffset}px`;
-      this.container.style.width = `${domRect.width}px`;
-      this.container.style.top = `${domRect.bottom + window.pageYOffset}px`;
       this.update();
+      this.reposition();
     });
 
     el.addEventListener("blur", () => {
@@ -71,7 +68,8 @@ export default class Autocomplete {
       } else if (e.key === "Enter") {
         if (this.default != null) {
           el.value = this.default;
-          e.preventDefault();
+          e.stopImmediatePropagation();
+          el.dispatchEvent(new InputEvent("input"));
           this.update();
         }
       } else if (e.key === "ArrowDown") {
@@ -93,6 +91,19 @@ export default class Autocomplete {
     });
   }
 
+  public reposition(): void {
+    if (!this.element) return;
+    const domRect = this.element.getBoundingClientRect();
+    if (!domRect.width) {
+      // Element has been removed
+      if (this.visible) this.hide();
+      return;
+    }
+    this.container.style.left = `${domRect.left + window.scrollX}px`;
+    this.container.style.width = `${domRect.width}px`;
+    this.container.style.top = `${domRect.bottom + window.scrollY}px`;
+  }
+
   private hide(): void {
     this.container.style.opacity = "0";
     this.visible = false;
@@ -159,6 +170,7 @@ export default class Autocomplete {
         e.addEventListener("mousedown", (ev) => {
           ev.preventDefault();
           el.value = suggestion.value;
+          el.dispatchEvent(new InputEvent("input"));
           if (this.element === el) this.update();
         });
         this.container.appendChild(e);

ui/change-password-component.ts

@@ -17,17 +17,23 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { VnodeDOM, ClosureComponent, Component } from "mithril";
+import { VnodeDOM, ClosureComponent } from "mithril";
 import * as notifications from "./notifications";
 import { changePassword } from "./store";
 import { m } from "./components";
 
-const component: ClosureComponent = (): Component => {
+interface Attrs {
+  noAuth?: boolean;
+  username?: string;
+  onPasswordChange: () => void;
+}
+
+const component: ClosureComponent<Attrs> = () => {
   return {
     view: (vnode) => {
-      const onPasswordChange = vnode.attrs["onPasswordChange"];
+      const onPasswordChange = vnode.attrs.onPasswordChange;
-      const enforceAuth = !vnode.attrs["noAuth"];
+      const enforceAuth = !vnode.attrs.noAuth;
-      const username = vnode.attrs["username"];
+      const username = vnode.attrs.username;
 
       if (username) vnode.state["username"] = username;
 

ui/code-editor-component.ts

@@ -17,44 +17,54 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ClosureComponent, Component } from "mithril";
+import { Editor } from "codemirror";
+import { ClosureComponent } from "mithril";
 import { m } from "./components";
 import { codeMirror } from "./dynamic-loader";
 
-const component: ClosureComponent = (): Component => {
+interface Attrs {
+  id: string;
+  value: string;
+  mode: string;
+  readOnly?: boolean;
+  focus?: boolean;
+  onSubmit?: (dom: Element) => void;
+  onChange?: (value: string) => void;
+  onReady?: (editor: Editor) => void;
+}
+
+const component: ClosureComponent<Attrs> = () => {
   return {
     view: (vnode) => {
       return m("textarea", {
-        name: vnode.attrs["id"],
+        name: vnode.attrs.id,
-        value: vnode.attrs["value"],
+        value: vnode.attrs.value,
         oncreate: (_vnode) => {
           const editor = codeMirror.fromTextArea(
             _vnode.dom as HTMLTextAreaElement,
             {
-              mode: vnode.attrs["mode"],
+              mode: vnode.attrs.mode,
               lineNumbers: true,
-              readOnly: vnode.attrs["readOnly"],
+              readOnly: vnode.attrs.readOnly,
               extraKeys: {
                 "Ctrl-Enter": () => {
-                  if (vnode.attrs["onSubmit"])
+                  if (vnode.attrs.onSubmit) vnode.attrs.onSubmit(_vnode.dom);
-                    vnode.attrs["onSubmit"](_vnode.dom);
                 },
                 "Cmd-Enter": () => {
-                  if (vnode.attrs["onSubmit"])
+                  if (vnode.attrs.onSubmit) vnode.attrs.onSubmit(_vnode.dom);
-                    vnode.attrs["onSubmit"](_vnode.dom);
                 },
               },
             }
           );
 
-          if (vnode.attrs["onChange"]) {
+          if (vnode.attrs.onChange) {
             editor.on("change", (e) => {
-              vnode.attrs["onChange"](e.getValue());
+              vnode.attrs.onChange(e.getValue());
             });
           }
 
-          if (vnode.attrs["focus"]) editor.focus();
+          if (vnode.attrs.focus) editor.focus();
-          if (vnode.attrs["onReady"]) vnode.attrs["onReady"](editor);
+          if (vnode.attrs.onReady) vnode.attrs.onReady(editor);
         },
       });
     },

ui/components.ts

@@ -22,7 +22,7 @@ import m, {
   Attributes,
   Children,
   ComponentTypes,
-  Lifecycle,
+  CommonAttributes,
   ClosureComponent,
   Vnode,
 } from "mithril";
@@ -81,7 +81,7 @@ interface MC extends Static {
     <Attrs, State>(
       ctx: Attributes,
       component: ComponentTypes<Attrs, State>,
-      attributes: Attrs & Lifecycle<Attrs, State> & { key?: string | number },
+      attributes: Attrs & CommonAttributes<Attrs, State>,
       ...args: Children[]
     ): Vnode<Attrs, State>;
   };
@@ -116,7 +116,7 @@ function applyContext(vnode, parentContext): void {
       vnodeContext.set(vnode, vc);
       vnode.attrs = Object.assign({}, vc, vnode.attrs);
     }
-    if (vnode.children && vnode.children.length)
+    if (vnode.children?.length)
       for (const c of vnode.children) applyContext(c, vc);
   }
 }
@@ -133,7 +133,7 @@ export function contextifyComponent(component: ComponentTypes): ComponentTypes {
         applyContext(res, context);
         return res;
       };
-    } else if (!component.prototype || !component.prototype.view) {
+    } else if (!component.prototype?.view) {
       c = (initialNode) => {
         const state = (component as ClosureComponent)(initialNode);
         const view = state.view;
@@ -144,7 +144,14 @@ export function contextifyComponent(component: ComponentTypes): ComponentTypes {
             applyContext(res, context);
             return res;
           } catch (err) {
-            return m("p.error", { title: err.message }, "Error!");
+            return m(
+              "p.error",
+              {
+                title: "Click to print stack trace to console",
+                onclick: () => console.error(err),
+              },
+              "Error!"
+            );
           }
         };
         return state;

ui/components/all-parameters.ts

@@ -23,6 +23,8 @@ import * as taskQueue from "../task-queue";
 import { parse } from "../../lib/common/expression-parser";
 import memoize from "../../lib/common/memoize";
 import { getIcon } from "../icons";
+import { evaluateExpression } from "../store";
+import debounce from "../../lib/common/debounce";
 
 const memoizedParse = memoize(parse);
 
@@ -51,29 +53,31 @@ function orderKeysByDepth(device: Record<string, unknown>): string[][] {
 }
 
 const component: ClosureComponent = (): Component => {
+  let queryString: string;
+  const formQueryString = debounce((args: string[]) => {
+    queryString = args[args.length - 1];
+    m.redraw();
+  }, 500);
+
   return {
     view: (vnode) => {
       const device = vnode.attrs["device"];
 
-      const limit = vnode.attrs["limit"] > 0 ? vnode.attrs["limit"] : 100;
+      const limit = evaluateExpression(vnode.attrs["limit"], device) || 100;
 
       const search = m("input", {
         type: "text",
         placeholder: "Search parameters",
         oninput: (e) => {
-          vnode.state["searchString"] = e.target.value;
+          formQueryString(e.target.value);
           e.redraw = false;
-          clearTimeout(vnode.state["timeout"]);
-          vnode.state["timeout"] = setTimeout(m.redraw, 500);
         },
       });
 
       const instanceRegex = /\.[0-9]+$/;
       let re;
-      if (vnode.state["searchString"]) {
+      if (queryString) {
-        const keywords = vnode.state["searchString"]
+        const keywords = queryString.split(" ").filter((s) => s);
-          .split(" ")
-          .filter((s) => s);
         if (keywords.length)
           re = new RegExp(keywords.map((s) => escapeRegExp(s)).join(".*"), "i");
       }
@@ -85,7 +89,7 @@ const component: ClosureComponent = (): Component => {
         let c = 0;
         for (const k of keys) {
           const p = device[k];
-          const str = p.value && p.value[0] ? `${k} ${p.value[0]}` : k;
+          const str = p.value?.[0] ? `${k} ${p.value[0]}` : k;
           if (re && !re.test(str)) continue;
           ++c;
           if (count < limit) filteredKeys.push(k);

ui/components/container.ts

@@ -17,29 +17,76 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ClosureComponent, Component } from "mithril";
+import { Attributes, ClosureComponent, Component } from "mithril";
+import { map } from "../../lib/common/expression-parser";
+import memoize from "../../lib/common/memoize";
+import { Expression } from "../../lib/types";
 import { m } from "../components";
-import * as store from "../store";
+import { evaluateExpression, getTimestamp } from "../store";
+
+const evaluateAttributes = memoize(
+  (
+    attrs: Record<string, Expression>,
+    obj: Record<string, unknown>,
+    now: number // eslint-disable-line @typescript-eslint/no-unused-vars
+  ): Attributes => {
+    const res: Attributes = {};
+    for (const [k, v] of Object.entries(attrs)) {
+      const vv = map(v, (e) => {
+        if (
+          Array.isArray(e) &&
+          e[0] === "FUNC" &&
+          e[1] === "ENCODEURICOMPONENT"
+        ) {
+          const a = evaluateExpression(e[2], obj);
+          if (a == null) return null;
+          return encodeURIComponent(a as string);
+        }
+        return e;
+      });
+      res[k] = evaluateExpression(vv, obj);
+    }
+    return res;
+  }
+);
 
 const component: ClosureComponent = (): Component => {
   return {
     view: (vnode) => {
-      if (vnode.attrs["filter"]) {
+      const device = vnode.attrs["device"];
-        if (
+      if ("filter" in vnode.attrs) {
-          !store.evaluateExpression(
+        if (!evaluateExpression(vnode.attrs["filter"], device || {}))
-            vnode.attrs["filter"],
-            vnode.attrs["device"]
-          )
-        )
           return null;
       }
 
       const children = Object.values(vnode.attrs["components"]).map((c) => {
+        if (Array.isArray(c)) c = evaluateExpression(c, device || {});
         if (typeof c !== "object") return `${c}`;
-        return m(c["type"], c);
+        const type = evaluateExpression(c["type"], device || {});
+        if (!type) return null;
+        return m(type as string, c);
       });
-      if (vnode.attrs["element"]) return m(vnode.attrs["element"], children);
+
-      else return children;
+      let el = vnode.attrs["element"];
+
+      if (el == null) return children;
+
+      let attrs: Attributes;
+      if (Array.isArray(el)) {
+        el = evaluateExpression(el, device || {});
+      } else if (typeof el === "object") {
+        if (el["attributes"] != null) {
+          attrs = evaluateAttributes(
+            el["attributes"],
+            device || {},
+            getTimestamp()
+          );
+        }
+
+        el = evaluateExpression(el["tag"], device || {});
+      }
+
+      return m(el, attrs, children);
     },
   };
 };

ui/components/device-link.ts

@@ -19,18 +19,22 @@
 
 import { ClosureComponent, Component } from "mithril";
 import { m } from "../components";
+import { evaluateExpression } from "../store";
 
 const component: ClosureComponent = (): Component => {
   return {
     view: (vnode) => {
       let deviceId;
-      if (vnode.attrs["device"])
+      const device = vnode.attrs["device"];
-        deviceId = vnode.attrs["device"]["DeviceID.ID"].value[0];
+      if (device) deviceId = device["DeviceID.ID"].value[0];
 
       const children = Object.values(vnode.attrs["components"]).map((c) => {
+        if (Array.isArray(c)) c = evaluateExpression(c, device || {});
         if (typeof c !== "object") return `${c}`;
+        const type = evaluateExpression(c["type"], device || {});
+        if (!type) return null;
         const attrs = Object.assign({}, vnode.attrs, c);
-        return m(attrs["type"], attrs);
+        return m(type as string, attrs);
       });
       if (deviceId) {
         return m(

ui/components/overview-dot.ts

@@ -20,7 +20,7 @@
 import { ClosureComponent, Component } from "mithril";
 import { m } from "../components";
 import config from "../config";
-import * as store from "../store";
+import { evaluateExpression } from "../store";
 
 const CHARTS = config.ui.overview.charts;
 
@@ -28,11 +28,12 @@ const component: ClosureComponent = (): Component => {
   return {
     view: (vnode) => {
       const device = vnode.attrs["device"];
-      const chart = CHARTS[vnode.attrs["chart"]] as Record<string, unknown>;
+      const chartName = evaluateExpression(vnode.attrs["chart"], device || {});
+      const chart = CHARTS[chartName as string] as Record<string, unknown>;
       if (!chart) return null;
       for (const slice of Object.values(chart.slices)) {
         const filter = slice["filter"];
-        if (store.evaluateExpression(filter, device)) {
+        if (evaluateExpression(filter, device || {})) {
           const dot = m(
             "svg",
             {

ui/components/parameter-list.ts

@@ -19,6 +19,7 @@
 
 import { ClosureComponent, Component, VnodeDOM } from "mithril";
 import { m } from "../components";
+import { evaluateExpression } from "../store";
 
 const component: ClosureComponent = (): Component => {
   return {
@@ -26,12 +27,13 @@ const component: ClosureComponent = (): Component => {
       const device = vnode.attrs["device"];
 
       const rows = Object.values(vnode.attrs["parameters"]).map((parameter) => {
+        const type = evaluateExpression(parameter["type"], device);
         const p = m.context(
           {
             device: device,
             parameter: parameter["parameter"],
           },
-          parameter["type"] || "parameter",
+          (type as string) || "parameter",
           parameter
         );
 

ui/components/parameter-table.ts

@@ -20,7 +20,7 @@
 import { ClosureComponent, Component } from "mithril";
 import { m } from "../components";
 import * as taskQueue from "../task-queue";
-import * as store from "../store";
+import { evaluateExpression } from "../store";
 import * as expressionParser from "../../lib/common/expression-parser";
 import { getIcon } from "../icons";
 
@@ -35,7 +35,7 @@ const component: ClosureComponent = (): Component => {
     },
     view: (vnode) => {
       const device = vnode.attrs["device"];
-      const object = store.evaluateExpression(vnode.state["object"], device);
+      const object = evaluateExpression(vnode.state["object"], device);
       const parameters = vnode.state["parameters"];
 
       if (typeof object !== "string" || !device[object]) return null;
@@ -56,8 +56,7 @@ const component: ClosureComponent = (): Component => {
 
       const rows = [];
       for (const i of instances) {
-        let filter =
+        let filter = "filter" in vnode.attrs ? vnode.attrs["filter"] : true;
-          vnode.attrs["filter"] != null ? vnode.attrs["filter"] : true;
 
         filter = expressionParser.map(filter, (e) => {
           if (Array.isArray(e) && e[0] === "PARAM")
@@ -65,7 +64,7 @@ const component: ClosureComponent = (): Component => {
           return e;
         });
 
-        if (!store.evaluateExpression(filter, device)) continue;
+        if (!evaluateExpression(filter, device)) continue;
 
         const row = parameters.map((p) => {
           const param = expressionParser.map(p.parameter, (e) => {
@@ -147,7 +146,9 @@ const component: ClosureComponent = (): Component => {
       }
 
       let label;
-      if (vnode.attrs["label"]) label = m("h2", vnode.attrs["label"]);
+
+      const l = evaluateExpression(vnode.attrs["label"], device);
+      if (l != null) label = m("h2", l);
 
       return [
         label,

ui/components/parameter.ts

@@ -38,7 +38,7 @@ const evaluateParam = memoize((exp, obj, now: number) => {
       ) {
         let v = null;
         const p = obj[e[i][1]];
-        if (p && p.value) {
+        if (p?.value) {
           v = p.value[0];
           timestamp = Math.min(timestamp, p.valueTimestamp);
         }
@@ -58,7 +58,7 @@ const evaluateParam = memoize((exp, obj, now: number) => {
     value = exp;
   } else if (exp[0] === "PARAM") {
     const p = obj[exp[1]];
-    if (p && p.value) {
+    if (p?.value) {
       timestamp = p.valueTimestamp;
       value = p.value[0];
       parameter = exp[1];
@@ -77,14 +77,14 @@ const component: ClosureComponent = (): Component => {
 
       const { value, timestamp, parameter } = evaluateParam(
         vnode.attrs["parameter"],
-        vnode.attrs["device"],
+        device,
-        store.getTimestamp()
+        store.getTimestamp() + store.getClockSkew()
       );
 
       if (value == null) return null;
 
       let edit;
-      if (device[parameter] && device[parameter].writable) {
+      if (device[parameter]?.writable) {
         edit = m(
           "button",
           {
@@ -117,7 +117,7 @@ const component: ClosureComponent = (): Component => {
             e.redraw = false;
             // Don't update any child element
             if (e.target === (el as VnodeDOM).dom) {
-              const now = Date.now();
+              const now = Date.now() + store.getClockSkew();
               const localeString = new Date(timestamp).toLocaleString();
               e.target.title = `${localeString} (${timeAgo(now - timestamp)})`;
             }

ui/components/tags.ts

@@ -29,7 +29,8 @@ const component: ClosureComponent = (): Component => {
     view: (vnode) => {
       const device = vnode.attrs["device"];
       let writable = true;
-      if (vnode.attrs["writable"] != null) writable = vnode.attrs["writable"];
+      if ("writable" in vnode.attrs)
+        writable = !!store.evaluateExpression(vnode.attrs["writable"], device);
 
       const tags = [];
       for (const p of Object.keys(device))

ui/config-functions.ts

@@ -45,7 +45,7 @@ export function flattenConfig(config: Record<string, unknown>): any {
 // Order keys such that nested objects come last
 function orderKeys(config: any): number {
   let res = 1;
-  if (typeof config !== "object") return res;
+  if (config == null || typeof config !== "object") return res;
   if (Array.isArray(config)) {
     for (const c of config) res += orderKeys(c);
     return res;
@@ -79,7 +79,7 @@ export function structureConfig(config: Config[]): any {
     let ref = _config;
     while (keys.length > 1) {
       const k = keys.shift();
-      if (typeof ref[k] !== "object") ref[k] = {};
+      if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
       ref = ref[k];
     }
     ref[keys[0]] = c.value;
@@ -137,7 +137,7 @@ export function diffConfig(
   };
 
   for (const [k, v] of Object.entries(target))
-    if (!current[k] || current[k] !== v) diff.add.push({ _id: k, value: v });
+    if (v && current[k] !== v) diff.add.push({ _id: k, value: v });
 
   for (const k of Object.keys(current)) if (!target[k]) diff.remove.push(k);
 

ui/config-page.ts

@@ -154,7 +154,7 @@ function renderTable(confsResponse, searchString): Children {
               {
                 base: conf,
                 actionHandler: (action, object) => {
-                  return new Promise((resolve) => {
+                  return new Promise<void>((resolve) => {
                     putActionHandler(action, object, false)
                       .then((errors) => {
                         const ErrorList = errors ? Object.values(errors) : [];
@@ -164,11 +164,11 @@ function renderTable(confsResponse, searchString): Children {
                         } else {
                           overlay.close(cb);
                         }
-                        resolve(null);
+                        resolve();
                       })
                       .catch((err) => {
                         notifications.push("error", err.message);
-                        resolve(null);
+                        resolve();
                       });
                   });
                 },
@@ -322,7 +322,7 @@ export const component: ClosureComponent = (): Component => {
                     uiConfigComponent,
                     Object.assign(
                       {
-                        onUpdate: (errs: string[]) => {
+                        onUpdate: (errs: Record<string, string>) => {
                           const errors = errs ? Object.values(errs) : [];
                           if (errors.length) {
                             for (const err of errors)

ui/css/app.css

@@ -87,7 +87,7 @@ i {
 }
 
 body {
-  height: 100%;
+  height: 100vh;
   display: flex;
   flex-direction: column;
 }
@@ -169,6 +169,7 @@ body {
 
         & input {
           width: 100%;
+          box-sizing: border-box;
           padding: 0.3em;
           margin-top: 0.3em;
           margin-bottom: 0.3em;
@@ -291,6 +292,7 @@ body {
     & > img {
       margin: 10px;
       height: 56px;
+      vertical-align: top;
     }
 
     & > .version {
@@ -581,6 +583,7 @@ span.na {
 .all-parameters {
   & > input {
     font-family: monospace;
+    font-size: 14px;
     margin-bottom: 2px;
     padding: 4px;
     width: 700px;
@@ -588,6 +591,10 @@ span.na {
 
   & > .parameter-list {
     font-family: monospace;
+    font-size: 14px;
+    & button {
+      font-size: 16px;
+    }
     overflow-y: scroll;
     height: 200px;
 
@@ -641,7 +648,7 @@ span.na {
 
 span.tag {
   display: inline-block;
-  background-image: url('data:image/svg+xml;utf-8,<svg xmlns="http://www.w3.org/2000/svg" width="190" height="16"><path d="M6.59 15.42l-6-6a2 2 0 0 1 0-2.83l6-6A2 2 0 0 1 8 0h180a2 2 0 0 1 2 2v12a2 2 0 0 1-2 2H8a2 2 0 0 1-1.41-.58zM4.35 6.59A2 2 0 1 0 7.17 9.4 2 2 0 0 0 4.35 6.6z" fill="#e8d5b2"></path></svg>');
+  background-image: url('data:image/svg+xml;utf-8,<svg xmlns="http://www.w3.org/2000/svg" width="190" height="16"><path d="M6.59 15.42l-6-6a2 2 0 0 1 0-2.83l6-6A2 2 0 0 1 8 0h180a2 2 0 0 1 2 2v12a2 2 0 0 1-2 2H8a2 2 0 0 1-1.41-.58zM4.35 6.59A2 2 0 1 0 7.17 9.4 2 2 0 0 0 4.35 6.6z" fill="%23e8d5b2"></path></svg>');
   background-size: cover;
   background-repeat: no-repeat;
   border-radius: 0.2em;
@@ -725,6 +732,7 @@ span.long-text-overflowed {
 
 textarea.long-text {
   font-family: monospace;
+  font-size: 14px;
   min-width: 10vw;
   min-height: 10vh;
 }
@@ -746,8 +754,7 @@ span.parameter-value {
 }
 
 .icon {
-  display: inline-block;
+  display: block;
-  vertical-align: middle;
   width: 1em;
   height: 1em;
   fill: none;
@@ -787,3 +794,19 @@ span.parameter-value {
     background-size: 100% 100%;
   }
 }
+
+.progress {
+  background-color: #eee;
+  overflow: hidden;
+  height: 3px;
+  & > .progress-bar {
+    height: 100%;
+    background-color: #b12d5c;
+  }
+}
+
+/* To keep consistent style after changing from Quirks to Standards mode */
+td,
+th {
+  line-height: normal;
+}

ui/devices-page.ts

@@ -268,16 +268,13 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortedAttrs): void {
-        const _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortedAttrs)) {
+        for (const index of sortedAttrs) {
-          const param = memoizedGetSortable(attributes[index].parameter);
+          const param = memoizedGetSortable(
-          if (param) {
+            attributes[Math.abs(index) - 1].parameter
-            // Changing the priority of columns
+          );
-            delete _sort[param];
+          _sort[param] = Math.sign(index);
-            _sort[param] = direction;
-          }
         }
-
         const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/devices", ops);
@@ -328,10 +325,11 @@ export const component: ClosureComponent = (): Component => {
       if (window.authorizer.hasAccess("devices", 3))
         attrs["actionsCallback"] = renderActions;
 
-      const filterAttrs = {};
+      const filterAttrs = {
-      filterAttrs["resource"] = "devices";
+        resource: "devices",
-      filterAttrs["filter"] = vnode.attrs["filter"];
+        filter: vnode.attrs["filter"],
-      filterAttrs["onChange"] = onFilterChanged;
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing devices"),

ui/drawer-component.ts

@@ -388,7 +388,7 @@ const component: ClosureComponent = (): Component => {
       function repositionNotifications(): void {
         let top = 10;
         for (const c of notificationElements as VnodeDOM[]) {
-          (c.dom as HTMLDivElement).style.top = `${top}`;
+          (c.dom as HTMLDivElement).style.top = `${top}px`;
           top += (c.dom as HTMLDivElement).offsetHeight + 10;
         }
       }
@@ -408,7 +408,7 @@ const component: ClosureComponent = (): Component => {
           for (const c of drawerElement.children)
             height = Math.max(height, c.dom.offsetTop + c.dom.offsetHeight);
         }
-        drawerElement.dom.style.height = height;
+        drawerElement.dom.style.height = height + "px";
       }
 
       if (stagingElements.length + queueElements.length) {

ui/faults-page.ts

@@ -107,13 +107,9 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortAttrs): void {
-        const _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
-          _sort[attributes[index].id] = direction;
-        }
-
         const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/faults", ops);
@@ -200,10 +196,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
+      const filterAttrs = {
-      filterAttrs["resource"] = "faults";
+        resource: "faults",
-      filterAttrs["filter"] = vnode.attrs["filter"];
+        filter: vnode.attrs["filter"],
-      filterAttrs["onChange"] = onFilterChanged;
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing faults"),

ui/files-page.ts

@@ -73,58 +73,32 @@ const unpackSmartQuery = memoize((query) => {
   });
 });
 
-interface ValidationErrors {
+function upload(
-  [prop: string]: string;
+  file: File,
-}
+  headers: Record<string, string>,
-
+  abortSignal?: AbortSignal,
-function putActionHandler(action, _object): Promise<ValidationErrors> {
+  progressListener?: (e: ProgressEvent) => void
-  return new Promise((resolve, reject) => {
+): Promise<void> {
-    const object = Object.assign({}, _object);
+  headers = Object.assign(
-    if (action === "save") {
+    {
-      const file = object["file"] ? object["file"][0] : null;
+      "Content-Type": "application/octet-stream",
-      delete object["file"];
+    },
-
+    headers
-      if (!file) return void resolve({ file: "File not selected" });
+  );
-
+  return store.xhrRequest({
-      const id = file.name;
+    method: "PUT",
-
+    headers: headers,
-      store
+    url: `api/files/${encodeURIComponent(file.name)}`,
-        .resourceExists("files", id)
+    serialize: (body) => body, // Identity function to prevent JSON.parse on blob data
-        .then((exists) => {
+    body: file,
-          if (exists) {
+    config: (xhr) => {
-            store.setTimestamp(Date.now());
+      if (progressListener)
-            return void resolve({ file: "File already exists" });
+        xhr.upload.addEventListener("progress", progressListener);
-          }
+      if (abortSignal) {
-          const headers = Object.assign(
+        if (abortSignal.aborted) xhr.abort();
-            {
+        abortSignal.addEventListener("abort", () => xhr.abort());
-              "Content-Type": "application/octet-stream",
+      }
-              Accept: "application/octet-stream",
+    },
-            },
-            object
-          );
-
-          store
-            .xhrRequest({
-              method: "PUT",
-              headers: headers,
-              url: `api/files/${encodeURIComponent(id)}`,
-              serialize: (body) => body, // Identity function to prevent JSON.parse on blob data
-              body: file,
-            })
-            .then(() => {
-              notifications.push(
-                "success",
-                `File ${exists ? "updated" : "created"}`
-              );
-              store.setTimestamp(Date.now());
-              resolve(null);
-            })
-            .catch(reject);
-        })
-        .catch(reject);
-    } else {
-      reject(new Error("Undefined action"));
-    }
   });
 }
 
@@ -177,13 +151,9 @@ export const component: ClosureComponent = (): Component => {
         sortAttributes[i] = sort[attributes[i].id] || 0;
 
       function onSortChange(sortAttrs): void {
-        const _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
-          _sort[attributes[index].id] = direction;
-        }
-
         const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/files", ops);
@@ -224,42 +194,83 @@ export const component: ClosureComponent = (): Component => {
                 title: "Create new file",
                 onclick: () => {
                   let cb: () => Children = null;
+                  const abortController = new AbortController();
+                  let progress = -1;
                   const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
-                        actionHandler: (action, object) => {
+                        actionHandler: async (action, obj) => {
-                          return new Promise<void>((resolve) => {
+                          if (action !== "save")
-                            putActionHandler(action, object)
+                            throw new Error("Undefined action");
-                              .then((errors) => {
+                          const file = obj["file"]?.[0];
-                                const errorList = errors
+
-                                  ? Object.values(errors)
+                          // nginx strips out headers with dot, so replace with dash
-                                  : [];
+                          const headers = {
-                                if (errorList.length) {
+                            "metadata-fileType": obj["metadata.fileType"] || "",
-                                  for (const err of errorList)
+                            "metadata-oui": obj["metadata.oui"] || "",
-                                    notifications.push("error", err);
+                            "metadata-productclass":
-                                } else {
+                              obj["metadata.productClass"] || "",
-                                  overlay.close(cb);
+                            "metadata-version": obj["metadata.version"] || "",
-                                }
+                          };
-                                resolve();
+
-                              })
+                          if (!file) {
-                              .catch((err) => {
+                            notifications.push("error", "File not selected");
-                                notifications.push("error", err.message);
+                            return;
-                                resolve();
+                          }
-                              });
+
-                          });
+                          if (await store.resourceExists("files", file.name)) {
+                            store.setTimestamp(Date.now());
+                            notifications.push("error", "File already exists");
+                            return;
+                          }
+
+                          const progressListener = (e: ProgressEvent): void => {
+                            progress = e.loaded / e.total;
+                            m.redraw();
+                          };
+
+                          progress = 0;
+                          try {
+                            await upload(
+                              file,
+                              headers,
+                              abortController.signal,
+                              progressListener
+                            );
+                            store.setTimestamp(Date.now());
+                            notifications.push("success", "File created");
+                            overlay.close(cb);
+                          } catch (err) {
+                            notifications.push("error", err.message);
+                          }
+                          progress = -1;
                         },
                       },
                       formData
                     )
                   );
-                  cb = () => comp;
+                  cb = () => {
-                  overlay.open(
+                    if (progress < 0) return [null, comp];
-                    cb,
+                    return [
-                    () =>
+                      m(
-                      !comp.state["current"]["modified"] ||
+                        "div.progress",
-                      confirm("You have unsaved changes. Close anyway?")
+                        m("div.progress-bar", {
-                  );
+                          style: `width: ${Math.trunc(progress * 100)}%`,
+                        })
+                      ),
+                      comp,
+                    ];
+                  };
+                  overlay.open(cb, () => {
+                    if (
+                      comp.state["current"]["modified"] &&
+                      !confirm("You have unsaved changes. Close anyway?")
+                    )
+                      return false;
+                    abortController.abort();
+                    return true;
+                  });
                 },
               },
               "New"
@@ -301,10 +312,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
+      const filterAttrs = {
-      filterAttrs["resource"] = "files";
+        resource: "files",
-      filterAttrs["filter"] = vnode.attrs["filter"];
+        filter: vnode.attrs["filter"],
-      filterAttrs["onChange"] = onFilterChanged;
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing files"),

ui/filter-component.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import m, { ClosureComponent, Component } from "mithril";
+import m, { ClosureComponent } from "mithril";
 import { parse, stringify, map } from "../lib/common/expression-parser";
 import memoize from "../lib/common/memoize";
 import Autocomplete from "./autocomplete-compnent";
@@ -82,9 +82,9 @@ function stringifyFilter(f: Expression): string {
   return stringify(f);
 }
 
-const splitFilter = memoize((filter) => {
+function splitFilter(filter: string): string[] {
   if (!filter) return [""];
-  const list = [];
+  const list: string[] = [];
   const f = parse(filter);
   if (Array.isArray(f) && f[0] === "AND")
     for (const ff of f.slice(1)) list.push(stringifyFilter(ff));
@@ -92,69 +92,82 @@ const splitFilter = memoize((filter) => {
 
   list.push("");
   return list;
-});
+}
+
+interface Attrs {
+  resource: string;
+  filter: string;
+  onChange: (filter: string) => void;
+}
+
+const component: ClosureComponent<Attrs> = (initialVnode) => {
+  let filterList = splitFilter(initialVnode.attrs.filter);
+  let filterInvalid = 0;
+  let filterTouched = false;
+  let attrs: Attrs = initialVnode.attrs;
+
+  function onChange(): void {
+    filterTouched = false;
+    filterInvalid = 0;
+    filterList = filterList.filter((f) => f);
+    const list = filterList.map((f, idx) => {
+      try {
+        return parseFilter(attrs.resource, f);
+      } catch (err) {
+        filterInvalid |= 1 << idx;
+      }
+      return null;
+    });
+    filterList.push("");
+
+    if (filterInvalid) {
+      m.redraw();
+      return;
+    }
+    if (list.length === 0) attrs.onChange("");
+    else if (list.length > 1) attrs.onChange(stringify(["AND", ...list]));
+    else attrs.onChange(stringify(list[0]));
+  }
 
-const component: ClosureComponent = (): Component => {
   return {
+    onupdate: (vnode) => {
+      getAutocomplete(vnode.attrs.resource).reposition();
+    },
     view: (vnode) => {
-      if (
+      if (attrs.filter !== vnode.attrs.filter) {
-        !vnode.state["filterList"] ||
+        filterInvalid = 0;
-        vnode.attrs["filter"] !== vnode.state["filter"]
+        filterList = splitFilter(vnode.attrs.filter);
-      ) {
-        vnode.state["filterInvalid"] = 0;
-        vnode.state["filter"] = vnode.attrs["filter"];
-        vnode.state["filterList"] = splitFilter(vnode.attrs["filter"]);
       }
 
-      function onChange(): void {
+      attrs = vnode.attrs;
-        vnode.state["filterInvalid"] = 0;
-        vnode.state["filterList"] = vnode.state["filterList"].filter((f) => f);
-        let filter = vnode.state["filterList"].map((f, idx) => {
-          try {
-            return parseFilter(vnode.attrs["resource"], f);
-          } catch (err) {
-            vnode.state["filterInvalid"] |= 1 << idx;
-          }
-          return null;
-        });
-        vnode.state["filterList"].push("");
 
-        if (!vnode.state["filterInvalid"]) {
+      return m("div.filter", [
-          delete vnode.state["filter"];
+        m("b", "Filter"),
-          if (filter.length === 0) {
+        ...filterList.map((fltr, idx) => {
-            vnode.attrs["onChange"]("");
+          return m("input", {
-          } else {
+            type: "text",
-            if (filter.length > 1) filter = ["AND"].concat(filter);
+            class: `${(filterInvalid >> idx) & 1 ? "error" : ""}`,
-            else filter = filter[0];
+            value: fltr,
-            vnode.attrs["onChange"](stringify(filter));
+            oninput: (e) => {
-          }
+              e.redraw = false;
-        }
+              filterList[idx] = e.target.value;
-      }
+              filterTouched = true;
+            },
+            oncreate: (vn) => {
+              const el = vn.dom as HTMLInputElement;
+              getAutocomplete(vnode.attrs.resource).attach(el);
 
-      return m(
+              el.addEventListener("blur", () => {
-        "div.filter",
+                if (filterTouched) onChange();
-        [m("b", "Filter")].concat(
+              });
-          vnode.state["filterList"].map((fltr, idx) => {
+
-            return m("input", {
+              el.addEventListener("keydown", (e) => {
-              type: "text",
+                if (e.key === "Enter" && filterTouched) onChange();
-              class: `${
+              });
-                (vnode.state["filterInvalid"] >> idx) & 1 ? "error" : ""
+            },
-              }`,
+          });
-              value: fltr,
+        }),
-              onchange: (e) => {
+      ]);
-                vnode.state["filterList"] = vnode.state["filterList"].slice();
-                vnode.state["filterList"][idx] = e.target.value.trim();
-                onChange();
-              },
-              oncreate: (vn) => {
-                getAutocomplete(vnode.attrs["resource"]).attach(
-                  vn.dom as HTMLInputElement
-                );
-              },
-            });
-          })
-        )
-      );
     },
   };
 };

ui/index-table-component.ts

@@ -20,6 +20,7 @@
 import { ClosureComponent, Component, Children } from "mithril";
 import { m } from "./components";
 import { getIcon } from "./icons";
+import debounce from "../lib/common/debounce";
 
 interface Attribute {
   id?: string;
@@ -36,7 +37,7 @@ function renderTable(
   showMoreCallback: () => void,
   selected: Set<string>,
   sortAttributes: Record<string, any>,
-  onSortChange: (obj: Record<string, number>) => void,
+  onSort: (i: number) => void,
   downloadUrl?: string,
   valueCallback?: (attr: Attribute, record: Record<string, any>) => Children,
   actionsCallback?: Children | ((sel: Set<string>) => Children),
@@ -79,8 +80,6 @@ function renderTable(
       continue;
     }
 
-    let direction = 1;
-
     let symbol = getIcon("unsorted");
     if (sortAttributes[i] > 0) symbol = getIcon("sorted-asc");
     else if (sortAttributes[i] < 0) symbol = getIcon("sorted-dsc");
@@ -88,9 +87,9 @@ function renderTable(
     const sortable = m(
       "button",
       {
-        onclick: () => {
+        onclick: (e) => {
-          if (sortAttributes[i] > 0) direction *= -1;
+          e.redraw = false;
-          return onSortChange({ [i]: direction });
+          onSort(i);
         },
       },
       symbol
@@ -215,7 +214,10 @@ function renderTable(
 
 const component: ClosureComponent = (): Component => {
   let selected = new Set<string>();
-
+  let sortingfunction: (events: number[]) => void;
+  const onSort = debounce((events: number[]) => {
+    sortingfunction(events);
+  }, 500);
   return {
     view: (vnode) => {
       const attributes = vnode.attrs["attributes"];
@@ -234,6 +236,20 @@ const component: ClosureComponent = (): Component => {
         const id = record["_id"] || record["DeviceID.ID"].value[0];
         if (selected.has(id)) _selected.add(id);
       }
+
+      sortingfunction = (events) => {
+        const sortArray = new Set(
+          Object.keys(sortAttributes)
+            .map((x) => (parseInt(x) + 1) * sortAttributes[x])
+            .filter((x) => x)
+        );
+        for (const num of events) {
+          if (sortArray.delete(num + 1)) sortArray.add(-(num + 1));
+          else if (!sortArray.delete((num + 1) * -1)) sortArray.add(num + 1);
+        }
+        onSortChange(Array.from(sortArray).reverse());
+      };
+
       selected = _selected;
 
       return renderTable(
@@ -243,7 +259,7 @@ const component: ClosureComponent = (): Component => {
         showMoreCallback,
         selected,
         sortAttributes,
-        onSortChange,
+        onSort,
         downloadUrl,
         valueCallback,
         actionsCallback,

ui/permissions-page.ts

@@ -205,13 +205,9 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortAttrs): void {
-        const _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
-          _sort[attributes[index].id] = direction;
-        }
-
         const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/permissions", ops);
@@ -365,10 +361,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
+      const filterAttrs = {
-      filterAttrs["resource"] = "permissions";
+        resource: "permissions",
-      filterAttrs["filter"] = vnode.attrs["filter"];
+        filter: vnode.attrs["filter"],
-      filterAttrs["onChange"] = onFilterChanged;
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing permissions"),

ui/presets-page.ts

@@ -190,13 +190,9 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortAttrs): void {
-        const _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
-          _sort[attributes[index].id] = direction;
-        }
-
         const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/presets", ops);
@@ -323,7 +319,7 @@ export const component: ClosureComponent = (): Component => {
                     return m(
                       "div",
                       { style: "margin:20px" },
-                      "This UI only supports presets with a single 'provision' configuraiton. If this preset was originally created from the old UI (genieacs-gui), you must edit it there."
+                      "This UI only supports presets with a single 'provision' configuration. If this preset was originally created from the old UI (genieacs-gui), you must edit it there."
                     );
                   }
                   return comp;
@@ -331,7 +327,7 @@ export const component: ClosureComponent = (): Component => {
                 overlay.open(
                   cb,
                   () =>
-                    !comp.state["current"]["modified"] ||
+                    !comp.state?.["current"]["modified"] ||
                     confirm("You have unsaved changes. Close anyway?")
                 );
               },
@@ -427,10 +423,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
+      const filterAttrs = {
-      filterAttrs["resource"] = "presets";
+        resource: "presets",
-      filterAttrs["filter"] = vnode.attrs["filter"];
+        filter: vnode.attrs["filter"],
-      filterAttrs["onChange"] = onFilterChanged;
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing presets"),

ui/provisions-page.ts

@@ -173,13 +173,9 @@ export const component: ClosureComponent = (): Component => {
         sortAttributes[i] = sort[attributes[i].id] || 0;
 
       function onSortChange(sortAttrs): void {
-        const _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
-          _sort[attributes[index].id] = direction;
-        }
-
         const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/provisions", ops);
@@ -346,10 +342,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
+      const filterAttrs = {
-      filterAttrs["resource"] = "provisions";
+        resource: "provisions",
-      filterAttrs["filter"] = vnode.attrs["filter"];
+        filter: vnode.attrs["filter"],
-      filterAttrs["onChange"] = onFilterChanged;
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing provisions"),

ui/put-form-component.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { VnodeDOM, ClosureComponent, Component, Children } from "mithril";
+import { VnodeDOM, ClosureComponent, Children } from "mithril";
 import { m } from "./components";
 import codeEditorComponent from "./code-editor-component";
 import { getDatalistId } from "./datalist";
@@ -175,13 +175,25 @@ function createField(current, attr, focus): Children {
   });
 }
 
-const component: ClosureComponent = (): Component => {
+interface Attrs {
+  base?: Record<string, any>;
+  actionHandler: (action: string, object: any) => Promise<void>;
+  resource: string;
+  attributes: {
+    id: string;
+    label: string;
+    type?: string;
+    options?: string[];
+  }[];
+}
+
+const component: ClosureComponent<Attrs> = () => {
   return {
     view: (vnode) => {
-      const actionHandler = vnode.attrs["actionHandler"];
+      const actionHandler = vnode.attrs.actionHandler;
-      const attributes = vnode.attrs["attributes"];
+      const attributes = vnode.attrs.attributes;
-      const resource = vnode.attrs["resource"];
+      const resource = vnode.attrs.resource;
-      const base = vnode.attrs["base"] || {};
+      const base = vnode.attrs.base || {};
       if (!vnode.state["current"]) {
         vnode.state["current"] = {
           isNew: !base["_id"],
@@ -226,7 +238,7 @@ const component: ClosureComponent = (): Component => {
               onclick: (e) => {
                 e.redraw = false;
                 e.target.disabled = true;
-                actionHandler("delete", current.object).then(() => {
+                actionHandler("delete", current.object).finally(() => {
                   e.target.disabled = false;
                 });
               },
@@ -255,7 +267,7 @@ const component: ClosureComponent = (): Component => {
               // e.target.onsubmit = null;
               (submit.dom as HTMLFormElement).disabled = true;
               // submit.dom.textContent = "Loading ...";
-              actionHandler("save", current.object).then(() => {
+              actionHandler("save", current.object).finally(() => {
                 // submit.dom.textContent = "Save";
                 // e.target.onsubmit = onsubmit;
                 (submit.dom as HTMLFormElement).disabled = false;

ui/smart-query.ts

@@ -145,7 +145,7 @@ function queryTag(tag: string): Expression {
 
 export function getTip(resource: string, label: string): string {
   let tip;
-  if (resources[resource] && resources[resource][label]) {
+  if (resources[resource]?.[label]) {
     const param = resources[resource][label];
     const types =
       resource === "devices" ? param["type"] : param["type"].split(",");

ui/store.ts

@@ -34,6 +34,8 @@ const memoizedEvaluate = memoize(evaluate);
 let fulfillTimestamp = 0;
 let connectionNotification, configNotification, versionNotification;
 
+let clockSkew = 0;
+
 const queries = {
   filter: new WeakMap(),
   bookmark: new WeakMap(),
@@ -93,11 +95,13 @@ class QueryResponse {
 }
 
 function checkConnection(): void {
+  const now1 = Date.now();
   m.request({
     url: "status",
     method: "GET",
     background: true,
     extract: (xhr) => {
+      const now2 = Date.now();
       if (xhr.status !== 200) {
         if (!connectionNotification) {
           connectionNotification = notifications.push(
@@ -112,6 +116,21 @@ function checkConnection(): void {
           connectionNotification = null;
         }
 
+        try {
+          const nowAvg = Math.trunc((now1 + now2) / 2);
+          const skew = Date.parse(xhr.getResponseHeader("Date")) - nowAvg;
+          if (Math.abs(skew - clockSkew) > 5000 && now2 - now1 < 1000) {
+            clockSkew = skew;
+            console.warn(
+              `System and server clocks are out of sync. Adding ${clockSkew}ms offset to any client-side time relative calculations.`
+            );
+            setTimestamp(now2);
+            m.redraw();
+          }
+        } catch (err) {
+          // Ignore in case of missing or invalid Date header
+        }
+
         const configChanged =
           xhr.getResponseHeader("x-config-snapshot") !== configSnapshot;
         const versionChanged =
@@ -209,7 +228,7 @@ export async function xhrRequest(
 
 export function unpackExpression(exp: Expression): Expression {
   if (!Array.isArray(exp)) return exp;
-  const e = memoizedEvaluate(exp, null, fulfillTimestamp);
+  const e = memoizedEvaluate(exp, null, fulfillTimestamp + clockSkew);
   return e;
 }
 
@@ -295,7 +314,7 @@ function compareFunction(sort: {
 function findMatches(resourceType, filter, sort, limit): any[] {
   let value = [];
   for (const obj of resources[resourceType].objects.values())
-    if (evaluate(filter, obj, fulfillTimestamp)) value.push(obj);
+    if (evaluate(filter, obj, fulfillTimestamp + clockSkew)) value.push(obj);
 
   value = value.sort(compareFunction(sort));
   if (limit) value = value.slice(0, limit);
@@ -461,7 +480,7 @@ export function fulfill(accessTimestamp: number): void {
 
         for (const queryResponse of toFetch) {
           let filter = queries.filter.get(queryResponse);
-          filter = memoizedEvaluate(filter, null, fulfillTimestamp);
+          filter = memoizedEvaluate(filter, null, fulfillTimestamp + clockSkew);
           const bookmark = queries.bookmark.get(queryResponse);
           const sort = queries.sort.get(queryResponse);
           if (bookmark) filter = limitFilter(filter, sort, bookmark);
@@ -476,7 +495,11 @@ export function fulfill(accessTimestamp: number): void {
         if (!diff) {
           for (const queryResponse of toFetch) {
             let filter = queries.filter.get(queryResponse);
-            filter = memoizedEvaluate(filter, null, fulfillTimestamp);
+            filter = memoizedEvaluate(
+              filter,
+              null,
+              fulfillTimestamp + clockSkew
+            );
             const limit = queries.limit.get(queryResponse);
             const bookmark = queries.bookmark.get(queryResponse);
             const sort = queries.sort.get(queryResponse);
@@ -521,7 +544,9 @@ export function fulfill(accessTimestamp: number): void {
 
             for (const d of deleted) {
               const obj = resources[resourceType].objects.get(d);
-              if (evaluate(combinedFilterDiff, obj, fulfillTimestamp))
+              if (
+                evaluate(combinedFilterDiff, obj, fulfillTimestamp + clockSkew)
+              )
                 resources[resourceType].objects.delete(d);
             }
 
@@ -563,6 +588,10 @@ export function setTimestamp(t: number): void {
   }
 }
 
+export function getClockSkew(): number {
+  return clockSkew;
+}
+
 export function postTasks(
   deviceId: string,
   tasks: QueueTask[]
@@ -580,9 +609,8 @@ export function postTasks(
       if (xhr.status === 403) throw new Error("Not authorized");
       if (!xhr.status) throw new Error("Server is unreachable");
       if (xhr.status !== 200) throw new Error(xhr.response);
-      const connectionRequestStatus = xhr.getResponseHeader(
+      const connectionRequestStatus =
-        "Connection-Request"
+        xhr.getResponseHeader("Connection-Request");
-      );
       const st = JSON.parse(xhr.response);
       for (const [i, t] of st.entries()) {
         tasks[i]._id = t._id;
@@ -664,7 +692,7 @@ export function evaluateExpression(
   obj: Record<string, unknown>
 ): Expression {
   if (!Array.isArray(exp)) return exp;
-  return memoizedEvaluate(exp, obj, fulfillTimestamp);
+  return memoizedEvaluate(exp, obj, fulfillTimestamp + clockSkew);
 }
 
 export function changePassword(

ui/ui-config-component.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ClosureComponent, Component } from "mithril";
+import { ClosureComponent } from "mithril";
 import { m } from "./components";
 import * as store from "./store";
 import { yaml } from "./dynamic-loader";
@@ -65,7 +65,7 @@ function putActionHandler(prefix: string[], dataYaml: string): Promise<any> {
               store.putResource(
                 "config",
                 obj._id,
-                (obj as unknown) as Record<string, unknown>
+                obj as unknown as Record<string, unknown>
               )
             );
           }
@@ -86,12 +86,20 @@ function putActionHandler(prefix: string[], dataYaml: string): Promise<any> {
   });
 }
 
-const component: ClosureComponent = (): Component => {
+interface Attrs {
+  prefix: string;
+  name: string;
+  data: { _id: string; value: string }[];
+  onUpdate: (errs: Record<string, string>) => void;
+  onError: (err: Error) => void;
+}
+
+const component: ClosureComponent<Attrs> = () => {
   return {
     view: (vnode) => {
-      const prefix = vnode.attrs["prefix"].split(".");
+      const prefix = vnode.attrs.prefix.split(".");
-      const name = vnode.attrs["name"];
+      const name = vnode.attrs.name;
-      const data = vnode.attrs["data"];
+      const data = vnode.attrs.data;
 
       if (prefix[prefix.length - 1] === "") prefix.pop();
 
@@ -135,8 +143,8 @@ const component: ClosureComponent = (): Component => {
                 vnode.state["updatedYaml"] = yamlString;
 
               putActionHandler(prefix, vnode.state["updatedYaml"])
-                .then(vnode.attrs["onUpdate"])
+                .then(vnode.attrs.onUpdate)
-                .catch(vnode.attrs["onError"]);
+                .catch(vnode.attrs.onError);
             },
           },
           [code, m(".actions-bar", [submit])]

ui/users-page.ts

@@ -186,13 +186,9 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortAttrs): void {
-        const _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
-          _sort[attributes[index].id] = direction;
-        }
-
         const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/users", ops);
@@ -401,10 +397,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
+      const filterAttrs = {
-      filterAttrs["resource"] = "users";
+        resource: "users",
-      filterAttrs["filter"] = vnode.attrs["filter"];
+        filter: vnode.attrs["filter"],
-      filterAttrs["onChange"] = onFilterChanged;
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing users"),

ui/virtual-parameters-page.ts

@@ -173,13 +173,9 @@ export const component: ClosureComponent = (): Component => {
         sortAttributes[i] = sort[attributes[i].id] || 0;
 
       function onSortChange(sortAttrs): void {
-        const _sort = Object.assign({}, sort);
+        const _sort = {};
-        for (const [index, direction] of Object.entries(sortAttrs)) {
+        for (const index of sortAttrs)
-          // Changing the priority of columns
+          _sort[attributes[Math.abs(index) - 1].id] = Math.sign(index);
-          delete _sort[attributes[index].id];
-          _sort[attributes[index].id] = direction;
-        }
-
         const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/virtualParameters", ops);
@@ -346,10 +342,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
+      const filterAttrs = {
-      filterAttrs["resource"] = "virtualParameters";
+        resource: "virtualParameters",
-      filterAttrs["filter"] = vnode.attrs["filter"];
+        filter: vnode.attrs["filter"],
-      filterAttrs["onChange"] = onFilterChanged;
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing virtual parameters"),