1.2.7

Fix issue where declarations against parameters with non-standard value
type trigger an SPV call when the value being declared is a boolean or
a number.

CHANGELOG.md

@@ -1,5 +1,42 @@
 # Change Log
 
+## 1.2.7 (2021-09-18)
+
+- Fix regression causing frequent invalid session errors.
+- Fix regression breaking digest authentication in connection requests.
+
+## 1.2.6 (2021-09-16)
+
+- New config option `cwmp.downloadSuccessOnTimeout` to enable a workaround for
+  CPEs that neglect to send a TransferComplete message after a successful
+  download.
+- Display a progress bar when uploading new files.
+- Default to dual stack interface binding (i.e. `::` instead of `0.0.0.0`).
+  Unless the binding interface is explicitly set, this will cause IPv4 addresses
+  in the logs to be displayed as IPv4-mapped IPv6 addresses (e.g.
+  `::ffff:127.0.0.1` instead of `127.0.0.1`).
+- Detect and correct for client side (browser) clock skew that would otherwise
+  alter the numbers in the pie charts.
+- Various improvements relating to dealing with buggy TR-069 client
+  implementations.
+- Fix a bug causing "lock expired" exceptions when a CWMP session remains open
+  for a very long time due to slow clients.
+- Fix metadata of uploaded files going missing due to nginx stripping away what
+  it considers to be invalid headers. The nginx directive
+  `ignore_invalid_headers` is no longer required.
+- Fix crash when a CPE is assigned a tag containing a dot.
+- Fix bug preventing the user from closing the preset pop-up after being
+  presented with unsupported preset message.
+- Fix exceptions raised from ext scripts manifesting as timeout faults.
+- Fix download getting triggered repeatedly when the value passed to `Download`
+  parameter is greater than the current timestamp.
+- Fix crash when passing invalid attributes to `declare`.
+- Fix crash in NBI when pushing tasks to non-existing devices.
+- Fix crash in NBI when passing invalid JSON to various API endpoints.
+- Fix crash when the output from `ping` command cannot be parsed in some rare
+  edge cases.
+- A number of other fixes and stability improvements.
+
 ## 1.2.5 (2021-03-12)
 
 - Support specifying custom types when uploading files.

README.md

@@ -72,5 +72,5 @@ For commercial support options, please visit
 
 ## License
 
-Copyright 2013-2020 GenieACS Inc. GenieACS is released under the
+Copyright 2013-2021 GenieACS Inc. GenieACS is released under the
 [AGPLv3 license terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

build/build.ts

@@ -29,7 +29,7 @@ import postcss from "postcss";
 import postcssImport from "postcss-import";
 import postcssPresetEnv from "postcss-preset-env";
 import cssnano from "cssnano";
-import { optimize, extendDefaultPlugins } from "svgo";
+import { optimize } from "svgo";
 import * as xmlParser from "../lib/xml-parser";
 
 const MODE = process.env["NODE_ENV"] || "production";
@@ -299,12 +299,16 @@ async function generateIconsSprite(): Promise<void> {
     const id = path.parse(file).name;
     const filePath = path.join(iconsDir, file);
     const { data } = await optimize(fs.readFileSync(filePath).toString(), {
-      plugins: extendDefaultPlugins([
+      plugins: [
         {
-          name: "removeViewBox",
-          active: false,
+          name: "preset-default",
+          params: {
+            overrides: {
+              removeViewBox: false,
+            },
+          },
         },
-      ]),
+      ],
     });
     symbols.push(generateSymbol(id, data));
   }

build/spellcheck-dict.pws

@@ -165,3 +165,6 @@ Brotli
 CSV
 hostInfo
 IPv
+downloadSuccessOnTimeout
+TransferComplete
+nginx

docs/environment-variables.rst

@@ -64,7 +64,7 @@ CWMP_PORT
 CWMP_INTERFACE
   The network interface that genieacs-cwmp binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 CWMP_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
@@ -102,7 +102,7 @@ NBI_PORT
 NBI_INTERFACE
   The network interface that genieacs-nbi binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 NBI_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
@@ -140,7 +140,7 @@ FS_PORT
 FS_INTERFACE
   The network interface that genieacs-fs binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 FS_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
@@ -186,7 +186,7 @@ UI_PORT
 UI_INTERFACE
   The network interface that genieacs-ui binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 UI_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.

docs/installation-guide.rst

@@ -19,7 +19,7 @@ Prerequisites
 
 .. topic:: Node.js
 
-  GenieACS requires Node.js 10.x and up. Refer to https://nodejs.org/ for
+  GenieACS requires Node.js 12.13 and up. Refer to https://nodejs.org/ for
   instructions.
 
 .. topic:: MongoDB

lib/api-functions.ts

@@ -17,7 +17,6 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { parse } from "url";
 import * as db from "./db";
 import * as common from "./common";
 import * as cache from "./cache";
@@ -80,7 +79,7 @@ export async function connectionRequest(
     ).value || [""])[0];
   }
 
-  const remoteAddress = parse(connectionRequestUrl).host;
+  const remoteAddress = new URL(connectionRequestUrl).hostname;
 
   const evalCallback = (exp): Expression => {
     if (!Array.isArray(exp)) return exp;
@@ -95,7 +94,7 @@ export async function connectionRequest(
       else if (name === "password") return password;
 
       const p = device[name];
-      if (p && p.value) return p.value[0];
+      if (p?.value) return p.value[0];
     } else if (exp[0] === "FUNC") {
       if (exp[1] === "REMOTE_ADDRESS") return remoteAddress;
       else if (exp[1] === "USERNAME") return username;
@@ -145,15 +144,24 @@ export async function connectionRequest(
 
   const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
 
-  let udpProm;
+  let udpProm = Promise.resolve(false);
   if (udpConnectionRequestAddress) {
-    udpProm = udpConnectionRequest(
-      udpConnectionRequestAddress,
-      authExp,
-      UDP_CONNECTION_REQUEST_PORT,
-      debug,
-      deviceId
-    );
+    try {
+      const u = new URL("udp://" + udpConnectionRequestAddress);
+      udpProm = udpConnectionRequest(
+        u.hostname,
+        parseInt(u.port || "80"),
+        authExp,
+        UDP_CONNECTION_REQUEST_PORT,
+        debug,
+        deviceId
+      ).then(
+        () => true,
+        () => false
+      );
+    } catch (err) {
+      // Ignore invalid address
+    }
   }
 
   const status = await httpConnectionRequest(
@@ -165,10 +173,7 @@ export async function connectionRequest(
     deviceId
   );
 
-  if (udpProm) {
-    await udpProm;
-    return "";
-  }
+  if (await udpProm) return "";
 
   return status;
 }
@@ -312,7 +317,7 @@ function sanitizeTask(task): void {
 
 export async function insertTasks(tasks: any[]): Promise<Task[]> {
   if (tasks && !Array.isArray(tasks)) tasks = [tasks];
-  else if (!tasks || tasks.length === 0) return tasks || [];
+  else if (!tasks?.length) return tasks || [];
 
   for (const task of tasks) {
     sanitizeTask(task);

lib/auth.ts

@@ -52,9 +52,9 @@ function parseHeaderFeilds(str: string): Record<string, string> {
   return res;
 }
 
-export function parseAuthorizationHeader(
-  authHeader: string
-): { method: string } {
+export function parseAuthorizationHeader(authHeader: string): {
+  method: string;
+} {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };

lib/cache.ts

@@ -59,7 +59,7 @@ export async function set(
 
 export async function pop(key: string): Promise<any> {
   const res = await cacheCollection.findOneAndDelete({ _id: key });
-  if (res && res["value"]) return res["value"]["value"];
+  if (res?.["value"]) return res["value"]["value"];
   return null;
 }
 
@@ -78,9 +78,9 @@ export async function acquireLock(
           value: token,
           expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
         },
-        $currentDate: { timestamp: true },
+        $currentDate: { timestamp: true as unknown as Date },
       },
-      { upsert: true, returnOriginal: false }
+      { upsert: true, returnDocument: "after" }
     );
     const v = r.value;
     if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
@@ -114,5 +114,5 @@ export async function releaseLock(
     _id: lockName,
     value: token,
   });
-  if (res["result"]["n"] !== 1) throw new Error("Lock expired");
+  if (res.deletedCount !== 1) throw new Error("Lock expired");
 }

lib/cluster.ts

@@ -17,14 +17,14 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import cluster from "cluster";
+import cluster, { Worker } from "cluster";
 import { cpus } from "os";
 import * as logger from "./logger";
 
 let respawnTimestamp = 0;
 let crashes: number[] = [];
 
-function fork(): cluster.Worker {
+function fork(): Worker {
   const w = cluster.fork();
   w.on("error", (err: NodeJS.ErrnoException) => {
     // Avoid exception when attempting to kill the process just as it's exiting

lib/common.ts

@@ -51,7 +51,7 @@ export function escapeRegExp(str: string): string {
 export function encodeTag(tag: string): string {
   return encodeURIComponent(tag)
     .replace(
-      /[!~*'()]/g,
+      /[!~*'().]/g,
       (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
     )
     .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")

lib/common/authorizer.ts

@@ -21,13 +21,13 @@ import { PermissionSet, Expression } from "../types";
 import { evaluate, or } from "./expression";
 
 export default class Authorizer {
-  private permissionSets: PermissionSet[];
-  private validatorCache: WeakMap<
+  private declare permissionSets: PermissionSet[];
+  private declare validatorCache: WeakMap<
     any,
     (mutationType, mutation, any) => boolean
   >;
-  private hasAccessCache: Map<string, boolean>;
-  private getFilterCache: Map<string, Expression>;
+  private declare hasAccessCache: Map<string, boolean>;
+  private declare getFilterCache: Map<string, Expression>;
 
   public constructor(permissionSets: PermissionSet[]) {
     this.permissionSets = permissionSets;

lib/common/boolean-expression.ts

@@ -29,8 +29,8 @@ const TWO = BI.BigInt(2);
 const NEGATIVE_ONE = BI.BigInt(-1);
 
 class Indeterminates {
-  public map: Map<string, number>;
-  public sortedKeys: string[];
+  public declare map: Map<string, number>;
+  public declare sortedKeys: string[];
 
   public constructor(str?: string) {
     this.map = new Map();
@@ -118,7 +118,7 @@ function findGcd(a: BI.bigint, b: BI.bigint): BI.bigint {
 }
 
 class Polynomial {
-  public terms: Term[];
+  public declare terms: Term[];
 
   public constructor(terms: Term[]) {
     this.terms = terms;
@@ -298,12 +298,11 @@ class Polynomial {
 }
 
 type Minterm = number[];
-type Context = Map<string, number>;
 
 abstract class BoolExprSynth {
-  public abstract true(context: Context): Minterm[];
-  public abstract false(context: Context): Minterm[];
-  public abstract null(context: Context): Minterm[];
+  public abstract true(context: SynthContext): Minterm[];
+  public abstract false(context: SynthContext): Minterm[];
+  public abstract null(context: SynthContext): Minterm[];
 }
 
 class TrueSynth extends BoolExprSynth {
@@ -343,8 +342,8 @@ class NullSynth extends BoolExprSynth {
 }
 
 class VarSynth extends BoolExprSynth {
-  private expStr: string;
-  private negate: boolean;
+  private declare exp: Expression;
+  private declare negate: boolean;
   public constructor(exp: Expression) {
     super();
     this.negate = false;
@@ -368,52 +367,49 @@ class VarSynth extends BoolExprSynth {
         this.negate = true;
       }
     }
-    this.expStr = JSON.stringify(exp);
+    this.exp = exp;
   }
-  public true(context: Context): Minterm[] {
-    let v = context.get(this.expStr);
-    if (v == null) context.set(this.expStr, (v = context.size));
+  public true(context: SynthContext): Minterm[] {
+    const v = context.getVariable(this.exp);
     return [[(v << 2) ^ (this.negate ? 1 : 3)]];
   }
-  public false(context: Context): Minterm[] {
-    let v = context.get(this.expStr);
-    if (v == null) context.set(this.expStr, (v = context.size));
+  public false(context: SynthContext): Minterm[] {
+    const v = context.getVariable(this.exp);
     return [[(v << 2) ^ (this.negate ? 3 : 1)]];
   }
-  public null(context: Context): Minterm[] {
-    let v = context.get(this.expStr);
-    if (v == null) context.set(this.expStr, (v = context.size));
+  public null(context: SynthContext): Minterm[] {
+    const v = context.getVariable(this.exp);
     return [[v << 2, (v << 2) ^ 2]];
   }
 }
 
 class NotSynth extends BoolExprSynth {
-  private exprSynth: BoolExprSynth;
+  private declare exprSynth: BoolExprSynth;
   public constructor(e: BoolExprSynth) {
     super();
     this.exprSynth = e;
   }
-  public true(context: Context): Minterm[] {
+  public true(context: SynthContext): Minterm[] {
     return this.exprSynth.false(context);
   }
-  public false(context: Context): Minterm[] {
+  public false(context: SynthContext): Minterm[] {
     return this.exprSynth.true(context);
   }
-  public null(context: Context): Minterm[] {
+  public null(context: SynthContext): Minterm[] {
     return this.exprSynth.null(context);
   }
 }
 
 class IsNullSynth extends BoolExprSynth {
-  private exprSynth: BoolExprSynth;
+  private declare exprSynth: BoolExprSynth;
   public constructor(e: BoolExprSynth) {
     super();
     this.exprSynth = e;
   }
-  public true(context: Context): Minterm[] {
+  public true(context: SynthContext): Minterm[] {
     return this.exprSynth.null(context);
   }
-  public false(context: Context): Minterm[] {
+  public false(context: SynthContext): Minterm[] {
     return [...this.exprSynth.true(context), ...this.exprSynth.false(context)];
   }
   public null(): Minterm[] {
@@ -422,7 +418,7 @@ class IsNullSynth extends BoolExprSynth {
 }
 
 class OrSynth extends BoolExprSynth {
-  private exprSynths: BoolExprSynth[];
+  private declare exprSynths: BoolExprSynth[];
   public constructor(...e: BoolExprSynth[]) {
     super();
     this.exprSynths = e.filter((ee) => !(ee instanceof FalseSynth));
@@ -436,13 +432,13 @@ class OrSynth extends BoolExprSynth {
     });
     this.exprSynths.push(...unpacked);
   }
-  public true(context: Context): Minterm[] {
+  public true(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].true(context);
     if (this.exprSynths.some((e) => e instanceof TrueSynth)) return [[]];
     return this.exprSynths.map((e) => e.true(context)).flat();
   }
-  public false(context: Context): Minterm[] {
+  public false(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [[]];
     if (this.exprSynths.length === 1) return this.exprSynths[0].false(context);
     if (
@@ -455,7 +451,7 @@ class OrSynth extends BoolExprSynth {
       this.exprSynths.map((e) => complement(e.false(context))).flat()
     );
   }
-  public null(context: Context): Minterm[] {
+  public null(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].null(context);
     const n = this.exprSynths.map((e) => e.null(context)).flat();
@@ -465,7 +461,7 @@ class OrSynth extends BoolExprSynth {
 }
 
 class AndSynth extends BoolExprSynth {
-  private exprSynths: BoolExprSynth[];
+  private declare exprSynths: BoolExprSynth[];
   public constructor(...e: BoolExprSynth[]) {
     super();
     this.exprSynths = e.filter((ee) => !(ee instanceof TrueSynth));
@@ -479,7 +475,7 @@ class AndSynth extends BoolExprSynth {
     });
     this.exprSynths.push(...unpacked);
   }
-  public true(context: Context): Minterm[] {
+  public true(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [[]];
     if (this.exprSynths.length === 1) return this.exprSynths[0].true(context);
     if (
@@ -492,13 +488,13 @@ class AndSynth extends BoolExprSynth {
       this.exprSynths.map((e) => complement(e.true(context))).flat()
     );
   }
-  public false(context: Context): Minterm[] {
+  public false(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].false(context);
     if (this.exprSynths.some((e) => e instanceof FalseSynth)) return [[]];
     return this.exprSynths.map((e) => e.false(context)).flat();
   }
-  public null(context: Context): Minterm[] {
+  public null(context: SynthContext): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].null(context);
     const n = this.exprSynths.map((e) => e.null(context)).flat();
@@ -508,12 +504,12 @@ class AndSynth extends BoolExprSynth {
 }
 
 class CaseSynth extends BoolExprSynth {
-  private exprSynths: BoolExprSynth[];
+  private declare exprSynths: BoolExprSynth[];
   public constructor(e: BoolExprSynth[]) {
     super();
     this.exprSynths = e;
   }
-  public true(context: Context): Minterm[] {
+  public true(context: SynthContext): Minterm[] {
     const minterms: Minterm[] = [];
     const cumulative: Minterm[] = [];
     for (let i = 0; i < this.exprSynths.length; i += 2) {
@@ -533,7 +529,7 @@ class CaseSynth extends BoolExprSynth {
     }
     return minterms;
   }
-  public false(context: Context): Minterm[] {
+  public false(context: SynthContext): Minterm[] {
     const minterms: Minterm[] = [];
     const cumulative: Minterm[] = [];
     for (let i = 0; i < this.exprSynths.length; i += 2) {
@@ -553,7 +549,7 @@ class CaseSynth extends BoolExprSynth {
     }
     return minterms;
   }
-  public null(context: Context): Minterm[] {
+  public null(context: SynthContext): Minterm[] {
     const minterms: Minterm[] = [];
     const cumulative: Minterm[] = [];
     for (let i = 0; i < this.exprSynths.length; i += 2) {
@@ -666,10 +662,10 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return (args.reduce(
+    return args.reduce(
       (previousValue, currentValue) => previousValue.add(currentValue),
       ADDITIVE_IDENTITY
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (op === "*") {
     const args: Polynomial[] = [];
     for (let i = 1; i < exp.length; ++i) {
@@ -677,10 +673,10 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return (args.reduce(
+    return args.reduce(
       (previousValue, currentValue) => previousValue.multiply(currentValue),
       MULTIPLICATIVE_IDENTITY
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (op === "-") {
     const args: Polynomial[] = [];
     for (let i = 1; i < exp.length; ++i) {
@@ -688,9 +684,9 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return (args.reduce((previousValue, currentValue) =>
+    return args.reduce((previousValue, currentValue) =>
       previousValue.subtract(currentValue)
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (op === "/") {
     const args: Polynomial[] = [];
     for (let i = 1; i < exp.length; ++i) {
@@ -698,9 +694,9 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return (args.reduce((previousValue, currentValue) =>
+    return args.reduce((previousValue, currentValue) =>
       previousValue.divide(currentValue)
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (["=", "<>", ">", ">=", "<", "<="].includes(op)) {
     if (exp[1] == null || exp[2] == null) return null;
     let lhs: Polynomial, rhs: Polynomial;
@@ -798,17 +794,14 @@ function toBoolExprSynth(e: Expression | BoolExprSynth): BoolExprSynth {
   return new FalseSynth();
 }
 
-function sopToExpression(
-  sop: number[][],
-  expressions: Map<number, Expression>
-): Expression {
+function sopToExpression(sop: number[][], context: SynthContext): Expression {
   if (!sop.length) return false;
   const res: Expression[] = [];
   for (const s of sop) {
     if (!s.length) return true;
     const conjs: Expression[] = [];
     for (const i of s) {
-      let expr = expressions.get(i >>> 2);
+      let expr = context.getExpression(i >>> 2);
       if (!(i & 1) !== !(i & 2)) expr = ["NOT", expr];
       if (Array.isArray(expr) && expr[0] === "NOT" && Array.isArray(expr[1])) {
         const e: Expression[] = expr[1];
@@ -832,163 +825,196 @@ function sopToExpression(
   return res[0];
 }
 
-function generateDcSetAndIsNull(
-  variables: Map<number, Expression>
-): { dcSet: number[][]; isNull: Map<number, number> } {
-  const relations: Map<
+function findIsNullDeps(exp: Expression): Expression[] {
+  if (!Array.isArray(exp)) return [];
+  const op = exp[0];
+  if (op === "IS NULL" || op === "IS NOT NULL") return [];
+
+  if (op === "FUNC") {
+    if (exp[1] === "NOW") {
+      return [];
+    } else if (exp[1] === "LOWER" || exp[1] === "UPPER") {
+      return findIsNullDeps(exp[2]);
+    } else if (exp[1] === "ROUND") {
+      return exp
+        .slice(2, 4)
+        .map((e) => findIsNullDeps(e))
+        .flat();
+    }
+  } else if (op !== "PARAM") {
+    return exp
+      .slice(1)
+      .map((e) => findIsNullDeps(e))
+      .flat();
+  }
+  return [exp];
+}
+
+class SynthContext {
+  private declare _dcSet: number[][];
+  private declare _variables: Map<string, number>;
+  private declare _expressions: Map<number, Expression>;
+  private declare _isNullRelations: Map<number, number[]>;
+  private declare _isNullVars: Set<number>;
+  private declare _comparisons: Map<
     string,
-    { op: string; rhs: boolean | number | string; var: number }[]
-  > = new Map();
+    Map<string, { eq: number; gt: number; lt: number }>
+  >;
 
-  for (const [n, e] of variables) {
-    const op = e[0] as string;
-    if (["=", ">", "<"].includes(op) && !Array.isArray(e[2])) {
-      const lhs = JSON.stringify(e[1]);
-      let r = relations.get(lhs);
-      if (!r) relations.set(lhs, (r = []));
-      r.push({ op: op, rhs: e[2], var: n });
-    } else if (op === "LIKE" && !Array.isArray(e[2]) && !Array.isArray(e[3])) {
-      const lhs = JSON.stringify(e[1]);
-      let r = relations.get(lhs);
-      if (!r) relations.set(lhs, (r = []));
-      r.push({ op: "", rhs: null, var: n });
-    } else {
-      const lhs = JSON.stringify(e);
-      let r = relations.get(lhs);
-      if (!r) relations.set(lhs, (r = []));
-      r.push({ op: "", rhs: null, var: n });
+  constructor() {
+    this._dcSet = [];
+    this._variables = new Map();
+    this._expressions = new Map();
+    this._isNullRelations = new Map();
+    this._isNullVars = new Set();
+    this._comparisons = new Map();
+  }
+
+  getVariable(exp: Expression): number {
+    const expStr = JSON.stringify(exp);
+    if (this._variables.has(expStr)) return this._variables.get(expStr);
+    const v = this._variables.size;
+    this._variables.set(expStr, v);
+    this._expressions.set(v, exp);
+    this.generateIsNullDc(v);
+    this.generateComparisionDc(v);
+
+    if (
+      expStr === '["IS NULL",["PARAM","DeviceID.ID"]]' ||
+      expStr === '["IS NULL",["PARAM","_id"]]'
+    )
+      this._dcSet.push([(v << 2) ^ 3]);
+
+    return v;
+  }
+
+  private generateIsNullDc(v: number): void {
+    this._isNullRelations.set(v, []);
+    const exp = this._expressions.get(v);
+
+    const isNull = new Set(
+      findIsNullDeps(exp).map((e) => this.getVariable(["IS NULL", e]))
+    );
+
+    if (!isNull.size) return;
+
+    for (const n of isNull) {
+      this._isNullVars.add(n);
+      this._isNullRelations.get(n).push(v);
+      this._isNullRelations.get(v).push(n);
+      this._dcSet.push([(n << 2) ^ 3, (v << 2) ^ 1]);
+      this._dcSet.push([(n << 2) ^ 3, (v << 2) ^ 3]);
+    }
+    this._dcSet.push(
+      [...isNull].map((n) => (n << 2) ^ 2).concat([(v << 2) ^ 0, (v << 2) ^ 2])
+    );
+  }
+
+  private generateComparisionDc(v: number): void {
+    const exp = this._expressions.get(v);
+    if (!Array.isArray(exp)) return;
+    const op = exp[0];
+    if (![">", "<", "="].includes(op)) return;
+    const rhs = exp[2];
+    const rhsStr = JSON.stringify(rhs);
+    const lhsStr = JSON.stringify(exp[1]);
+    let allComps = this._comparisons.get(lhsStr);
+    if (!allComps) this._comparisons.set(lhsStr, (allComps = new Map()));
+    if (allComps.has(rhsStr)) return;
+
+    const comp = {
+      eq: -1,
+      gt: -1,
+      lt: -1,
+    };
+
+    allComps.set(rhsStr, comp);
+
+    comp.eq = this.getVariable(["=", exp[1], exp[2]]);
+    comp.gt = this.getVariable([">", exp[1], exp[2]]);
+    comp.lt = this.getVariable(["<", exp[1], exp[2]]);
+
+    this._dcSet.push([
+      (comp.eq << 2) ^ 1,
+      (comp.gt << 2) ^ 1,
+      (comp.lt << 2) ^ 1,
+    ]);
+    this._dcSet.push([(comp.eq << 2) ^ 3, (comp.gt << 2) ^ 3]);
+    this._dcSet.push([(comp.eq << 2) ^ 3, (comp.lt << 2) ^ 3]);
+    this._dcSet.push([(comp.gt << 2) ^ 3, (comp.lt << 2) ^ 3]);
+
+    const type1 = typeof exp[2];
+    if (!["boolean", "number", "string"].includes(type1)) return;
+
+    for (const [rhs2Str, comp2] of allComps) {
+      if (comp2 === comp) continue;
+      const rhs2 = JSON.parse(rhs2Str);
+      const type2 = typeof rhs2;
+
+      if (!["boolean", "number", "string"].includes(type2)) continue;
+      let cmp = 0;
+      if (type1 === type2) cmp = rhs > rhs2 ? 1 : -1;
+      else if (type1 === "string") cmp = 1;
+      else if (type2 === "string") cmp = -1;
+      else cmp = +rhs - +rhs2;
+
+      const c1 = cmp > 0 ? comp : comp2;
+      const c2 = cmp > 0 ? comp2 : comp;
+
+      // This is the minimum clauses required if all relavent vars
+      // were included in the DC set.
+      // this._dcSet.push([(c1.eq << 2) ^ 3, (c2.eq << 2) ^ 3]);
+      // this._dcSet.push([(c1.lt << 2) ^ 1, (c2.gt << 2) ^ 1]);
+
+      // But we use non-minimal set because intermediate vars
+      // between any two may not be present in the DC set.
+      this._dcSet.push([(c1.lt << 2) ^ 1, (c2.gt << 2) ^ 1]);
+      this._dcSet.push([(c1.lt << 2) ^ 1, (c2.eq << 2) ^ 3]);
+      this._dcSet.push([(c1.lt << 2) ^ 1, (c2.lt << 2) ^ 3]);
+      this._dcSet.push([(c1.eq << 2) ^ 3, (c2.gt << 2) ^ 1]);
+      this._dcSet.push([(c1.eq << 2) ^ 3, (c2.eq << 2) ^ 3]);
+      this._dcSet.push([(c1.eq << 2) ^ 3, (c2.lt << 2) ^ 3]);
+      this._dcSet.push([(c1.gt << 2) ^ 3, (c2.gt << 2) ^ 1]);
+      this._dcSet.push([(c1.gt << 2) ^ 3, (c2.eq << 2) ^ 3]);
+      this._dcSet.push([(c1.gt << 2) ^ 3, (c2.lt << 2) ^ 3]);
     }
   }
 
-  const dcSet: number[][] = [];
-  const isNull: Map<number, number> = new Map();
-
-  for (const [key, rels] of relations) {
-    const n = variables.size;
-    variables.set(n, ["IS NULL", JSON.parse(key)]);
-
-    if (key === '["PARAM","DeviceID.ID"]' || key === '["PARAM","_id"]')
-      dcSet.push([(n << 2) ^ 3]);
-
-    const existingOps = new Map() as Map<string, Map<string, number>>;
-    for (const rel of rels) {
-      if (!rel.op) continue;
-      const rhsStr = JSON.stringify(rel.rhs);
-      const s = existingOps.get(rhsStr);
-      if (s) s.set(rel.op, rel.var);
-      else existingOps.set(rhsStr, new Map([[rel.op, rel.var]]));
-    }
-
-    for (const [k, v] of existingOps) {
-      if (v.size > 1) {
-        if (!v.has("=")) {
-          const va = variables.size;
-          variables.set(va, ["=", JSON.parse(key), JSON.parse(k)]);
-          rels.push({ op: "=", rhs: JSON.parse(k), var: va });
-          v.set("=", va);
-        }
-
-        if (!v.has(">")) {
-          const va = variables.size;
-          variables.set(va, [">", JSON.parse(key), JSON.parse(k)]);
-          rels.push({ op: ">", rhs: JSON.parse(k), var: va });
-          v.set(">", va);
-        }
-
-        if (!v.has("<")) {
-          const va = variables.size;
-          variables.set(va, ["<", JSON.parse(key), JSON.parse(k)]);
-          rels.push({ op: "<", rhs: JSON.parse(k), var: va });
-          v.set("<", va);
-        }
-
-        dcSet.push([
-          (v.get("=") << 2) ^ 1,
-          (v.get(">") << 2) ^ 1,
-          (v.get("<") << 2) ^ 1,
-        ]);
-      }
-    }
-
-    for (let i = 0; i < rels.length; ++i) {
-      const rel1 = rels[i];
-      isNull.set(rel1.var, n);
-
-      dcSet.push([(n << 2) ^ 3, (rel1.var << 2) ^ 1]);
-      dcSet.push([(n << 2) ^ 3, (rel1.var << 2) ^ 3]);
-      dcSet.push([(rel1.var << 2) ^ 0, (rel1.var << 2) ^ 2, (n << 2) ^ 2]);
-
-      if (!rel1.op) continue;
-      for (let j = i + 1; j < rels.length; ++j) {
-        const rel2 = rels[j];
-        if (!rel2.op) continue;
-        let comp = 0;
-        if (Array.isArray(rel1.rhs) || Array.isArray(rel2.rhs)) {
-          if (JSON.stringify(rel1.rhs) !== JSON.stringify(rel2.rhs)) continue;
-        } else if (typeof rel1.rhs === "string") {
-          if (typeof rel2.rhs === "string") {
-            if (rel1.rhs > rel2.rhs) comp = 1;
-            else if (rel1.rhs < rel2.rhs) comp = -1;
-          } else {
-            comp = 1;
-          }
-        } else if (typeof rel2.rhs === "string") {
-          comp = -1;
-        } else {
-          comp = +rel1.rhs - +rel2.rhs;
-        }
-
-        if (rel1.op === rel2.op) {
-          if (comp === 0)
-            dcSet.push([(rel1.var << 2) ^ 1, (rel2.var << 2) ^ 1]);
-          else if (rel1.op === "=")
-            dcSet.push([(rel1.var << 2) ^ 3, (rel2.var << 2) ^ 3]);
-          else if (
-            (rel1.op === ">" && comp > 0) ||
-            (rel1.op === "<" && comp < 0)
-          )
-            dcSet.push([(rel1.var << 2) ^ 3, (rel2.var << 2) ^ 1]);
-          else if (
-            (rel1.op === "<" && comp > 0) ||
-            (rel1.op === ">" && comp < 0)
-          )
-            dcSet.push([(rel1.var << 2) ^ 1, (rel2.var << 2) ^ 3]);
-        } else if (rel1.op === "=" || rel2.op === "=") {
-          let r1 = rel1,
-            r2 = rel2,
-            cmp = comp;
-          if (rel1.op !== "=") {
-            r2 = rel1;
-            r1 = rel2;
-            cmp = -comp;
-          }
-          if (r2.op === ">" || r2.op === "<") {
-            if (cmp === 0) {
-              dcSet.push([(r1.var << 2) ^ 3, (r2.var << 2) ^ 3]);
-            } else {
-              if ((cmp > 0 && r2.op === ">") || (cmp < 0 && r2.op === "<"))
-                dcSet.push([(r1.var << 2) ^ 3, (r2.var << 2) ^ 1]);
-              else dcSet.push([(r1.var << 2) ^ 3, (r2.var << 2) ^ 3]);
-            }
-          }
-        } else if (
-          [">", "<"].includes(rel1.op) &&
-          [">", "<"].includes(rel2.op)
-        ) {
-          if ((rel1.op === ">" && comp < 0) || (rel1.op === "<" && comp > 0))
-            dcSet.push([(rel1.var << 2) ^ 1, (rel2.var << 2) ^ 1]);
-          else dcSet.push([(rel1.var << 2) ^ 3, (rel2.var << 2) ^ 3]);
-        }
-      }
-    }
+  getExpression(v: number): Expression {
+    return this._expressions.get(v);
   }
 
-  return { dcSet, isNull };
+  getIsNullRelations(v: number): number[] {
+    return this._isNullRelations.get(v);
+  }
+
+  isIsNull(v: number): boolean {
+    return this._isNullVars.has(v);
+  }
+
+  getDcSet(minterms?: Minterm[]): number[][] {
+    if (!minterms) return this._dcSet;
+    const vars = new Set(minterms.flat().map((v) => v >> 2));
+
+    for (const comps of this._comparisons.values()) {
+      for (const comp of comps.values()) {
+        const c = Object.values(comp).filter((v) => !vars.has(v));
+        if (c.length === 1) vars.add(c[0]);
+      }
+    }
+
+    for (const n of this._isNullVars)
+      if (this._isNullRelations.get(n).some((v) => vars.has(v))) vars.add(n);
+
+    const dcSet = this._dcSet.filter((m) => m.every((v) => vars.has(v >> 2)));
+
+    return dcSet;
+  }
 }
 
 function sanitizeMinterms(
   minterms: number[][],
-  isNull: Map<number, number>
+  context: SynthContext
 ): number[][] {
   const res = [] as number[][];
   loop: for (const m of minterms) {
@@ -999,13 +1025,20 @@ function sanitizeMinterms(
     const perms: number[][] = [];
     for (const [k, v] of merged) {
       if (v === 0b1010) continue loop;
-      const n = isNull.get(k) << 2;
+      const isNullVars = context.getIsNullRelations(k);
       const t = k << 2;
-      if (v === 0b0101) minterm.push(n ^ 3);
-      else if (v === 0b0001) perms.push([n ^ 3, t ^ 3]);
-      else if (v === 0b0100) perms.push([n ^ 3, t ^ 1]);
-      else if (v & 0b1000) minterm.push(t ^ 3);
-      else if (v & 0b0010) minterm.push(t ^ 1);
+      if (v === 0b0101) {
+        if (isNullVars.length === 1) minterm.push((isNullVars[0] << 2) ^ 3);
+        else perms.push(isNullVars.map((n) => (n << 2) ^ 3));
+      } else if (v === 0b0001) {
+        perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 3]);
+      } else if (v === 0b0100) {
+        perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 1]);
+      } else if (v & 0b1000) {
+        minterm.push(t ^ 3);
+      } else if (v & 0b0010) {
+        minterm.push(t ^ 1);
+      }
     }
     let ms = [minterm];
     while (perms.length) {
@@ -1020,17 +1053,13 @@ function sanitizeMinterms(
 }
 
 function boolExprSynthToExpression(boolExpr: BoolExprSynth): Expression {
-  const context: Map<string, number> = new Map();
+  const context = new SynthContext();
   let minterms = boolExpr.true(context);
-  const variables = new Map(
-    Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
-  );
-  const { dcSet, isNull } = generateDcSetAndIsNull(variables);
-  minterms = sanitizeMinterms(minterms, isNull);
-
-  const canRaise = getCanRaiseCallback(isNull);
+  minterms = sanitizeMinterms(minterms, context);
+  const canRaise = getCanRaiseCallback(context);
+  const dcSet = context.getDcSet(minterms);
   minterms = espresso(minterms, dcSet, { canRaise });
-  return sopToExpression(minterms, variables);
+  return sopToExpression(minterms, context);
 }
 
 function mapCallback(exp: Expression | BoolExprSynth): Expression {
@@ -1043,22 +1072,22 @@ function mapCallback(exp: Expression | BoolExprSynth): Expression {
 
   const op = exp[0];
   if (op === "IS NULL")
-    return (new IsNullSynth(toBoolExprSynth(exp[1])) as unknown) as Expression;
+    return new IsNullSynth(toBoolExprSynth(exp[1])) as unknown as Expression;
 
   if (op === "IS NOT NULL") {
-    return (new NotSynth(
+    return new NotSynth(
       new IsNullSynth(toBoolExprSynth(exp[1]))
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (op === "NOT") {
-    return (new NotSynth(toBoolExprSynth(exp[1])) as unknown) as Expression;
+    return new NotSynth(toBoolExprSynth(exp[1])) as unknown as Expression;
   } else if (op === "OR") {
-    return (new OrSynth(
+    return new OrSynth(
       ...exp.slice(1).map((a) => toBoolExprSynth(a))
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   } else if (op === "AND") {
-    return (new AndSynth(
+    return new AndSynth(
       ...exp.slice(1).map((a) => toBoolExprSynth(a))
-    ) as unknown) as Expression;
+    ) as unknown as Expression;
   }
   for (let i = 1; i < exp.length; ++i) {
     if (exp[i] instanceof BoolExprSynth) {
@@ -1074,19 +1103,20 @@ function mapCallback(exp: Expression | BoolExprSynth): Expression {
 }
 
 function getCanRaiseCallback(
-  isNull: Map<number, number>
+  context: SynthContext
 ): (idx: number, set: Set<number>) => boolean {
   return (idx: number, set: Set<number>): boolean => {
     const i = idx >> 2;
-    const n = isNull.get(i);
-    if (n != null) {
+    const vars = context.getIsNullRelations(i);
+    if (!context.isIsNull(i)) {
       if (!(idx & 1)) return true;
-      if (set.has((n << 2) ^ 2)) return true;
-      if (set.has((n << 2) ^ 3)) return true;
-      return !set.has(idx ^ 3);
+      if (!set.has(idx ^ 3)) return true;
+      for (const n of vars) if (set.has((n << 2) ^ 3)) return true;
+      for (const n of vars) if (!set.has((n << 2) ^ 2)) return false;
+      return true;
     }
-    for (const [k, v] of isNull) {
-      if (v !== i) continue;
+
+    for (const k of vars) {
       if (set.has((k << 2) ^ 1)) continue;
       if (set.has((k << 2) ^ 3)) continue;
       if (set.has((k << 2) ^ 0)) return false;
@@ -1101,33 +1131,31 @@ export function minimize(expr: Expression, boolean = false): Expression {
   expr = map(expr, mapCallback);
   if (Array.isArray(expr) && expr[0] === "CASE") {
     if (!boolean) {
-      const context: Map<string, number> = new Map();
+      const context = new SynthContext();
       const whens = expr.filter((e, i) => i % 2).map((e) => e.true(context));
-      const variables = new Map(
-        Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
-      );
-      const { dcSet, isNull } = generateDcSetAndIsNull(variables);
-      const canRaise = getCanRaiseCallback(isNull);
+      const caseDcSet: Minterm[] = [];
+      const canRaise = getCanRaiseCallback(context);
       const res: Expression = ["CASE"];
       for (let i = 1; i < expr.length; i += 2) {
-        let minterms = sanitizeMinterms(whens[(i - 1) / 2], isNull);
+        let minterms = sanitizeMinterms(whens[(i - 1) / 2], context);
+        const dcSet = context.getDcSet(minterms.concat(caseDcSet));
         minterms = espresso(minterms, dcSet, { canRaise });
         if (!minterms.length) continue;
-        const w = sopToExpression(minterms, variables);
+        const w = sopToExpression(minterms, context);
         let t = expr[i + 1];
         if (t instanceof BoolExprSynth) t = boolExprSynthToExpression(t);
         res.push(w, t);
         if (w === true) break;
-        dcSet.push(...minterms);
+        caseDcSet.push(...minterms);
       }
       while (res[res.length - 1] == null) res.splice(-2);
       if (res.length < 3) return null;
       return res;
     }
-    expr = (toBoolExprSynth(expr) as unknown) as Expression;
+    expr = toBoolExprSynth(expr) as unknown as Expression;
   }
 
-  if (boolean) expr = (toBoolExprSynth(expr) as unknown) as Expression;
+  if (boolean) expr = toBoolExprSynth(expr) as unknown as Expression;
 
   if (expr instanceof BoolExprSynth) expr = boolExprSynthToExpression(expr);
   return expr;
@@ -1145,7 +1173,7 @@ export function unionDiff(
 
   if (!expr1) {
     if (Array.isArray(expr2) && expr2[0] === "CASE")
-      expr2 = (toBoolExprSynth(expr2) as unknown) as Expression;
+      expr2 = toBoolExprSynth(expr2) as unknown as Expression;
     if (expr2 instanceof BoolExprSynth)
       expr2 = boolExprSynthToExpression(expr2);
     return [expr2, expr2];
@@ -1156,35 +1184,37 @@ export function unionDiff(
   expr1 = map(expr1, mapCallback);
   const b1 = toBoolExprSynth(expr1);
 
-  const context: Map<string, number> = new Map();
+  const context = new SynthContext();
 
   const expr2Minterms = b2.true(context);
   const expr1Minterms = b1.true(context);
   const expr1NullMinterms = b1.null(context);
   const expr1FalseMinterms = b1.false(context);
-  const variables = new Map(
-    Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
-  );
-  const { dcSet, isNull } = generateDcSetAndIsNull(variables);
 
   const unionMinterms = sanitizeMinterms(
     [...expr1Minterms, ...expr2Minterms],
-    isNull
+    context
   );
+
   const diffMinterms = sanitizeMinterms(
     complement([
       ...complement([...expr1NullMinterms, ...expr1FalseMinterms]),
       ...complement(expr2Minterms),
     ]),
-    isNull
+    context
   );
 
-  const canRaise = getCanRaiseCallback(isNull);
+  const canRaise = getCanRaiseCallback(context);
 
-  const union = espresso(unionMinterms, dcSet, { canRaise });
-  const diff = espresso(diffMinterms, dcSet, { canRaise });
+  const union = espresso(unionMinterms, context.getDcSet(unionMinterms), {
+    canRaise,
+  });
 
-  return [sopToExpression(union, variables), sopToExpression(diff, variables)];
+  const diff = espresso(diffMinterms, context.getDcSet(diffMinterms), {
+    canRaise,
+  });
+
+  return [sopToExpression(union, context), sopToExpression(diff, context)];
 }
 
 export function covers(expr1: Expression, expr2: Expression): boolean {
@@ -1198,13 +1228,13 @@ export function covers(expr1: Expression, expr2: Expression): boolean {
   expr2 = map(expr2, mapCallback);
   const b2 = toBoolExprSynth(expr2);
 
-  const context: Map<string, number> = new Map();
+  const context = new SynthContext();
   const expr1Minterms = b1.true(context);
   const expr2Minterms = b2.true(context);
-  const variables = new Map(
-    Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
-  );
-  const { dcSet } = generateDcSetAndIsNull(variables);
 
-  return tautology([...complement(expr2Minterms), ...dcSet, ...expr1Minterms]);
+  return tautology([
+    ...complement(expr2Minterms),
+    ...context.getDcSet(),
+    ...expr1Minterms,
+  ]);
 }

lib/common/expression-parser.ts

@@ -29,7 +29,7 @@ function interpretEscapes(str): string {
     r: "\r",
     t: "\t",
   };
-  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
+  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/g, (_, escape) => {
     const type = escape.charAt(0);
     const hex = escape.slice(1);
     if (type === "u") return String.fromCharCode(parseInt(hex, 16));

lib/common/expression.ts

@@ -178,11 +178,11 @@ export function evaluateCallback(exp: Expression): Expression {
   } else if (exp[0] === "IS NULL") {
     if (isArray(exp[1])) return exp;
     else if (exp[1] == null) return true;
-    else return null;
+    else return false;
   } else if (exp[0] === "IS NOT NULL") {
     if (isArray(exp[1])) return exp;
     else if (exp[1] != null) return true;
-    else return null;
+    else return false;
   } else if (exp[0] === "LIKE") {
     if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
     else if (
@@ -334,15 +334,7 @@ export async function evaluateAsync(
     if (!isArray(e)) return e;
 
     if (e[0] === "FUNC") {
-      if (e[1] === "NOW") {
-        if (now) return now;
-      } else if (e[1] === "UPPER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
-      } else if (e[1] === "LOWER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
-      }
+      if (e[1] === "NOW") if (now) return now;
     } else if (e[0] === "PARAM") {
       if (e[1] == null) return null;
       if (obj && !isArray(e[1])) {

lib/common/path-set.ts

@@ -20,9 +20,9 @@
 import Path from "./path";
 
 export default class PathSet {
-  private lengthIndex: Set<Path>[];
-  private fragmentIndex: Map<string, Set<Path>>[];
-  private stringIndex: Map<string, Path>;
+  private declare lengthIndex: Set<Path>[];
+  private declare fragmentIndex: Map<string, Set<Path>>[];
+  private declare stringIndex: Map<string, Path>;
 
   public constructor() {
     this.lengthIndex = [];

lib/common/path.ts

@@ -23,44 +23,102 @@ type Alias = [Path, string][];
 let cache1 = new Map<string, Path>();
 let cache2 = new Map<string, Path>();
 
+const CHAR_SPACE = 32;
+const CHAR_BACKSLASH = 92;
+const CHAR_COLON = 58;
+const CHAR_QUOTE = 34;
+const CHAR_OPEN_BRACKET = 91;
+const CHAR_CLOSE_BRACKET = 93;
+const CHAR_ASTERISK = 42;
+const CHAR_COMMA = 44;
+const CHAR_DOT = 46;
+
+function charCodeAt(str: string, index: number): number {
+  if (index >= str.length) return 0;
+  return str.charCodeAt(index);
+}
+
+function legalChar(c: number): boolean {
+  return (
+    (c >= 97 && c <= 122) ||
+    (c >= 65 && c <= 90) ||
+    (c >= 48 && c <= 57) ||
+    c === 95 ||
+    c === 45
+  );
+}
+
 export default class Path {
-  public readonly segments: Segments;
-  public readonly wildcard: number;
-  public readonly alias: number;
-  protected _string: string;
-  protected _stringIndex: number[];
+  public declare readonly segments: Segments;
+  public declare readonly wildcard: number;
+  public declare readonly alias: number;
+  protected declare _string: string;
+  protected declare _stringIndex: number[];
+
+  protected static parseAliasValue(
+    pattern: string,
+    index: number
+  ): { index: number; value: string } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) === CHAR_QUOTE) {
+      for (let j = i + 1; j < pattern.length; ++j) {
+        if (
+          pattern.charCodeAt(j) === CHAR_QUOTE &&
+          pattern.charCodeAt(j - 1) !== CHAR_BACKSLASH
+        ) {
+          try {
+            ++j;
+            const v = JSON.parse(pattern.slice(i, j));
+            return { index: j, value: v };
+          } catch {
+            return { index, value: null };
+          }
+        }
+      }
+      return { index, value: null };
+    }
+
+    for (; i < pattern.length; ++i) {
+      const c = pattern.charCodeAt(i);
+      if (c === CHAR_CLOSE_BRACKET || c === CHAR_COMMA) break;
+    }
+
+    return { index: i, value: pattern.slice(index, i).trim() };
+  }
+
+  protected static parseAliasPath(
+    pattern: string,
+    index: number
+  ): { index: number; path: Path } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    const { index: idx, segments } = Path.parsePath(pattern, i);
+    if (idx === i) return { index, path: null };
+    i = idx;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) !== CHAR_COLON) return { index, path: null };
+    return { index: i + 1, path: new Path(segments) };
+  }
 
   protected static parseAlias(
     pattern: string,
     index: number
   ): { index: number; alias: Alias } {
     const alias: Alias = [];
-    while (index < pattern.length && pattern[index] !== "]") {
-      const { index: idx, segments } = Path.parsePath(pattern, index);
-      let j = (index = idx + 1);
-      while (pattern[j] !== "]" && pattern[j] !== ",") {
-        if (pattern[j] === '"' && index === j) {
-          ++j;
-          while (pattern[j] !== '"' || pattern[j - 1] === "\\") {
-            if (++j >= pattern.length)
-              throw new Error("Invalid alias expression");
-          }
-        }
-        if (++j >= pattern.length) throw new Error("Invalid alias expression");
-      }
+    let i = index;
+    for (;;) {
+      let path: Path;
+      let value: string;
+      ({ index: i, path } = Path.parseAliasPath(pattern, i));
+      if (!path) break;
+      ({ index: i, value } = Path.parseAliasValue(pattern, i));
+      if (value == null) break;
+      alias.push([path, value]);
+      index = i;
 
-      let value = pattern.slice(index, j).trim();
-      index = j;
-      if (value[0] === '"') {
-        try {
-          value = JSON.parse(value);
-        } catch (error) {
-          throw new Error("Invalid alias expression");
-        }
-      }
-
-      alias.push([new Path(segments), value]);
-      if (pattern[index] === ",") ++index;
+      if (charCodeAt(pattern, index) !== CHAR_COMMA) break;
+      ++i;
     }
 
     // Ensure identical expressions have idential string representation
@@ -81,30 +139,26 @@ export default class Path {
     index: number
   ): { index: number; segments: Segments } {
     const segments = [];
-    // Colon separator is needed for parseAlias
-    if (index < pattern.length && pattern[index] !== ":") {
-      for (;;) {
-        if (pattern[index] === "[") {
-          const { index: idx, alias } = Path.parseAlias(pattern, index + 1);
-          index = idx + 1;
-          segments.push(alias);
-        } else {
-          const j = index;
-          while (
-            index < pattern.length &&
-            pattern[index] !== ":" &&
-            pattern[index] !== "."
-          )
-            ++index;
-          const s = pattern.slice(j, index).trim();
-          if (!s) throw new Error("Invalid parameter path");
-          segments.push(s);
-        }
 
-        if (index >= pattern.length || pattern[index] === ":") break;
-        else if (pattern[index] !== ".")
-          throw new Error("Invalid alias expression");
-        ++index;
+    let segStart = index;
+    let wildcard = -1;
+    for (let i = index; i <= pattern.length; ++i) {
+      const c = charCodeAt(pattern, i);
+      if (c === CHAR_OPEN_BRACKET && i === segStart) {
+        const { index: idx, alias } = Path.parseAlias(pattern, i + 1);
+        if (charCodeAt(pattern, idx) !== CHAR_CLOSE_BRACKET) break;
+        segments.push(alias);
+        index = i = idx + 1;
+        segStart = i + 1;
+      } else if (c === CHAR_ASTERISK && i === segStart) {
+        wildcard = i + 1;
+      } else if (!legalChar(c) || i === wildcard) {
+        if (i === segStart) break;
+        const s = pattern.slice(segStart, i);
+        segments.push(s);
+        index = i;
+        if (c !== CHAR_DOT) break;
+        segStart = i + 1;
       }
     }
 
@@ -143,7 +197,8 @@ export default class Path {
     if (!path) {
       path = cache2.get(str);
       if (!path) {
-        const { segments } = Path.parsePath(str, 0);
+        const { index, segments } = Path.parsePath(str, 0);
+        if (index < str.length) throw new Error("Invalid parameter path");
         path = new Path(segments);
         if (path.toString() !== str) cache1.set(path.toString(), path);
       }

lib/config.ts

@@ -43,7 +43,7 @@ const options = {
 
   CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
   CWMP_PORT: { type: "int", default: 7547 },
-  CWMP_INTERFACE: { type: "string", default: "0.0.0.0" },
+  CWMP_INTERFACE: { type: "string", default: "::" },
   CWMP_SSL_CERT: { type: "string", default: "" },
   CWMP_SSL_KEY: { type: "string", default: "" },
   CWMP_LOG_FILE: { type: "path", default: "" },
@@ -51,7 +51,7 @@ const options = {
 
   NBI_WORKER_PROCESSES: { type: "int", default: 0 },
   NBI_PORT: { type: "int", default: 7557 },
-  NBI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  NBI_INTERFACE: { type: "string", default: "::" },
   NBI_SSL_CERT: { type: "string", default: "" },
   NBI_SSL_KEY: { type: "string", default: "" },
   NBI_LOG_FILE: { type: "path", default: "" },
@@ -59,7 +59,7 @@ const options = {
 
   FS_WORKER_PROCESSES: { type: "int", default: 0 },
   FS_PORT: { type: "int", default: 7567 },
-  FS_INTERFACE: { type: "string", default: "0.0.0.0" },
+  FS_INTERFACE: { type: "string", default: "::" },
   FS_SSL_CERT: { type: "string", default: "" },
   FS_SSL_KEY: { type: "string", default: "" },
   FS_URL_PREFIX: { type: "string", default: "" },
@@ -68,7 +68,7 @@ const options = {
 
   UI_WORKER_PROCESSES: { type: "int", default: 0 },
   UI_PORT: { type: "int", default: 3000 },
-  UI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  UI_INTERFACE: { type: "string", default: "::" },
   UI_SSL_CERT: { type: "string", default: "" },
   UI_SSL_KEY: { type: "string", default: "" },
   UI_LOG_FILE: { type: "path", default: "" },

lib/connection-request.ts

@@ -19,7 +19,7 @@
 
 import * as crypto from "crypto";
 import * as dgram from "dgram";
-import { parse } from "url";
+import { URL } from "url";
 import * as http from "http";
 import { evaluateAsync } from "./common/expression";
 import { Expression } from "./types";
@@ -61,24 +61,26 @@ async function extractAuth(
 }
 
 function httpGet(
+  url: URL,
   options: http.RequestOptions,
   _debug: boolean,
   deviceId: string
 ): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
   return new Promise((resolve, reject) => {
     const req = http
-      .get(options, (res) => {
+      .get(url, options, (res) => {
         res.resume();
         resolve({ statusCode: res.statusCode, headers: res.headers });
         if (_debug) {
-          debug.outgoingHttpRequest(req, deviceId, options, null);
+          debug.outgoingHttpRequest(req, deviceId, "GET", url, null);
           debug.incomingHttpResponse(res, deviceId, null);
         }
       })
       .on("error", (err) => {
         req.destroy();
         reject(err);
-        if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
+        if (_debug)
+          debug.outgoingHttpRequestError(req, deviceId, "GET", url, err);
       })
       .on("timeout", () => {
         req.destroy();
@@ -94,15 +96,13 @@ export async function httpConnectionRequest(
   _debug: boolean,
   deviceId: string
 ): Promise<string> {
-  const options: http.RequestOptions = parse(address);
-  if (options.protocol !== "http:")
+  const url = new URL(address);
+  if (url.protocol !== "http:")
     return "Invalid connection request URL or protocol";
 
-  options.agent = new http.Agent({
-    maxSockets: 1,
-    keepAlive: true,
-    timeout: timeout,
-  });
+  const options: http.RequestOptions = {
+    agent: new http.Agent({ maxSockets: 1, keepAlive: true, timeout: timeout }),
+  };
 
   let authHeader: Record<string, string>;
   let username: string;
@@ -129,7 +129,7 @@ export async function httpConnectionRequest(
               Authorization: auth.solveDigest(
                 username,
                 password,
-                options.path,
+                url.pathname + url.search,
                 "GET",
                 null,
                 authHeader
@@ -145,12 +145,12 @@ export async function httpConnectionRequest(
 
     let res: { statusCode: number; headers: http.IncomingHttpHeaders };
     try {
-      res = await httpGet(opts, _debug, deviceId);
+      res = await httpGet(url, opts, _debug, deviceId);
     } catch (err) {
       // Workaround for some devices unexpectedly closing the connection
       if (authHeader) {
         try {
-          res = await httpGet(opts, _debug, deviceId);
+          res = await httpGet(url, opts, _debug, deviceId);
         } catch (err) {
           return `Connection request error: ${err.message}`;
         }
@@ -169,9 +169,13 @@ export async function httpConnectionRequest(
     if (res.statusCode === 503) return "Device is offline";
 
     if (res.statusCode === 401 && res.headers["www-authenticate"]) {
-      authHeader = auth.parseWwwAuthenticateHeader(
-        res.headers["www-authenticate"]
-      );
+      try {
+        authHeader = auth.parseWwwAuthenticateHeader(
+          res.headers["www-authenticate"]
+        );
+      } catch (err) {
+        return "Connection request error: Error parsing www-authenticate header";
+      }
       [username, password, authExp] = await extractAuth(authExp, false);
     } else {
       return `Connection request error: Unexpected status code ${res.statusCode}`;
@@ -182,14 +186,13 @@ export async function httpConnectionRequest(
 }
 
 export async function udpConnectionRequest(
-  address: string,
+  host: string,
+  port: number,
   authExp: Expression,
   sourcePort = 0,
   _debug: boolean,
   deviceId: string
 ): Promise<void> {
-  const [host, portStr] = address.split(":", 2);
-  const port = portStr ? parseInt(portStr) : 80;
   const now = Date.now();
 
   const client = dgram.createSocket({ type: "udp4", reuseAddr: true });
@@ -216,8 +219,8 @@ export async function udpConnectionRequest(
       .createHmac("sha1", password)
       .update(`${ts}${id}${username}${cn}`)
       .digest("hex");
-    const uri = `http://${address}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
-    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${address}\r\n\r\n`;
+    const uri = `http://${host}:${port}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
+    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${host}:${port}\r\n\r\n`;
     const message = Buffer.from(msg);
 
     for (let i = 0; i < 3; ++i) {

lib/cwmp.ts

@@ -84,12 +84,16 @@ async function authenticate(
   let authentication;
 
   if (sessionContext.httpRequest.headers["authorization"]) {
-    authentication = auth.parseAuthorizationHeader(
-      sessionContext.httpRequest.headers["authorization"]
-    );
+    try {
+      authentication = auth.parseAuthorizationHeader(
+        sessionContext.httpRequest.headers["authorization"]
+      );
+    } catch (err) {
+      return false;
+    }
   }
 
-  if (authentication && authentication.method === "Digest") {
+  if (authentication?.method === "Digest") {
     const sessionNonce = sessionsNonces.get(sessionContext.httpRequest.socket);
 
     if (
@@ -183,7 +187,8 @@ async function writeResponse(
   }
 
   const httpResponse = sessionContext.httpResponse;
-  const connection = httpResponse.socket;
+  // Don't use httpResponse.socket as it may be null, even before end() is called
+  const connection = sessionContext.httpRequest.socket;
 
   httpResponse.setHeader("Content-Length", Buffer.byteLength(data));
   httpResponse.writeHead(res.code, res.headers);
@@ -208,7 +213,7 @@ async function writeResponse(
       sessionContext.extendLock = now + 10000;
       await cache.acquireLock(
         `cwmp_session_${sessionContext.deviceId}`,
-        sessionContext.timeout + 15000,
+        sessionContext.timeout * 1000 + 15000,
         0,
         sessionContext.sessionId
       );
@@ -430,7 +435,7 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
   const deviceEvents = {};
   for (const p of deviceData.paths.find(Path.parse("Events.*"), false, true)) {
     const attrs = deviceData.attributes.get(p);
-    if (attrs && attrs.value && attrs.value[1][0] >= sessionContext.timestamp)
+    if (attrs?.value && attrs.value[1][0] >= sessionContext.timestamp)
       deviceEvents[p.segments[1] as string] = true;
   }
 
@@ -454,7 +459,7 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
 
     if (!eventsMatch) continue;
 
-    if (preset.schedule && preset.schedule.schedule) {
+    if (preset.schedule?.schedule) {
       const r = scheduling.cron(
         sessionContext.timestamp,
         preset.schedule.schedule
@@ -493,10 +498,11 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
     defer: true,
   }));
 
-  const { fault: flt, rpcId: reqId, rpc: acsReq } = await session.rpcRequest(
-    sessionContext,
-    declarations
-  );
+  const {
+    fault: flt,
+    rpcId: reqId,
+    rpc: acsReq,
+  } = await session.rpcRequest(sessionContext, declarations);
 
   if (flt) {
     recordFault(sessionContext, flt);
@@ -568,10 +574,11 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
 
   deviceData.timestamps.dirty = 0;
   deviceData.attributes.dirty = 0;
-  const { fault: fault, rpcId: id, rpc: acsRequest } = await session.rpcRequest(
-    sessionContext,
-    null
-  );
+  const {
+    fault: fault,
+    rpcId: id,
+    rpc: acsRequest,
+  } = await session.rpcRequest(sessionContext, null);
 
   if (fault) {
     recordFault(sessionContext, fault);
@@ -601,10 +608,11 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
 }
 
 async function nextRpc(sessionContext: SessionContext): Promise<void> {
-  const { fault: fault, rpcId: id, rpc: acsRequest } = await session.rpcRequest(
-    sessionContext,
-    null
-  );
+  const {
+    fault: fault,
+    rpcId: id,
+    rpc: acsRequest,
+  } = await session.rpcRequest(sessionContext, null);
 
   if (fault) {
     recordFault(sessionContext, fault);
@@ -782,7 +790,7 @@ async function endSession(sessionContext: SessionContext): Promise<void> {
     }
   }
 
-  if (sessionContext.doneTasks && sessionContext.doneTasks.length) {
+  if (sessionContext.doneTasks?.length) {
     saveCache = true;
     promises.push(
       db.clearTasks(sessionContext.deviceId, sessionContext.doneTasks)
@@ -874,27 +882,6 @@ async function sendAcsRequest(
   return writeResponse(sessionContext, res);
 }
 
-async function getSession(
-  connection: Socket,
-  sessionId: string
-): Promise<SessionContext> {
-  let sessionContext = currentSessions.get(connection);
-  if (sessionContext) {
-    currentSessions.delete(connection);
-    return sessionContext;
-  }
-
-  if (!sessionId) return null;
-
-  await new Promise((resolve) => setTimeout(resolve, 100));
-
-  const sessionContextString = await cache.pop(`session_${sessionId}`);
-  if (!sessionContextString) return null;
-  sessionContext = await session.deserialize(sessionContextString);
-  connection.setTimeout(sessionContext.timeout);
-  return sessionContext;
-}
-
 // Only needed to prevent tree shaking from removing the remoteAddress
 // workaround in onConnection function.
 const remoteAddressWorkaround = new WeakMap<Socket, string>();
@@ -1041,13 +1028,13 @@ async function reportBadState(sessionContext: SessionContext): Promise<void> {
     sessionContext: sessionContext,
   });
   const httpResponse = sessionContext.httpResponse;
-  currentSessions.delete(httpResponse.socket);
   const body = "Bad session state";
   httpResponse.setHeader("Content-Length", Buffer.byteLength(body));
   httpResponse.writeHead(400, { Connection: "close" });
   if (sessionContext.debug)
     debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, body);
   httpResponse.end(body);
+  if (sessionContext.state) return endSession(sessionContext);
 }
 
 async function responseUnauthorized(
@@ -1099,7 +1086,7 @@ async function processRequest(
   }
 
   if (sessionContext.state === 0) {
-    if (!rpc.cpeRequest || rpc.cpeRequest.name !== "Inform")
+    if (rpc.cpeRequest?.name !== "Inform")
       return reportBadState(sessionContext);
 
     const res = await inform(sessionContext, rpc);
@@ -1146,7 +1133,7 @@ async function processRequest(
       sessionContext.extendLock = sessionContext.timestamp + 10000;
       await cache.acquireLock(
         `cwmp_session_${sessionContext.deviceId}`,
-        sessionContext.timeout + 15000,
+        sessionContext.timeout * 1000 + 15000,
         0,
         sessionContext.sessionId
       );
@@ -1304,13 +1291,13 @@ export function listener(
       stats.concurrentRequests -= 1;
     })
     .catch((err) => {
-      currentSessions.delete(httpResponse.socket);
+      currentSessions.delete(httpRequest.socket);
       stats.concurrentRequests -= 1;
       setTimeout(() => {
         throw err;
       });
       try {
-        httpResponse.socket.unref();
+        httpRequest.socket.unref();
         httpResponse.writeHead(500, { Connection: "close" });
         httpResponse.end(`${err.name}: ${err.message}`);
       } catch (err) {
@@ -1319,6 +1306,48 @@ export function listener(
     });
 }
 
+async function clientError(
+  httpRequest: IncomingMessage,
+  httpResponse: ServerResponse,
+  sessionContext: SessionContext,
+  body: string,
+  msg: string
+): Promise<void> {
+  let debugEnabled = false;
+  let deviceId: string = null;
+
+  if (sessionContext) {
+    debugEnabled = sessionContext.debug;
+    deviceId = sessionContext.deviceId;
+  } else {
+    const cacheSnapshot = await localCache.getCurrentSnapshot();
+    debugEnabled = !!localCache.getConfig(
+      cacheSnapshot,
+      "cwmp.debug",
+      {
+        remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
+      },
+      Date.now(),
+      (e) => {
+        if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
+          return getRequestOrigin(httpRequest).remoteAddress;
+        return e;
+      }
+    );
+  }
+
+  httpResponse.setHeader("Content-Length", Buffer.byteLength(msg));
+  httpResponse.writeHead(400, { Connection: "close" });
+
+  if (debugEnabled) {
+    debug.incomingHttpRequest(httpRequest, deviceId, body);
+    debug.outgoingHttpResponse(httpResponse, deviceId, msg);
+  }
+
+  httpResponse.end(msg);
+  if (sessionContext?.state) await endSession(sessionContext);
+}
+
 function decodeString(buffer: Buffer, charset: string): string {
   try {
     return buffer.toString(charset as BufferEncoding);
@@ -1345,7 +1374,8 @@ async function listenerAsync(
 
   let sessionId;
   // Separation by comma is important as some devices don't comform to standard
-  const COOKIE_REGEX = /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g;
+  const COOKIE_REGEX =
+    /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g;
   let match;
   while ((match = COOKIE_REGEX.exec(httpRequest.headers.cookie)))
     if (match[1] === "session") sessionId = match[2];
@@ -1406,15 +1436,14 @@ async function listenerAsync(
   // Request aborted
   if (!body) return;
 
-  const newConnection = !currentSessions.has(httpRequest.socket);
-
-  const sessionContext = await getSession(httpRequest.socket, sessionId);
+  let sessionContext = currentSessions.get(httpRequest.socket);
 
   if (sessionContext) {
+    currentSessions.delete(httpRequest.socket);
     sessionContext.httpRequest = httpRequest;
     sessionContext.httpResponse = httpResponse;
     if (
-      (newConnection && sessionContext.sessionId !== sessionId) ||
+      (sessionContext.sessionId !== sessionId && sessionContext.state) ||
       sessionContext.lastActivity + sessionContext.timeout * 1000 < Date.now()
     ) {
       logger.accessError({
@@ -1422,29 +1451,14 @@ async function listenerAsync(
         sessionContext: sessionContext,
       });
 
-      const _body = "Invalid session";
-      httpResponse.setHeader("Content-Length", Buffer.byteLength(_body));
-      httpResponse.writeHead(400, { Connection: "close" });
-      if (sessionContext.debug) {
-        debug.outgoingHttpResponse(
-          httpResponse,
-          sessionContext.deviceId,
-          _body
-        );
-      }
-      httpResponse.end(_body);
-      await endSession(sessionContext);
-      return;
+      return clientError(
+        httpRequest,
+        httpResponse,
+        sessionContext,
+        body.toString(),
+        "Invalid session"
+      );
     }
-    if (newConnection && sessionContext.authState !== 1)
-      sessionContext.authState = 0;
-  } else if (stats.concurrentRequests > MAX_CONCURRENT_REQUESTS) {
-    // Check again just in case device included old session ID
-    // from the previous session
-    httpResponse.writeHead(503, { "Retry-after": 60, Connection: "close" });
-    httpResponse.end("503 Service Unavailable");
-    stats.droppedRequests += 1;
-    return;
   }
 
   let charset: string;
@@ -1464,6 +1478,16 @@ async function listenerAsync(
   const bodyStr = decodeString(body, charset);
 
   if (bodyStr == null) {
+    if (!sessionContext && sessionId) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      const sessionContextString = await cache.pop(`session_${sessionId}`);
+      if (sessionContextString) {
+        sessionContext = await session.deserialize(sessionContextString);
+        sessionContext.httpRequest = httpRequest;
+        sessionContext.httpResponse = httpResponse;
+      }
+    }
+
     const msg = `Unknown encoding '${charset}'`;
     logger.accessError({
       message: "XML parse error",
@@ -1473,132 +1497,87 @@ async function listenerAsync(
         httpResponse: httpResponse,
       },
     });
-    httpResponse.setHeader("Content-Length", Buffer.byteLength(msg));
-    httpResponse.writeHead(400, { Connection: "close" });
-    if (sessionContext) {
-      if (sessionContext.debug) {
-        debug.incomingHttpRequest(
-          httpRequest,
-          sessionContext.deviceId,
-          body.toString()
-        );
-        debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, msg);
-      }
-      await endSession(sessionContext);
-    } else {
-      const cacheSnapshot = await localCache.getCurrentSnapshot();
-      const d = !!localCache.getConfig(
-        cacheSnapshot,
-        "cwmp.debug",
-        {
-          remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
-        },
-        Date.now(),
-        (e) => {
-          if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
-            return getRequestOrigin(httpRequest).remoteAddress;
-          return e;
-        }
-      );
-      if (d) {
-        debug.incomingHttpRequest(httpRequest, null, body.toString());
-        debug.outgoingHttpResponse(httpResponse, null, msg);
-      }
-    }
-    httpResponse.end(msg);
-    return;
+    return clientError(
+      httpRequest,
+      httpResponse,
+      sessionContext,
+      body.toString(),
+      msg
+    );
   }
 
   const parseWarnings = [];
   let rpc: SoapMessage;
   try {
-    rpc = soap.request(
-      bodyStr,
-      sessionContext ? sessionContext.cwmpVersion : null,
-      parseWarnings
-    );
-  } catch (error) {
+    rpc = soap.request(bodyStr, parseWarnings);
+  } catch (err) {
+    if (!sessionContext && sessionId) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      const sessionContextString = await cache.pop(`session_${sessionId}`);
+      if (sessionContextString) {
+        sessionContext = await session.deserialize(sessionContextString);
+        sessionContext.httpRequest = httpRequest;
+        sessionContext.httpResponse = httpResponse;
+      }
+    }
+
     logger.accessError({
       message: "XML parse error",
-      parseError: error.message.trim(),
+      parseError: err.message,
       sessionContext: sessionContext || {
         httpRequest: httpRequest,
         httpResponse: httpResponse,
       },
     });
-    httpResponse.setHeader("Content-Length", Buffer.byteLength(error.message));
-    httpResponse.writeHead(400, { Connection: "close" });
-    if (sessionContext) {
-      if (sessionContext.debug) {
-        debug.incomingHttpRequest(
-          httpRequest,
-          sessionContext.deviceId,
-          bodyStr
-        );
-        debug.outgoingHttpResponse(
-          httpResponse,
-          sessionContext.deviceId,
-          error.message
-        );
-      }
-      await endSession(sessionContext);
-    } else {
-      const cacheSnapshot = await localCache.getCurrentSnapshot();
-      const d = !!localCache.getConfig(
-        cacheSnapshot,
-        "cwmp.debug",
-        {
-          remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
-        },
-        Date.now(),
-        (e) => {
-          if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
-            return getRequestOrigin(httpRequest).remoteAddress;
-          return e;
-        }
-      );
-      if (d) {
-        debug.incomingHttpRequest(httpRequest, null, bodyStr);
-        debug.outgoingHttpResponse(httpResponse, null, error.message);
-      }
+
+    return clientError(
+      httpRequest,
+      httpResponse,
+      sessionContext,
+      bodyStr,
+      err.message
+    );
+  }
+
+  if (!sessionContext && sessionId && rpc.cpeRequest?.name !== "Inform") {
+    await new Promise((resolve) => setTimeout(resolve, 100));
+    const sessionContextString = await cache.pop(`session_${sessionId}`);
+    if (sessionContextString) {
+      sessionContext = await session.deserialize(sessionContextString);
+      sessionContext.httpRequest = httpRequest;
+      sessionContext.httpResponse = httpResponse;
+      httpRequest.socket.setTimeout(sessionContext.timeout * 1000);
+      if (sessionContext.authState !== 1) sessionContext.authState = 0;
     }
-    httpResponse.end(error.message);
-    return;
   }
 
   if (sessionContext)
     return processRequest(sessionContext, rpc, parseWarnings, bodyStr);
 
-  if (!(rpc.cpeRequest && rpc.cpeRequest.name === "Inform")) {
+  if (rpc.cpeRequest?.name !== "Inform") {
     logger.accessError({
       message: "Invalid session",
-      sessionContext: sessionContext || {
+      sessionContext: {
         httpRequest: httpRequest,
         httpResponse: httpResponse,
       },
     });
-    const _body = "Invalid session";
-    httpResponse.setHeader("Content-Length", Buffer.byteLength(_body));
-    httpResponse.writeHead(400, { Connection: "close" });
-    const cacheSnapshot = await localCache.getCurrentSnapshot();
-    const d = !!localCache.getConfig(
-      cacheSnapshot,
-      "cwmp.debug",
-      {
-        remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
-      },
-      Date.now(),
-      (e) => {
-        if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
-          return getRequestOrigin(httpRequest).remoteAddress;
-        return e;
-      }
+
+    return clientError(
+      httpRequest,
+      httpResponse,
+      null,
+      bodyStr,
+      "Invalid session"
     );
-    if (d) {
-      debug.incomingHttpRequest(httpRequest, null, bodyStr);
-      debug.outgoingHttpResponse(httpResponse, null, _body);
-    }
-    httpResponse.end(_body);
+  }
+
+  if (stats.concurrentRequests > MAX_CONCURRENT_REQUESTS) {
+    // Check again just in case device included old session ID
+    // from the previous session
+    httpResponse.writeHead(503, { "Retry-after": 60, Connection: "close" });
+    httpResponse.end("503 Service Unavailable");
+    stats.droppedRequests += 1;
     return;
   }
 

lib/db.ts

@@ -17,9 +17,9 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, ObjectID, Collection, Db } from "mongodb";
+import { MongoClient, ObjectId, Collection, Db } from "mongodb";
 import { get } from "./config";
-import { encodeTag, escapeRegExp } from "./common";
+import { decodeTag, encodeTag, escapeRegExp } from "./common";
 import { parse } from "./common/expression-parser";
 import {
   DeviceData,
@@ -46,6 +46,8 @@ export let tasksCollection: Collection,
 
 let clientPromise: Promise<MongoClient>;
 
+const INVALID_PATH_SUFFIX = "__invalid";
+
 function compareAccessLists(list1: string[], list2: string[]): boolean {
   if (list1.length !== list2.length) return false;
   for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
@@ -59,10 +61,7 @@ export function onConnect(callback: (db: Db) => Promise<void>): void {
 }
 
 export async function connect(): Promise<void> {
-  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
-    useNewUrlParser: true,
-    useUnifiedTopology: true,
-  });
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"));
 
   const client = await clientPromise;
   const db = client.db();
@@ -167,7 +166,19 @@ export async function fetchDevice(
     if (obj["_accessList"] != null)
       attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
 
-    res.push([Path.parse(path), t, attrs]);
+    try {
+      res.push([Path.parse(path), t, attrs]);
+    } catch (err) {
+      // The path parser is now more strict so we might be in a situation where
+      // the database contains invalid paths from before this change So here we
+      // encode the invalid characters.
+      const splits = path.split(".");
+      splits[splits.length - 1] =
+        encodeTag(splits[splits.length - 1]) + INVALID_PATH_SUFFIX;
+      path = splits.join(".");
+      res.push([Path.parse(path), t, attrs]);
+      return;
+    }
 
     for (const [k, v] of Object.entries(obj)) {
       if (!k.startsWith("_")) {
@@ -449,7 +460,16 @@ export async function saveDevice(
         break;
       default:
         if (!diff[2]) {
-          update["$unset"][path.toString()] = 1;
+          let pathStr = path.toString();
+          // Paths with that suffix are encoded and need to be decoded
+          if (pathStr.endsWith(INVALID_PATH_SUFFIX)) {
+            const splits = pathStr.split(".");
+            splits[splits.length - 1] = decodeTag(
+              splits[splits.length - 1].slice(0, 0 - INVALID_PATH_SUFFIX.length)
+            );
+            pathStr = splits.join(".");
+          }
+          update["$unset"][pathStr] = 1;
           continue;
         }
 
@@ -485,7 +505,7 @@ export async function saveDevice(
 
                 break;
               case "object":
-                if (!diff[1] || !diff[1].object || object2 !== object1) {
+                if (!diff[1]?.object || object2 !== object1) {
                   update["$set"][
                     path.length ? path.toString() + "._object" : "_object"
                   ] = !!object2;
@@ -493,7 +513,7 @@ export async function saveDevice(
 
                 break;
               case "writable":
-                if (!diff[1] || !diff[1].writable || writable2 !== writable1) {
+                if (!diff[1]?.writable || writable2 !== writable1) {
                   update["$set"][
                     path.length ? path.toString() + "._writable" : "_writable"
                   ] = !!writable2;
@@ -514,9 +534,8 @@ export async function saveDevice(
                 }
 
                 if (attributesTimestamp2 !== attributesTimestamp1) {
-                  update["$set"][
-                    path.toString() + "._attributesTimestamp"
-                  ] = new Date(attributesTimestamp2);
+                  update["$set"][path.toString() + "._attributesTimestamp"] =
+                    new Date(attributesTimestamp2);
                 }
 
                 break;
@@ -534,9 +553,8 @@ export async function saveDevice(
                 }
 
                 if (attributesTimestamp2 !== attributesTimestamp1) {
-                  update["$set"][
-                    path.toString() + "._attributesTimestamp"
-                  ] = new Date(attributesTimestamp2);
+                  update["$set"][path.toString() + "._attributesTimestamp"] =
+                    new Date(attributesTimestamp2);
                 }
             }
           }
@@ -546,7 +564,7 @@ export async function saveDevice(
           for (const attrName of Object.keys(diff[1])) {
             if (
               diff[1][attrName][1] != null &&
-              (!diff[2] || !diff[2][attrName] || diff[2][attrName][1] == null)
+              diff[2]?.[attrName]?.[1] == null
             ) {
               const p = path.length ? path.toString() + "." : "";
               update["$unset"][`${p}_${attrName}`] = 1;
@@ -600,7 +618,7 @@ export async function saveDevice(
     upsert: isNew,
   });
 
-  if (result.result.n !== 1)
+  if (!result.matchedCount && !result.upsertedCount)
     throw new Error(`Device ${deviceId} not found in database`);
 
   if (update2) {
@@ -624,7 +642,7 @@ export async function getFaults(
     delete r.device;
     r.timestamp = +r.timestamp;
     r.provisions = JSON.parse(r.provisions);
-    faults[channel] = r;
+    faults[channel] = r as SessionFault;
   }
 
   return faults;
@@ -665,13 +683,13 @@ export async function getDueTasks(
     if (task.timestamp >= timestamp) return [tasks, +task.timestamp];
     task._id = String(task._id);
 
-    tasks.push(task);
+    tasks.push(task as Task);
 
     // For API compatibility
     if (task.name === "download" && task.file) {
       let q;
-      if (ObjectID.isValid(task.file))
-        q = { _id: { $in: [task.file, new ObjectID(task.file)] } };
+      if (ObjectId.isValid(task.file))
+        q = { _id: { $in: [task.file, new ObjectId(task.file)] } };
       else q = { _id: task.file };
 
       const res = await filesCollection.find(q).toArray();
@@ -692,7 +710,7 @@ export async function clearTasks(
   taskIds: string[]
 ): Promise<void> {
   await tasksCollection.deleteMany({
-    _id: { $in: taskIds.map((id) => new ObjectID(id)) },
+    _id: { $in: taskIds.map((id) => new ObjectId(id)) },
   });
 }
 
@@ -709,14 +727,14 @@ export async function getOperations(
     delete r._id;
     // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
     if (typeof r.provisions !== "string") {
-      operations[commandKey] = r;
+      operations[commandKey] = r as Operation;
       continue;
     }
     r.timestamp = +r.timestamp;
     if (r.args) r.args = JSON.parse(r.args);
     r.provisions = JSON.parse(r.provisions);
     r.retries = JSON.parse(r.retries);
-    operations[commandKey] = r;
+    operations[commandKey] = r as Operation;
   }
   return operations;
 }
@@ -784,7 +802,7 @@ interface Permission {
 }
 
 export async function getPermissions(): Promise<Permission[]> {
-  return permissionsCollection.find().toArray();
+  return permissionsCollection.find().toArray() as unknown as Permission[];
 }
 
 interface User {
@@ -795,5 +813,5 @@ interface User {
 }
 
 export async function getUsers(): Promise<User[]> {
-  return usersCollection.find().toArray();
+  return usersCollection.find().toArray() as unknown as User[];
 }

lib/debug.ts

@@ -17,12 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import {
-  IncomingMessage,
-  ServerResponse,
-  ClientRequest,
-  RequestOptions,
-} from "http";
+import { IncomingMessage, ServerResponse, ClientRequest } from "http";
 import { Socket } from "net";
 import { appendFileSync } from "fs";
 import { stringify } from "./common/yaml";
@@ -99,7 +94,8 @@ export function outgoingHttpResponse(
 export function outgoingHttpRequest(
   httpRequest: ClientRequest,
   deviceId: string,
-  options: RequestOptions,
+  method: "GET" | "PUT" | "POST" | "DELETE",
+  url: URL,
   body: string
 ): void {
   if (!DEBUG_FILE) return;
@@ -111,9 +107,9 @@ export function outgoingHttpRequest(
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
-    remotePort: options.port,
-    method: options.method || "GET",
-    url: options.path,
+    remotePort: url.port,
+    method: method,
+    url: url.pathname + url.search,
     headers: httpRequest.getHeaders(),
     body: body,
   };
@@ -128,7 +124,8 @@ export function outgoingHttpRequest(
 export function outgoingHttpRequestError(
   httpRequest: ClientRequest,
   deviceId: string,
-  options: RequestOptions,
+  method: "GET" | "PUT" | "POST" | "DELETE",
+  url: URL,
   err: Error
 ): void {
   if (!DEBUG_FILE) return;
@@ -136,12 +133,12 @@ export function outgoingHttpRequestError(
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
-    remoteAddress: options.hostname,
+    remoteAddress: url.hostname,
     deviceId: deviceId,
     connection: null,
-    remotePort: options.port,
-    method: options.method,
-    url: options.path,
+    remotePort: url.port,
+    method: method,
+    url: url.pathname + url.search,
     headers: httpRequest.getHeaders(),
     error: err.message,
   };

lib/device.ts

@@ -54,13 +54,6 @@ export function sanitizeParameterValue(
           else parameterValue[0] = b;
         }
         break;
-      case "xsd:string":
-      case "xsd:base64":
-      case "xsd:hexBinary":
-        if (typeof parameterValue[0] !== "string")
-          parameterValue[0] = "" + parameterValue[0];
-
-        break;
       case "xsd:int":
       case "xsd:unsignedInt":
         if (typeof parameterValue[0] !== "number") {
@@ -83,7 +76,7 @@ export function sanitizeParameterValue(
         }
         break;
       default:
-        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
+        parameterValue[0] = "" + parameterValue[0];
         break;
     }
   }
@@ -208,7 +201,7 @@ export function clear(
   timestamp = timestamp || 0;
 
   let descendantsTimestamp = timestamp;
-  if (attributes && attributes.object) {
+  if (attributes?.object) {
     if (attributes.object > descendantsTimestamp)
       descendantsTimestamp = attributes.object;
     if (!(attributes.object <= attributes.value))

lib/extensions.ts

@@ -30,14 +30,6 @@ const TIMEOUT = +config.get("EXT_TIMEOUT");
 const processes: { [script: string]: ChildProcess } = {};
 const jobs = new Map();
 
-function messageHandler(message): void {
-  const func = jobs.get(message[0]);
-  if (func) {
-    jobs.delete(message[0]);
-    func({ fault: message[1], value: message[2] });
-  }
-}
-
 export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
   return new Promise((resolve) => {
     const scriptName = args[0];
@@ -70,7 +62,16 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
         if (processes[scriptName] === p) delete processes[scriptName];
       });
 
-      p.on("message", messageHandler);
+      p.on("message", (message) => {
+        const func = jobs.get(message[0]);
+        if (func) {
+          jobs.delete(message[0]);
+          // Wait for any disconnect even to fire
+          setTimeout(() => {
+            func({ fault: message[1], value: message[2] });
+          });
+        }
+      });
 
       const rlstdout = readline.createInterface(p.stdout);
       rlstdout.on("line", (line) => {

lib/fs.ts

@@ -18,7 +18,6 @@
  */
 
 import * as url from "url";
-import * as querystring from "querystring";
 import { IncomingMessage, ServerResponse } from "http";
 import { Collection, GridFSBucket } from "mongodb";
 import { onConnect } from "./db";
@@ -35,7 +34,7 @@ onConnect(async (db) => {
 });
 
 const getFile = memoize(
-  (md5: string, size: number, filename: string): Promise<Buffer> => {
+  (uploadDate: number, size: number, filename: string): Promise<Buffer> => {
     return new Promise((resolve, reject) => {
       const buffer = Buffer.allocUnsafe(size);
       let i = 0;
@@ -64,7 +63,7 @@ export async function listener(
   }
 
   const urlParts = url.parse(request.url, true);
-  const filename = querystring.unescape(urlParts.pathname.substring(1));
+  const filename = decodeURIComponent(urlParts.pathname.substring(1));
 
   const log = {
     message: "Fetch file",
@@ -82,10 +81,14 @@ export async function listener(
     return;
   }
 
-  const buffer = await getFile(file.md5, file.length, filename);
+  const buffer = await getFile(
+    file["uploadDate"].getTime(),
+    file.length,
+    filename
+  );
 
   response.writeHead(200, {
-    "Content-Type": file.contentType || "application/octet-stream",
+    "Content-Type": "application/octet-stream",
     "Content-Length": file.length,
   });
 

lib/init.ts

@@ -59,7 +59,7 @@ const INFORM_SCRIPT = `
 // Device ID as user name
 const username = declare("DeviceID.ID", {value: 1}).value[0]
 
-// Password will be fixed a given device because Math.random() is seeded with devcie ID by default.
+// Password will be fixed for a given device because Math.random() is seeded with device ID by default.
 const password = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER).toString(36);
 
 const informInterval = 300;
@@ -257,7 +257,7 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
           "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
       },
       { _id: "ui.device.3.type", value: "'parameter-table'" },
-      { _id: "ui.device.3.label", value: "'WLAN hosts'" },
+      { _id: "ui.device.3.label", value: "'LAN hosts'" },
       {
         _id: "ui.device.3.parameter",
         value: "InternetGatewayDevice.LANDevice.1.Hosts.Host",

lib/instance-set.ts

@@ -22,7 +22,7 @@ interface Instance {
 }
 
 export default class InstanceSet {
-  private set: Set<Instance>;
+  private declare set: Set<Instance>;
 
   public constructor() {
     this.set = new Set();

lib/local-cache.ts

@@ -131,7 +131,7 @@ async function fetchPresets(): Promise<Preset[]> {
 
     // If no keys are defined, consider all parameters as keys to keep the
     // same behavior from v1.0
-    if (!obj["_keys"] || !obj["_keys"].length)
+    if (!obj["_keys"]?.length)
       obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
 
     return obj;
@@ -266,9 +266,7 @@ async function fetchProvisions(): Promise<Provisions> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    provisions[
-      r["_id"]
-    ].script = new vm.Script(
+    provisions[r["_id"]].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -287,9 +285,7 @@ async function fetchVirtualParameters(): Promise<VirtualParameters> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    virtualParameters[
-      r["_id"]
-    ].script = new vm.Script(
+    virtualParameters[r["_id"]].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -325,8 +321,6 @@ async function fetchFiles(): Promise<Files> {
     const id = r["filename"] || r["_id"].toString();
     files[id] = {};
     files[id].length = r["length"];
-    files[id].md5 = r["md5"];
-    files[id].contentType = r["contentType"];
   }
 
   return files;
@@ -366,12 +360,13 @@ async function fetchConfig(): Promise<[Config, UiConfig]> {
     _config[c.id] = val;
     if (c.id.startsWith("ui.")) {
       const keys = c.id.split(".");
+      if (!(keys[1] in ui)) continue;
       // remove the first key(ui)
       keys.shift();
       let ref = ui;
       while (keys.length > 1) {
         const k = keys.shift();
-        if (typeof ref[k] !== "object") ref[k] = {};
+        if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
         ref = ref[k];
       }
       ref[keys[0]] = val;
@@ -501,7 +496,7 @@ export function getConfig(
   if (!(key in snapshot.config)) {
     if (key in oldOpts) {
       let id;
-      if (context && context["id"]) {
+      if (context?.["id"]) {
         id = context["id"];
       } else if (cb) {
         id = cb(["PARAM", "DeviceID.ID"]);

lib/logger.ts

@@ -292,11 +292,9 @@ function formatSimple(
     return `${severity}${remote}${details.message}${meta}${os.EOL}`;
   }
 
-  return `${
-    details.timestamp
-  } [${(details.severity as string).toUpperCase()}] ${remote}${
-    details.message
-  }${meta}${os.EOL}`;
+  return `${details.timestamp} [${(
+    details.severity as string
+  ).toUpperCase()}] ${remote}${details.message}${meta}${os.EOL}`;
 }
 
 function log(details: Record<string, unknown>): void {

lib/mongodb-functions.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ObjectID } from "mongodb";
+import { ObjectId } from "mongodb";
 import { map, parse, stringify, parseList } from "./common/expression-parser";
 import { likePatternToRegExp } from "./common/expression";
 import { Expression, Fault, Task } from "./types";
@@ -90,7 +90,7 @@ export function processTasksFilter(filter: Expression): Expression {
     if (!isArray(exp)) return exp;
     if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
       const e = exp.slice();
-      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectID(e[2]);
+      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectId(e[2]);
       else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
         e[2] = new Date(e[2]);
       else if (e[1][0] === "PARAM" && e[1][1] === "expiry")

lib/nbi.ts

@@ -19,7 +19,6 @@
 
 import * as url from "url";
 import { Collection, GridFSBucket, ObjectId } from "mongodb";
-import * as querystring from "querystring";
 import * as vm from "vm";
 import * as config from "./config";
 import { onConnect } from "./db";
@@ -30,10 +29,12 @@ import * as cache from "./cache";
 import { version as VERSION } from "../package.json";
 import { ping } from "./ping";
 import * as logger from "./logger";
+import { flattenDevice } from "./mongodb-functions";
 
 const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
 const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
-const TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tags\/([a-zA-Z0-9\-_%]+)\/?$/;
+const TAGS_REGEX =
+  /^\/devices\/([a-zA-Z0-9\-_%]+)\/tags\/([a-zA-Z0-9\-_%]+)\/?$/;
 const PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-_%]+)\/?$/;
 const OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FILES_REGEX = /^\/files\/([a-zA-Z0-9%!*'();:@&=+$,?#[\]\-_.~]+)\/?$/;
@@ -41,10 +42,11 @@ const PING_REGEX = /^\/ping\/([a-zA-Z0-9\-_.:]+)\/?$/;
 const QUERY_REGEX = /^\/([a-zA-Z0-9_]+)\/?$/;
 const DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/?$/;
 const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
-const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
+const VIRTUAL_PARAMETERS_REGEX =
+  /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
 
-const collections = {
+const collections: Record<string, Collection> = {
   tasks: null as Collection,
   devices: null as Collection,
   presets: null as Collection,
@@ -102,17 +104,24 @@ export function listener(
 
     logger.accessInfo(
       Object.assign({}, urlParts.query, {
-        remoteAddress: request.connection.remoteAddress,
+        remoteAddress: request.socket.remoteAddress,
         message: `${request.method} ${urlParts.pathname}`,
       })
     );
 
     if (PRESETS_REGEX.test(urlParts.pathname)) {
-      const presetName = querystring.unescape(
+      const presetName = decodeURIComponent(
         PRESETS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
-        const preset = JSON.parse(body.toString());
+        let preset;
+        try {
+          preset = JSON.parse(body.toString());
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
         preset._id = presetName;
         collections.presets.replaceOne(
           { _id: presetName },
@@ -155,11 +164,18 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (OBJECTS_REGEX.test(urlParts.pathname)) {
-      const objectName = querystring.unescape(
+      const objectName = decodeURIComponent(
         OBJECTS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
-        const object = JSON.parse(body.toString());
+        let object;
+        try {
+          object = JSON.parse(body.toString());
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
         object._id = objectName;
         collections.objects.replaceOne(
           { _id: objectName },
@@ -202,7 +218,7 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (PROVISIONS_REGEX.test(urlParts.pathname)) {
-      const provisionName = querystring.unescape(
+      const provisionName = decodeURIComponent(
         PROVISIONS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
@@ -260,7 +276,7 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (VIRTUAL_PARAMETERS_REGEX.test(urlParts.pathname)) {
-      const virtualParameterName = querystring.unescape(
+      const virtualParameterName = decodeURIComponent(
         VIRTUAL_PARAMETERS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
@@ -322,8 +338,8 @@ export function listener(
       }
     } else if (TAGS_REGEX.test(urlParts.pathname)) {
       const r = TAGS_REGEX.exec(urlParts.pathname);
-      const deviceId = querystring.unescape(r[1]);
-      const tag = querystring.unescape(r[2]);
+      const deviceId = decodeURIComponent(r[1]);
+      const tag = decodeURIComponent(r[2]);
       if (request.method === "POST") {
         collections.devices.updateOne(
           { _id: deviceId },
@@ -351,7 +367,7 @@ export function listener(
       }
     } else if (FAULTS_REGEX.test(urlParts.pathname)) {
       if (request.method === "DELETE") {
-        const faultId = querystring.unescape(
+        const faultId = decodeURIComponent(
           FAULTS_REGEX.exec(urlParts.pathname)[1]
         );
         const deviceId = faultId.split(":", 1)[0];
@@ -396,15 +412,31 @@ export function listener(
       }
     } else if (DEVICE_TASKS_REGEX.test(urlParts.pathname)) {
       if (request.method === "POST") {
-        const deviceId = querystring.unescape(
+        const deviceId = decodeURIComponent(
           DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1]
         );
         if (body.length) {
-          const task = JSON.parse(body.toString());
+          let task;
+          try {
+            task = JSON.parse(body.toString());
+          } catch (err) {
+            response.writeHead(400);
+            response.end(`${err.name}: ${err.message}`);
+            return;
+          }
           task.device = deviceId;
-          apiFunctions
-            .insertTasks(task)
-            .then(async () => {
+          collections.devices
+            .findOne({ _id: deviceId })
+            .then(async (dev) => {
+              if (!dev) {
+                response.writeHead(404);
+                response.end("No such device");
+                return;
+              }
+
+              const device = flattenDevice(dev);
+              await apiFunctions.insertTasks(task);
+
               const lastInform = Date.now();
 
               const socketTimeout: number = request.socket["timeout"];
@@ -435,7 +467,10 @@ export function listener(
                 return;
               }
 
-              const status = await apiFunctions.connectionRequest(deviceId);
+              const status = await apiFunctions.connectionRequest(
+                deviceId,
+                device
+              );
 
               if (status) {
                 if (socketTimeout) request.socket.setTimeout(socketTimeout);
@@ -518,15 +553,28 @@ export function listener(
             });
         } else if (urlParts.query.connection_request != null) {
           // No task, send connection request only
-          apiFunctions
-            .connectionRequest(deviceId)
-            .then(() => {
-              response.writeHead(200);
-              response.end();
+          collections.devices
+            .findOne({ _id: deviceId })
+            .then(async (dev) => {
+              if (!dev) {
+                response.writeHead(404);
+                response.end("No such device");
+                return;
+              }
+              return apiFunctions.connectionRequest(deviceId).then((status) => {
+                if (status) {
+                  response.writeHead(504, status);
+                  response.end(status);
+                  return;
+                }
+                response.writeHead(200);
+                response.end();
+              });
             })
             .catch((err) => {
-              response.writeHead(504);
-              response.end(`${err.name}: ${err.message}`);
+              setTimeout(() => {
+                throwError(err, response);
+              });
             });
         } else {
           response.writeHead(400);
@@ -538,7 +586,7 @@ export function listener(
       }
     } else if (TASKS_REGEX.test(urlParts.pathname)) {
       const r = TASKS_REGEX.exec(urlParts.pathname);
-      const taskId = querystring.unescape(r[1]);
+      const taskId = decodeURIComponent(r[1]);
       const action = r[2];
       if (!action || action === "/") {
         if (request.method === "DELETE") {
@@ -624,7 +672,7 @@ export function listener(
         response.end();
       }
     } else if (FILES_REGEX.test(urlParts.pathname)) {
-      const filename = querystring.unescape(
+      const filename = decodeURIComponent(
         FILES_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
@@ -634,9 +682,9 @@ export function listener(
           productClass: request.headers.productclass,
           version: request.headers.version,
         };
-        filesBucket.delete((filename as unknown) as ObjectId, () => {
+        filesBucket.delete(filename as unknown as ObjectId, () => {
           const uploadStream = filesBucket.openUploadStreamWithId(
-            filename,
+            filename as unknown as ObjectId,
             filename,
             {
               metadata: metadata,
@@ -653,7 +701,7 @@ export function listener(
           });
         });
       } else if (request.method === "DELETE") {
-        filesBucket.delete((filename as unknown) as ObjectId, (err) => {
+        filesBucket.delete(filename as unknown as ObjectId, (err) => {
           if (err) {
             if (err.message.startsWith("FileNotFound")) {
               response.writeHead(404);
@@ -671,7 +719,7 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (PING_REGEX.test(urlParts.pathname)) {
-      const host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1]);
+      const host = decodeURIComponent(PING_REGEX.exec(urlParts.pathname)[1]);
       ping(host, (err, res, stdout) => {
         if (err) {
           if (!res) {
@@ -697,7 +745,7 @@ export function listener(
         return;
       }
 
-      const deviceId = querystring.unescape(
+      const deviceId = decodeURIComponent(
         DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1]
       );
       apiFunctions
@@ -776,7 +824,14 @@ export function listener(
       const cur = collection.find(q, { projection: projection });
 
       if (urlParts.query.sort) {
-        const s = JSON.parse(urlParts.query.sort as string);
+        let s;
+        try {
+          s = JSON.parse(urlParts.query.sort as string);
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
         const sort = {};
         for (const [k, v] of Object.entries(s)) {
           if (k[k.lastIndexOf(".") + 1] !== "_" && collectionName === "devices")
@@ -787,13 +842,7 @@ export function listener(
         cur.sort(sort);
       }
 
-      if (urlParts.query.skip)
-        cur.skip(parseInt(urlParts.query.skip as string));
-
-      if (urlParts.query.limit)
-        cur.limit(parseInt(urlParts.query.limit as string));
-
-      cur.count(false, (err, total) => {
+      cur.count((err, total) => {
         if (err) return void throwError(err);
 
         response.writeHead(200, {
@@ -806,6 +855,12 @@ export function listener(
           return;
         }
 
+        if (urlParts.query.skip)
+          cur.skip(parseInt(urlParts.query.skip as string));
+
+        if (urlParts.query.limit)
+          cur.limit(parseInt(urlParts.query.limit as string));
+
         response.write("[\n");
         i = 0;
         cur.forEach(

lib/ping.ts

@@ -39,16 +39,20 @@ export function ping(
   switch (platform()) {
     case "linux":
       cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
-      parseRegExp1 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
-      parseRegExp2 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
+      parseRegExp1 =
+        /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+      parseRegExp2 =
+        /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
       break;
 
     case "freebsd":
       // Send a single packet because on FreeBSD only superuser can send
       // packets that are only 200 ms apart.
       cmd = `ping -t 1 -c 3 ${host}`;
-      parseRegExp1 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
-      parseRegExp2 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
+      parseRegExp1 =
+        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
+      parseRegExp2 =
+        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
       break;
 
     default:

lib/session.ts

@@ -59,6 +59,7 @@ import {
   GetParameterValuesResponse,
 } from "./types";
 import { getRequestOrigin } from "./forwarded";
+import * as logger from "./logger";
 
 const VALID_PARAM_TYPES = new Set([
   "xsd:int",
@@ -135,7 +136,7 @@ export function configContextCallback(
     const path = paths.get(Path.parse(name));
     if (path) {
       const attrs = deviceData.attributes.get(path, 1);
-      if (attrs && attrs.value && attrs.value[1]) return attrs.value[1][0];
+      if (attrs?.value?.[1]) return attrs.value[1][0];
     }
   } else if (exp[0] === "FUNC") {
     if (exp[1] === "REMOTE_ADDRESS")
@@ -298,7 +299,7 @@ export async function transferComplete(
   if (!sessionContext.operationsTouched) sessionContext.operationsTouched = {};
   sessionContext.operationsTouched[commandKey] = 1;
 
-  if (rpcReq.faultStruct && rpcReq.faultStruct.faultCode !== "0") {
+  if (rpcReq.faultStruct?.faultCode !== "0") {
     revertDownloadParameters(sessionContext, operation.args.instance);
 
     const fault: Fault = {
@@ -417,13 +418,7 @@ function revertDownloadParameters(
   );
 
   const toClear = device.set(sessionContext.deviceData, p, timestamp, {
-    value: [
-      timestamp,
-      [
-        lastDownload && lastDownload.value[1] ? lastDownload.value[1][0] : 0,
-        "xsd:dateTime",
-      ],
-    ],
+    value: [timestamp, [lastDownload?.value[1]?.[0] || 0, "xsd:dateTime"]],
   });
 
   if (toClear) {
@@ -457,22 +452,50 @@ export async function timeoutOperations(
         (e) => configContextCallback(sessionContext, e)
       ) * 1000;
 
-    if (sessionContext.timestamp > operation.timestamp + DOWNLOAD_TIMEOUT) {
-      delete sessionContext.operations[commandKey];
-      if (!sessionContext.operationsTouched)
-        sessionContext.operationsTouched = {};
-      sessionContext.operationsTouched[commandKey] = 1;
+    if (sessionContext.timestamp < operation.timestamp + DOWNLOAD_TIMEOUT)
+      continue;
 
-      faults.push({
-        code: "timeout",
-        message: "Download operation timed out",
-        timestamp: operation.timestamp,
-      });
+    logger.accessWarn({
+      sessionContext: sessionContext,
+      message: "Download operation timed out",
+      commandKey: commandKey,
+    });
 
-      operations.push(operation);
+    const SUCCESS_ON_TIMEOUT = +localCache.getConfig(
+      sessionContext.cacheSnapshot,
+      "cwmp.downloadSuccessOnTimeout",
+      {},
+      sessionContext.timestamp,
+      (e) => configContextCallback(sessionContext, e)
+    );
 
-      revertDownloadParameters(sessionContext, operation.args.instance);
+    if (SUCCESS_ON_TIMEOUT) {
+      const r = {
+        name: "TransferComplete" as const,
+        commandKey: commandKey,
+        startTime: 0,
+        completeTime: 0,
+      };
+
+      // Call transferComplete code and ignore the response
+      await transferComplete(sessionContext, r);
+      continue;
     }
+
+    delete sessionContext.operations[commandKey];
+    if (!sessionContext.operationsTouched)
+      sessionContext.operationsTouched = {};
+    sessionContext.operationsTouched[commandKey] = 1;
+
+    faults.push({
+      code: "timeout",
+      message: "Download operation timed out",
+      timestamp: operation.timestamp,
+    });
+
+    operations.push(operation);
+
+    revertDownloadParameters(sessionContext, operation.args.instance);
   }
 
   return { faults, operations };
@@ -1012,7 +1035,7 @@ export async function rpcRequest(
   if (
     !sessionContext.virtualParameters.length &&
     !sessionContext.declarations.length &&
-    !(_declarations && _declarations.length) &&
+    !_declarations?.length &&
     !sessionContext.provisions.length
   )
     return { fault: null, rpcId: null, rpc: null };
@@ -1087,12 +1110,11 @@ export async function rpcRequest(
     return rpcRequest(sessionContext, _declarations);
   }
 
-  if (_declarations && _declarations.length) {
+  if (_declarations?.length) {
     delete sessionContext.syncState;
     if (!sessionContext.declarations[0]) sessionContext.declarations[0] = [];
-    sessionContext.declarations[0] = sessionContext.declarations[0].concat(
-      _declarations
-    );
+    sessionContext.declarations[0] =
+      sessionContext.declarations[0].concat(_declarations);
     return rpcRequest(sessionContext, null);
   }
 
@@ -1423,7 +1445,7 @@ export async function rpcRequest(
       for (const [p, v] of sessionContext.syncState.downloadsValues) {
         const attrs = sessionContext.deviceData.attributes.get(p);
         if (attrs) {
-          if (attrs.writable && attrs.writable[1] && attrs.value) {
+          if (attrs.writable?.[1] && attrs.value) {
             const val = device.sanitizeParameterValue([v, attrs.value[1][1]]);
             if (val[0] !== attrs.value[1][0]) {
               toClear = device.set(
@@ -1664,7 +1686,7 @@ function generateGetRpcRequest(
       syncState.refreshAttributes.value.delete(path);
       // Need to check in case param is deleted or changed to object
       const attrs = sessionContext.deviceData.attributes.get(path);
-      if (attrs && attrs.object && attrs.object[1] === 0) {
+      if (attrs?.object?.[1] === 0) {
         parameterNames.push(path.toString());
         if (parameterNames.length >= GPV_BATCH_SIZE) break;
       }
@@ -1817,7 +1839,7 @@ function generateSetRpcRequest(
   for (const [k, v] of syncState.spv) {
     syncState.spv.delete(k);
     const attrs = sessionContext.deviceData.attributes.get(k);
-    const curVal = attrs.value ? attrs.value[1] : null;
+    const curVal = attrs.value?.[1];
     if (curVal && canWrite(attrs)) {
       const val = v.slice() as [string | number | boolean, string];
       if (!val[1]) val[1] = curVal[1];
@@ -1881,8 +1903,9 @@ function generateSetRpcRequest(
 
   // Downloads
   for (const [p, t] of syncState.downloadsDownload) {
+    if (!(t > 0 && t <= sessionContext.timestamp)) continue;
     const attrs = deviceData.attributes.get(p);
-    if (!(attrs && attrs.value && t <= attrs.value[1][0])) {
+    if (!(t <= attrs?.value?.[1]?.[0])) {
       const fileTypeAttrs = deviceData.attributes.get(
         deviceData.paths.get(p.slice(0, -1).concat(Path.parse("FileType")))
       );
@@ -1899,42 +1922,31 @@ function generateSetRpcRequest(
         name: "Download",
         commandKey: generateRpcId(sessionContext),
         instance: p.segments[1] as string,
-        fileType: fileTypeAttrs
-          ? fileTypeAttrs.value
-            ? (fileTypeAttrs.value[1][0] as string)
-            : null
-          : null,
-        fileName: fileNameAttrs
-          ? fileNameAttrs.value
-            ? (fileNameAttrs.value[1][0] as string)
-            : null
-          : null,
-        targetFileName: targetFileNameAttrs
-          ? targetFileNameAttrs.value
-            ? (targetFileNameAttrs.value[1][0] as string)
-            : null
-          : null,
+        fileType: fileTypeAttrs?.value?.[1][0] as string,
+        fileName: fileNameAttrs?.value?.[1][0] as string,
+        targetFileName: targetFileNameAttrs?.value?.[1][0] as string,
       };
     }
   }
 
   // Reboot
-  if (syncState.reboot) {
+  if (syncState.reboot > 0 && syncState.reboot <= sessionContext.timestamp) {
     const p = sessionContext.deviceData.paths.get(Path.parse("Reboot"));
     const attrs = p ? sessionContext.deviceData.attributes.get(p) : null;
-    if (!(attrs && attrs.value && attrs.value[1][0] >= syncState.reboot)) {
+    if (!(attrs?.value?.[1][0] >= syncState.reboot)) {
       delete syncState.reboot;
       return { name: "Reboot" };
     }
   }
 
   // Factory reset
-  if (syncState.factoryReset) {
+  if (
+    syncState.factoryReset > 0 &&
+    syncState.factoryReset <= sessionContext.timestamp
+  ) {
     const p = sessionContext.deviceData.paths.get(Path.parse("FactoryReset"));
     const attrs = p ? sessionContext.deviceData.attributes.get(p) : null;
-    if (
-      !(attrs && attrs.value && attrs.value[1][0] >= syncState.factoryReset)
-    ) {
+    if (!(attrs?.value?.[1][0] >= syncState.factoryReset)) {
       delete syncState.factoryReset;
       return { name: "FactoryReset" };
     }
@@ -2002,7 +2014,7 @@ function generateSetVirtualParameterProvisions(
   let provisions;
   if (virtualParameterDeclarations) {
     for (const declaration of virtualParameterDeclarations) {
-      if (declaration[2] && declaration[2].value != null) {
+      if (declaration[2]?.value != null) {
         const attrs = sessionContext.deviceData.attributes.get(declaration[0]);
         if (
           attrs &&
@@ -2131,9 +2143,7 @@ function processDeclarations(
 
     if (currentAttributes) {
       leafParam = currentPath;
-      leafIsObject = currentAttributes.object
-        ? currentAttributes.object[1]
-        : null;
+      leafIsObject = currentAttributes.object?.[1];
       // Possible V8 bug causes null === 0
       if (leafIsObject != null && leafIsObject === 0)
         leafTimestamp = Math.max(leafTimestamp, currentAttributes.object[0]);
@@ -2148,13 +2158,13 @@ function processDeclarations(
     ) {
       case "Reboot":
         if (currentPath.length === 1) {
-          if (declareAttributeValues && declareAttributeValues.value)
+          if (declareAttributeValues?.value)
             syncState.reboot = +new Date(declareAttributeValues.value[0]);
         }
         break;
       case "FactoryReset":
         if (currentPath.length === 1) {
-          if (declareAttributeValues && declareAttributeValues.value)
+          if (declareAttributeValues?.value)
             syncState.factoryReset = +new Date(declareAttributeValues.value[0]);
         }
         break;
@@ -2290,7 +2300,7 @@ function processDeclarations(
                     syncState.refreshAttributes.object.add(currentPath);
                   else if (currentAttributes.object[1] === 0)
                     syncState.refreshAttributes.value.add(currentPath);
-                } else {
+                } else if (attrName in syncState.refreshAttributes) {
                   syncState.refreshAttributes[attrName].add(currentPath);
                 }
               }
@@ -2420,8 +2430,15 @@ export async function rpcResponse(
   id: string,
   _rpcRes: CpeResponse
 ): Promise<Fault> {
+  function invalidResponse(message: string): Fault {
+    return {
+      code: "invalid_response",
+      message: message,
+    };
+  }
+
   if (id !== generateRpcId(sessionContext))
-    throw new Error("Request ID not recognized");
+    return invalidResponse("Request ID not recognized");
 
   ++sessionContext.rpcCount;
 
@@ -2512,47 +2529,99 @@ export async function rpcResponse(
 
   if (rpcRes.name === "GetParameterValuesResponse") {
     if (rpcReq.name !== "GetParameterValues")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
-    if (rpcRes.parameterList.length !== rpcReq.parameterNames.length) {
-      return {
-        code: `invalid_response`,
-        message:
-          "The list of parameters returned in the GetParameterValuesResponse message does not match the requested parameters",
-      };
-    }
+    const requested = new Set(rpcReq.parameterNames);
 
-    for (const p of rpcRes.parameterList) {
+    for (const [path, value, type] of rpcRes.parameterList) {
+      if (!requested.delete(path.toString())) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Unexpected parameter in response",
+          parameter: path.toString(),
+        });
+        continue;
+      }
       toClear = device.set(
         sessionContext.deviceData,
-        p[0],
+        path,
         timestamp,
         {
           object: [timestamp, 0],
-          value: [timestamp, p.slice(1) as [string | number | boolean, string]],
+          value: [timestamp, [value, type]],
         },
         toClear
       );
     }
+
+    if (requested.size) {
+      for (const p of requested) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Missing parameter in response",
+          parameter: p,
+        });
+        toClear = device.set(
+          sessionContext.deviceData,
+          Path.parse(p),
+          timestamp,
+          {
+            object: [timestamp, 0],
+            value: [timestamp, ["", "xsd:string"]],
+          },
+          toClear
+        );
+      }
+    }
   } else if (rpcRes.name === "GetParameterAttributesResponse") {
     if (rpcReq.name !== "GetParameterAttributes")
       throw new Error("Response name does not match request name");
 
-    for (const p of rpcRes.parameterList) {
+    const requested = new Set(rpcReq.parameterNames);
+
+    for (const [path, notification, accessList] of rpcRes.parameterList) {
+      if (!requested.delete(path.toString())) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Unexpected parameter in response",
+          parameter: path.toString(),
+        });
+        continue;
+      }
       toClear = device.set(
         sessionContext.deviceData,
-        p[0],
+        path,
         timestamp,
         {
-          notification: [timestamp, p[1] as number],
-          accessList: [timestamp, p[2] as string[]],
+          notification: [timestamp, notification],
+          accessList: [timestamp, accessList],
         },
         toClear
       );
     }
+
+    if (requested.size) {
+      for (const p of requested) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Missing parameter in response",
+          parameter: p,
+        });
+        toClear = device.set(
+          sessionContext.deviceData,
+          Path.parse(p),
+          timestamp,
+          {
+            notification: [timestamp, 0],
+            accessList: [timestamp, []],
+          },
+          toClear
+        );
+      }
+    }
   } else if (rpcRes.name === "GetParameterNamesResponse") {
     if (rpcReq.name !== "GetParameterNames")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     let root: Path;
     if (rpcReq.parameterPath.endsWith("."))
@@ -2609,6 +2678,14 @@ export async function rpcResponse(
     const wildcardParams: Path[] = [root.concat(wildcardPath)];
 
     for (const [path, object, writable] of rpcRes.parameterList) {
+      if (!path.toString().startsWith(rpcReq.parameterPath)) {
+        logger.accessWarn({
+          sessionContext: sessionContext,
+          message: "Unexpected parameter in response",
+          parameter: path.toString(),
+        });
+        continue;
+      }
       if (object && !rpcReq.nextLevel)
         wildcardParams.push(path.concat(wildcardPath));
 
@@ -2635,7 +2712,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "SetParameterValuesResponse") {
     if (rpcReq.name !== "SetParameterValues")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     for (const p of rpcReq.parameterList) {
       toClear = device.set(
@@ -2654,7 +2731,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "SetParameterAttributesResponse") {
     if (rpcReq.name !== "SetParameterAttributes")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     for (const p of rpcReq.parameterList) {
       let attrs;
@@ -2684,7 +2761,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "AddObjectResponse") {
     if (rpcReq.name !== "AddObject")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2695,7 +2772,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "DeleteObjectResponse") {
     if (rpcReq.name !== "DeleteObject")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2706,7 +2783,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "RebootResponse") {
     if (rpcReq.name !== "Reboot")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2717,7 +2794,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "FactoryResetResponse") {
     if (rpcReq.name !== "FactoryReset")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2728,7 +2805,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "DownloadResponse") {
     if (rpcReq.name !== "Download")
-      throw new Error("Response name does not match request name");
+      return invalidResponse("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2814,7 +2891,7 @@ export async function rpcResponse(
       sessionContext.operationsTouched[rpcReq.commandKey] = 1;
     }
   } else {
-    throw new Error("Response name not recognized");
+    return invalidResponse("Response name not recognized");
   }
 
   if (toClear) {
@@ -2855,13 +2932,9 @@ export async function rpcFault(
         (p) => [Path.parse(p.replace(/\.$/, "")), timestamp] as Clear
       );
     } else if (rpcReq.name === "SetParameterValues") {
-      toClear = (rpcReq.parameterList as [
-        string,
-        string | number | boolean,
-        string
-      ][]).map(
-        (p) => [Path.parse(p[0].replace(/\.$/, "")), timestamp] as Clear
-      );
+      toClear = (
+        rpcReq.parameterList as [string, string | number | boolean, string][]
+      ).map((p) => [Path.parse(p[0].replace(/\.$/, "")), timestamp] as Clear);
     } else if (rpcReq.name === "AddObject") {
       toClear = [[Path.parse(rpcReq.objectName.replace(/\.$/, "")), timestamp]];
     } else if (rpcReq.name === "DeleteObject") {
@@ -2898,10 +2971,10 @@ export async function deserialize(
   const sessionContext = JSON.parse(sessionContextString) as SessionContext;
 
   for (const decs of sessionContext.declarations)
-    for (const d of decs) d.path = Path.parse((d.path as unknown) as string);
+    for (const d of decs) d.path = Path.parse(d.path as unknown as string);
 
   const deviceData = initDeviceData();
-  for (const r of (sessionContext.deviceData as unknown) as any[]) {
+  for (const r of sessionContext.deviceData as unknown as any[]) {
     const path = deviceData.paths.add(Path.parse(r[0]));
 
     if (r[1]) deviceData.trackers.set(path, r[1]);

lib/soap.ts

@@ -109,8 +109,8 @@ function event(xml: Element): string[] {
 
 function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterInfoStruct")
     .map<[Path, boolean, boolean]>((e) => {
+      if (e.localName !== "ParameterInfoStruct") return null;
       let param: string, value: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -134,10 +134,20 @@ function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
         parsed = false;
       }
 
-      if (param && !param.endsWith("."))
-        return [Path.parse(param), false, parsed];
-      else return [Path.parse(param.slice(0, -1)), true, parsed];
-    });
+      try {
+        if (param && !param.endsWith("."))
+          return [Path.parse(param), false, parsed];
+        else return [Path.parse(param.slice(0, -1)), true, parsed];
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 const getValueType = memoize((str: string) => {
@@ -151,8 +161,8 @@ function parameterValueList(
   xml: Element
 ): [Path, string | number | boolean, string][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterValueStruct")
     .map<[Path, string | number | boolean, string]>((e) => {
+      if (e.localName !== "ParameterValueStruct") return null;
       let valueElement: Element, param: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -208,15 +218,24 @@ function parameterValueList(
           parsed = value;
         }
       }
-
-      return [Path.parse(param), parsed, valueType];
-    });
+      try {
+        return [Path.parse(param), parsed, valueType];
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 function parameterAttributeList(xml: Element): [Path, number, string[]][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterAttributeStruct")
     .map<[Path, number, string[]]>((e) => {
+      if (e.localName !== "ParameterAttributeStruct") return null;
       let notificationElement: Element,
         accessListElement: Element,
         param: string;
@@ -248,8 +267,18 @@ function parameterAttributeList(xml: Element): [Path, number, string[]][] {
         .filter((c) => c.localName === "string")
         .map((c) => decodeEntities(c.text));
 
-      return [Path.parse(param), notification, accessList];
-    });
+      try {
+        return [Path.parse(param), notification, accessList];
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 function GetParameterNames(methodRequest): string {
@@ -798,14 +827,13 @@ function fault(xml: Element): CpeFault {
 
 export function request(
   body: string,
-  cwmpVersion: string,
   warn: Record<string, unknown>[]
 ): SoapMessage {
   warnings = warn;
 
   const rpc = {
     id: null,
-    cwmpVersion: cwmpVersion,
+    cwmpVersion: null,
     sessionTimeout: null,
     cpeRequest: null,
     cpeFault: null,
@@ -848,7 +876,7 @@ export function request(
 
   const methodElement = bodyElement.children[0];
 
-  if (!rpc.cwmpVersion && methodElement.localName !== "Fault") {
+  if (methodElement.localName === "Inform") {
     let namespace, namespaceHref;
     for (const e of [methodElement, bodyElement, envelope]) {
       namespace = namespace || e.namespace;

lib/types.ts

@@ -477,7 +477,7 @@ export interface VirtualParameters {
 }
 
 export interface Files {
-  [name: string]: { length: number; md5: string; contentType: string };
+  [name: string]: { length: number };
 }
 
 export interface Users {

lib/ui.ts

@@ -59,8 +59,10 @@ const getAuthorizer = memoize(
   }
 );
 
-koa.on("error", async (err) => {
+koa.on("error", async (err, ctx) => {
   setTimeout(() => {
+    // Ignored errors resulting from aborted requests
+    if (ctx?.req.aborted) return;
     throw err;
   });
 });
@@ -93,7 +95,7 @@ koa.use(
 koa.use(async (ctx, next) => {
   let roles: string[] = [];
 
-  if (ctx.state.user && ctx.state.user.username) {
+  if (ctx.state.user?.username) {
     let user;
     if (ctx.state.user.authMethod === "local") {
       user = localCache.getUsers(ctx.state.configSnapshot)[
@@ -219,7 +221,8 @@ router.post("/init", async (ctx) => {
 });
 
 router.get("/", async (ctx) => {
-  const permissionSets: PermissionSet[] = ctx.state.authorizer.getPermissionSets();
+  const permissionSets: PermissionSet[] =
+    ctx.state.authorizer.getPermissionSets();
 
   let wizard = "";
   if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)

lib/ui/api-functions.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ObjectID } from "mongodb";
+import { ObjectId } from "mongodb";
 import * as db from "./db";
 import { del } from "../cache";
 import { getUsers } from "../local-cache";
@@ -38,7 +38,7 @@ async function deleteFault(id): Promise<void> {
   await Promise.all([
     db.deleteFault(id),
     channel.startsWith("task_")
-      ? db.deleteTask(new ObjectID(channel.slice(5)))
+      ? db.deleteTask(new ObjectId(channel.slice(5)))
       : null,
   ]);
 
@@ -108,7 +108,7 @@ export async function postTasks(
   const lastInform = Date.now();
   const notInSession = await awaitSessionEnd(deviceId, 30000);
   if (!notInSession) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
     return {
       connectionRequest: "Session took too long to complete",
       tasks: statuses,
@@ -120,7 +120,7 @@ export async function postTasks(
   const status = await connectionRequest(deviceId, device);
 
   if (status) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
     return {
       connectionRequest: status,
       tasks: statuses,
@@ -129,7 +129,7 @@ export async function postTasks(
 
   const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
   if (!sessionStarted) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
     return {
       connectionRequest: "No contact from CPE",
       tasks: statuses,
@@ -138,7 +138,7 @@ export async function postTasks(
 
   const sessionEnded = await awaitSessionEnd(deviceId, 120000);
   if (!sessionEnded) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
     return {
       connectionRequest: "Session took too long to complete",
       tasks: statuses,
@@ -163,7 +163,7 @@ export async function postTasks(
     }
   }
 
-  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
 
   return { connectionRequest: "OK", tasks: statuses };
 }
@@ -200,7 +200,7 @@ export function authLocal(
   return new Promise((resolve, reject) => {
     const users = getUsers(snapshot);
     const user = users[username];
-    if (!user || !user.password) return void resolve(null);
+    if (!user?.password) return void resolve(null);
     hashPassword(password, user.salt)
       .then((hash) => {
         if (hash === user.password) resolve(true);

lib/ui/api.ts

@@ -30,7 +30,6 @@ import { generateSalt, hashPassword } from "../auth";
 import { del } from "../cache";
 import Authorizer from "../common/authorizer";
 import { ping } from "../ping";
-import * as url from "url";
 import { decodeTag } from "../common";
 import { stringify as yamlStringify } from "../common/yaml";
 
@@ -509,11 +508,11 @@ router.put("/files/:id", async (ctx) => {
   }
 
   const metadata = {
-    fileType: singleParam(ctx.request.headers["metadata.filetype"]) || "",
-    oui: singleParam(ctx.request.headers["metadata.oui"]) || "",
+    fileType: singleParam(ctx.request.headers["metadata-filetype"]) || "",
+    oui: singleParam(ctx.request.headers["metadata-oui"]) || "",
     productClass:
-      singleParam(ctx.request.headers["metadata.productclass"]) || "",
-    version: singleParam(ctx.request.headers["metadata.version"]) || "",
+      singleParam(ctx.request.headers["metadata-productclass"]) || "",
+    version: singleParam(ctx.request.headers["metadata-version"]) || "",
   };
 
   const validate = authorizer.getValidator(resource, metadata);
@@ -574,12 +573,12 @@ router.post("/devices/:id/tasks", async (ctx) => {
       if (!Array.isArray(exp)) return exp;
       if (exp[0] === "PARAM") {
         const p = device[exp[1]];
-        if (p && p.value) return p.value[0];
+        if (p?.value) return p.value[0];
       } else if (exp[0] === "FUNC") {
         if (exp[1] === "REMOTE_ADDRESS") {
           for (const root of ["InternetGatewayDevice", "Device"]) {
             const p = device[`${root}.ManagementServer.ConnectionRequestURL`];
-            if (p && p.value) return url.parse(p.value[0]).host;
+            if (p?.value) return new URL(p.value[0]).hostname;
           }
           return null;
         }
@@ -659,7 +658,7 @@ router.get("/ping/:host", async (ctx) => {
         ctx.body = parsed;
       } else {
         ctx.status = 500;
-        ctx.body = `${err.name}: ${err.message}`;
+        ctx.body = err ? `${err.name}: ${err.message}` : "Unknown error";
       }
       resolve();
     });

lib/ui/db.ts

@@ -17,13 +17,13 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Db, GridFSBucket, ObjectID } from "mongodb";
+import { Db, GridFSBucket, ObjectId } from "mongodb";
 import { Script } from "vm";
 import { onConnect } from "../db";
 import * as mongodbFunctions from "../mongodb-functions";
 import * as expression from "../common/expression";
 import { QueryOptions, Expression } from "../types";
-import { Readable } from "stream";
+import { Readable, Writable } from "stream";
 import { minimize } from "../common/boolean-expression";
 
 const RESOURCE_COLLECTION = {
@@ -190,8 +190,8 @@ export async function updateDeviceTags(
   const collection = db.collection("devices");
   const object = {};
 
-  if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
-  if (pull && pull.length) object["$pullAll"] = { _tags: pull };
+  if (add?.length) object["$addToSet"] = { _tags: { $each: add } };
+  if (pull?.length) object["$pullAll"] = { _tags: pull };
 
   await collection.updateOne({ _id: deviceId }, object);
 }
@@ -208,7 +208,7 @@ function putResource(resource, id, object): Promise<void> {
 
 function deleteResource(
   resource: string,
-  id: string | ObjectID
+  id: string | ObjectId
 ): Promise<void> {
   return new Promise((resolve, reject) => {
     const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
@@ -242,7 +242,7 @@ export function putProvision(
       lineOffset: -1,
     });
   } catch (err) {
-    if (err.stack && err.stack.startsWith(`${id}:`)) {
+    if (err.stack?.startsWith(`${id}:`)) {
       return Promise.reject(
         new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`)
       );
@@ -267,7 +267,7 @@ export function putVirtualParameter(
       lineOffset: -1,
     });
   } catch (err) {
-    if (err.stack && err.stack.startsWith(`${id}:`)) {
+    if (err.stack?.startsWith(`${id}:`)) {
       return Promise.reject(
         new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`)
       );
@@ -338,13 +338,17 @@ export function putFile(
 ): Promise<void> {
   return new Promise((resolve, reject) => {
     const bucket = new GridFSBucket(db);
-    const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
-      metadata: metadata,
-    });
+    const uploadStream = bucket.openUploadStreamWithId(
+      filename as unknown as ObjectId,
+      filename,
+      {
+        metadata: metadata,
+      }
+    );
     uploadStream.on("error", reject);
     contentStream.on("error", reject);
     uploadStream.on("finish", resolve);
-    contentStream.pipe(uploadStream);
+    contentStream.pipe(uploadStream as Writable);
   });
 }
 
@@ -362,6 +366,6 @@ export function deleteFault(id: string): Promise<void> {
   return deleteResource("faults", id);
 }
 
-export function deleteTask(id: ObjectID): Promise<void> {
+export function deleteTask(id: ObjectId): Promise<void> {
   return deleteResource("tasks", id);
 }

lib/versioned-map.ts

@@ -26,10 +26,10 @@ interface Revisions<V> {
 }
 
 export default class VersionedMap<K, V> {
-  private _sizeDiff: number[];
-  private _revision: number;
-  private map: Map<K, (V | symbol)[]>;
-  public dirty: number;
+  private declare _sizeDiff: number[];
+  private declare _revision: number;
+  private declare map: Map<K, (V | symbol)[]>;
+  public declare dirty: number;
 
   public constructor() {
     this._sizeDiff = [0];

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.5",
+  "version": "1.2.7",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -19,56 +19,56 @@
     "bin": "bin"
   },
   "dependencies": {
-    "espresso-iisojs": "^1.0.3",
-    "iconv-lite": "^0.6.2",
-    "ipaddr.js": "^2.0.0",
+    "espresso-iisojs": "^1.0.4",
+    "iconv-lite": "^0.6.3",
+    "ipaddr.js": "^2.0.1",
     "jsonwebtoken": "^8.5.1",
     "koa": "^2.13.1",
     "koa-bodyparser": "^4.3.0",
-    "koa-compress": "^5.0.1",
-    "koa-jwt": "^4.0.0",
-    "koa-router": "^10.0.0",
+    "koa-compress": "^5.1.0",
+    "koa-jwt": "^4.0.1",
+    "koa-router": "^10.1.1",
     "koa-static": "^5.0.0",
     "later": "^1.2.0",
-    "mongodb": "^3.6.3",
-    "parsimmon": "^1.16.0",
+    "mongodb": "^4.1.2",
+    "parsimmon": "^1.18.0",
     "seedrandom": "^3.0.5"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^17.1.0",
+    "@rollup/plugin-commonjs": "^20.0.0",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^11.2.0",
-    "@rollup/plugin-typescript": "^8.2.0",
-    "@types/codemirror": "0.0.108",
-    "@types/jsonwebtoken": "^8.5.0",
-    "@types/koa": "^2.13.1",
-    "@types/koa-compress": "^4.0.1",
-    "@types/koa-router": "^7.4.1",
-    "@types/koa-static": "^4.0.1",
-    "@types/mithril": "^2.0.6",
-    "@types/mongodb": "^3.6.9",
-    "@types/node": "^14.14.34",
+    "@rollup/plugin-node-resolve": "^13.0.4",
+    "@rollup/plugin-typescript": "^8.2.5",
+    "@types/codemirror": "^5.60.2",
+    "@types/jsonwebtoken": "^8.5.5",
+    "@types/koa": "^2.13.4",
+    "@types/koa-compress": "^4.0.3",
+    "@types/koa-router": "^7.4.4",
+    "@types/koa-static": "^4.0.2",
+    "@types/mithril": "^2.0.8",
+    "@types/node": "^16.9.2",
     "@types/parsimmon": "^1.10.6",
-    "@types/seedrandom": "^3.0.0",
-    "@typescript-eslint/eslint-plugin": "^4.17.0",
-    "@typescript-eslint/parser": "^4.17.0",
+    "@types/seedrandom": "^3.0.1",
+    "@typescript-eslint/eslint-plugin": "^4.31.1",
+    "@typescript-eslint/parser": "^4.31.1",
     "ava": "^3.15.0",
-    "codemirror": "^5.59.4",
-    "cssnano": "^4.1.10",
-    "eslint": "^7.22.0",
-    "eslint-config-prettier": "^8.1.0",
+    "codemirror": "^5.62.3",
+    "cssnano": "^5.0.8",
+    "eslint": "^7.32.0",
+    "eslint-config-prettier": "^8.3.0",
     "esm": "^3.2.25",
     "mithril": "^2.0.4",
-    "postcss": "^8.2.8",
-    "postcss-import": "^14.0.0",
+    "postcss": "^8.3.6",
+    "postcss-import": "^14.0.2",
     "postcss-preset-env": "^6.7.0",
-    "prettier": "^2.2.1",
-    "rollup": "^2.41.2",
+    "prettier": "^2.4.1",
+    "rollup": "^2.56.3",
     "rollup-plugin-terser": "^7.0.2",
-    "svgo": "^2.2.2",
-    "ts-node": "^9.1.1",
-    "typescript": "^4.2.3",
-    "yaml": "^1.10.0"
+    "sql.js": "^1.6.1",
+    "svgo": "^2.6.1",
+    "ts-node": "^10.2.1",
+    "typescript": "^4.4.3",
+    "yaml": "^1.10.2"
   },
   "engines": {
     "node": ">=12.13.0"

test/boolean-expression.ts

@@ -0,0 +1,118 @@
+import ava, { TestInterface } from "ava";
+import { minimize, unionDiff } from "../lib/common/boolean-expression";
+import { parse, stringify } from "../lib/common/expression-parser";
+import initSqlJs from "sql.js";
+
+const test = ava as TestInterface<{
+  query: (filter: string) => Promise<Set<number>>;
+}>;
+
+const STRING_VALUES = [null, "", "a", "ab", "ab10", "ab-10"];
+const DECIMAL_VALUES = [null, 0, -10, 10];
+
+test.before(async (t) => {
+  const sql = await initSqlJs();
+  const db = new sql.Database();
+
+  t.context.query = async (filter: string): Promise<Set<number>> => {
+    const res = db.exec(`SELECT id FROM test WHERE ${filter}`);
+    if (!res.length) return new Set();
+    return new Set(res[0].values.flat());
+  };
+
+  db.run(
+    "CREATE TABLE test (id INTEGER PRIMARY KEY, string STRING, decimal DECIMAL(4,2))"
+  );
+
+  const stmt = db.prepare("INSERT INTO test (string, decimal) VALUES (?, ?)");
+  const count = STRING_VALUES.length * DECIMAL_VALUES.length;
+  for (let i = 0; i < count; ++i) {
+    const str = i % STRING_VALUES.length;
+    const dec = Math.trunc(i / STRING_VALUES.length) % DECIMAL_VALUES.length;
+    stmt.run([STRING_VALUES[str], DECIMAL_VALUES[dec]]);
+  }
+  stmt.free();
+});
+
+function setsEqual(set1: Set<number>, set2: Set<number>): boolean {
+  if (set1.size !== set2.size) return false;
+  for (const s of set1) if (!set2.has(s)) return false;
+  return true;
+}
+
+function getPermutations(...arrs: any[][]): any[][] {
+  const count = arrs.reduce((total, arr) => total * arr.length, 1);
+  const res = [];
+  for (let i = 0; i < count; ++i) {
+    let j = i;
+    const row = [];
+    for (const arr of arrs) {
+      const v = arr[j % arr.length];
+      j = Math.trunc(j / arr.length);
+      row.push(v);
+    }
+    res.push(row);
+  }
+  return res;
+}
+
+test("minimize", async (t) => {
+  const cases: string[] = [];
+
+  cases.push("null");
+  cases.push("false");
+  cases.push("true");
+  cases.push("string");
+  cases.push("(string + decimal) IS NULL");
+  cases.push("(string + decimal) = NULL");
+  cases.push("COALESCE(string, decimal) = 0");
+
+  for (const [s1, s2, s3, op1, op2] of getPermutations(
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s),
+    [">", "=", "<"],
+    ["<>", ">="]
+  )) {
+    cases.push(
+      `string ${op1} "${s1}" OR string ='${s2}' OR NOT string ${op2} '${s3}'`
+    );
+  }
+
+  for (const [s1, s2] of getPermutations(
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s)
+  ))
+    cases.push(`string > "${s1}" AND string < '${s2}'`);
+
+  for (const c of cases) {
+    const res1 = await t.context.query(c);
+    const min = stringify(minimize(parse(c), true));
+    const res2 = await t.context.query(min);
+    t.true(setsEqual(res1, res2));
+  }
+});
+
+test("unionDiff", async (t) => {
+  const cases = [
+    "true",
+    "decimal > 0",
+    "decimal > 10",
+    "UPPER(string || decimal) LIKE 'AB10'",
+    "COALESCE(string, decimal) = 0",
+  ];
+
+  for (const [c1, c2] of getPermutations(cases, cases)) {
+    const res1 = await t.context.query(c1);
+    const res2 = await t.context.query(c2);
+    const [union, diff] = unionDiff(parse(c1), parse(c2));
+    const res3 = await t.context.query(stringify(union));
+    const res4 = await t.context.query(stringify(diff));
+
+    const unionSet = new Set([...res1, ...res2]);
+    const diffSet = new Set(Array.from(res2).filter((r) => !res1.has(r)));
+
+    t.true(setsEqual(res3, unionSet));
+    t.true(setsEqual(res4, diffSet));
+  }
+});

test/mongodb-functions.ts

@@ -49,8 +49,16 @@ ava("convertOldPrecondition", (t) => {
   ];
 
   t.plan(tests.length + 2 * shouldFailTests.length);
-  for (const test of tests)
-    t.is(stringify(mongodbFunctions.convertOldPrecondition(test[0])), test[1]);
+  for (const test of tests) {
+    t.is(
+      stringify(
+        mongodbFunctions.convertOldPrecondition(
+          test[0] as Record<string, unknown>
+        )
+      ),
+      test[1]
+    );
+  }
 
   for (const test of shouldFailTests) {
     const func = (): void => {

test/path.ts

@@ -1,6 +1,25 @@
 import ava from "ava";
 import Path from "../lib/common/path";
 
+ava("parse", (t) => {
+  t.throws(() => Path.parse("."));
+  t.throws(() => Path.parse("a "));
+  t.throws(() => Path.parse(".a"));
+  t.throws(() => Path.parse("a."));
+  t.throws(() => Path.parse("a..b"));
+  t.throws(() => Path.parse("b*"));
+  t.throws(() => Path.parse("*b"));
+  t.throws(() => Path.parse("a.b c.d"));
+  t.throws(() => Path.parse("a["));
+  t.throws(() => Path.parse("a[b"));
+  t.throws(() => Path.parse("a[b:"));
+  t.throws(() => Path.parse('a[b:"waef]'));
+  t.notThrows(() => Path.parse("[]"));
+  t.notThrows(() => Path.parse("[a:]"));
+  t.notThrows(() => Path.parse("*"));
+  t.notThrows(() => Path.parse(""));
+});
+
 ava("toString", (t) => {
   const path1 = Path.parse('abc.[ abc : 123 , 123 : abc , 123: " abc "].123');
   const path2 = Path.parse('abc.[123:  " abc ",abc:123,123:abc].123');

tsconfig.json

@@ -1,9 +1,9 @@
 {
   "compilerOptions": {
-    "lib": ["esnext", "dom"],
-    "module": "esnext",
+    "lib": ["ES2019", "dom"],
+    "module": "ES2020",
     "moduleResolution": "node",
-    "target": "esnext",
+    "target": "ES2019",
     "resolveJsonModule": true,
     "allowSyntheticDefaultImports": true,
     "alwaysStrict": true,

ui/app.ts

@@ -69,7 +69,7 @@ function pagify(pageName, page): RouteResolver {
     render: () => {
       const lastRenderTimestamp = Date.now();
       let p;
-      if (state && state.error) p = m(errorPage.component, state);
+      if (state?.error) p = m(errorPage.component, state);
       else p = m(contextifyComponent(page.component), state);
       const attrs = {};
       attrs["page"] = pageName;

ui/autocomplete-compnent.ts

@@ -23,13 +23,13 @@ type AutocompleteCallback = (
 ) => void;
 
 export default class Autocomplete {
-  private callback: AutocompleteCallback;
-  private element: HTMLInputElement;
-  private hideTimeout: NodeJS.Timeout;
-  private visible: boolean;
-  private default: string;
-  private selection: number;
-  private container: HTMLElement;
+  private declare callback: AutocompleteCallback;
+  private declare element: HTMLInputElement;
+  private declare hideTimeout: NodeJS.Timeout;
+  private declare visible: boolean;
+  private declare default: string;
+  private declare selection: number;
+  private declare container: HTMLElement;
 
   public constructor(className: string, callback: AutocompleteCallback) {
     this.callback = callback;
@@ -51,11 +51,8 @@ export default class Autocomplete {
 
     el.addEventListener("focus", () => {
       this.element = el;
-      const domRect = el.getBoundingClientRect();
-      this.container.style.left = `${domRect.left + window.pageXOffset}px`;
-      this.container.style.width = `${domRect.width}px`;
-      this.container.style.top = `${domRect.bottom + window.pageYOffset}px`;
       this.update();
+      this.reposition();
     });
 
     el.addEventListener("blur", () => {
@@ -71,7 +68,8 @@ export default class Autocomplete {
       } else if (e.key === "Enter") {
         if (this.default != null) {
           el.value = this.default;
-          e.preventDefault();
+          e.stopImmediatePropagation();
+          el.dispatchEvent(new InputEvent("input"));
           this.update();
         }
       } else if (e.key === "ArrowDown") {
@@ -93,6 +91,19 @@ export default class Autocomplete {
     });
   }
 
+  public reposition(): void {
+    if (!this.element) return;
+    const domRect = this.element.getBoundingClientRect();
+    if (!domRect.width) {
+      // Element has been removed
+      if (this.visible) this.hide();
+      return;
+    }
+    this.container.style.left = `${domRect.left + window.pageXOffset}px`;
+    this.container.style.width = `${domRect.width}px`;
+    this.container.style.top = `${domRect.bottom + window.pageYOffset}px`;
+  }
+
   private hide(): void {
     this.container.style.opacity = "0";
     this.visible = false;
@@ -159,6 +170,7 @@ export default class Autocomplete {
         e.addEventListener("mousedown", (ev) => {
           ev.preventDefault();
           el.value = suggestion.value;
+          el.dispatchEvent(new InputEvent("input"));
           if (this.element === el) this.update();
         });
         this.container.appendChild(e);

ui/change-password-component.ts

@@ -17,17 +17,23 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { VnodeDOM, ClosureComponent, Component } from "mithril";
+import { VnodeDOM, ClosureComponent } from "mithril";
 import * as notifications from "./notifications";
 import { changePassword } from "./store";
 import { m } from "./components";
 
-const component: ClosureComponent = (): Component => {
+interface Attrs {
+  noAuth?: boolean;
+  username?: string;
+  onPasswordChange: () => void;
+}
+
+const component: ClosureComponent<Attrs> = () => {
   return {
     view: (vnode) => {
-      const onPasswordChange = vnode.attrs["onPasswordChange"];
-      const enforceAuth = !vnode.attrs["noAuth"];
-      const username = vnode.attrs["username"];
+      const onPasswordChange = vnode.attrs.onPasswordChange;
+      const enforceAuth = !vnode.attrs.noAuth;
+      const username = vnode.attrs.username;
 
       if (username) vnode.state["username"] = username;
 

ui/code-editor-component.ts

@@ -17,44 +17,54 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ClosureComponent, Component } from "mithril";
+import { Editor } from "codemirror";
+import { ClosureComponent } from "mithril";
 import { m } from "./components";
 import { codeMirror } from "./dynamic-loader";
 
-const component: ClosureComponent = (): Component => {
+interface Attrs {
+  id: string;
+  value: string;
+  mode: string;
+  readOnly?: boolean;
+  focus?: boolean;
+  onSubmit?: (dom: Element) => void;
+  onChange?: (value: string) => void;
+  onReady?: (editor: Editor) => void;
+}
+
+const component: ClosureComponent<Attrs> = () => {
   return {
     view: (vnode) => {
       return m("textarea", {
-        name: vnode.attrs["id"],
-        value: vnode.attrs["value"],
+        name: vnode.attrs.id,
+        value: vnode.attrs.value,
         oncreate: (_vnode) => {
           const editor = codeMirror.fromTextArea(
             _vnode.dom as HTMLTextAreaElement,
             {
-              mode: vnode.attrs["mode"],
+              mode: vnode.attrs.mode,
               lineNumbers: true,
-              readOnly: vnode.attrs["readOnly"],
+              readOnly: vnode.attrs.readOnly,
               extraKeys: {
                 "Ctrl-Enter": () => {
-                  if (vnode.attrs["onSubmit"])
-                    vnode.attrs["onSubmit"](_vnode.dom);
+                  if (vnode.attrs.onSubmit) vnode.attrs.onSubmit(_vnode.dom);
                 },
                 "Cmd-Enter": () => {
-                  if (vnode.attrs["onSubmit"])
-                    vnode.attrs["onSubmit"](_vnode.dom);
+                  if (vnode.attrs.onSubmit) vnode.attrs.onSubmit(_vnode.dom);
                 },
               },
             }
           );
 
-          if (vnode.attrs["onChange"]) {
+          if (vnode.attrs.onChange) {
             editor.on("change", (e) => {
-              vnode.attrs["onChange"](e.getValue());
+              vnode.attrs.onChange(e.getValue());
             });
           }
 
-          if (vnode.attrs["focus"]) editor.focus();
-          if (vnode.attrs["onReady"]) vnode.attrs["onReady"](editor);
+          if (vnode.attrs.focus) editor.focus();
+          if (vnode.attrs.onReady) vnode.attrs.onReady(editor);
         },
       });
     },

ui/components.ts

@@ -22,7 +22,7 @@ import m, {
   Attributes,
   Children,
   ComponentTypes,
-  Lifecycle,
+  CommonAttributes,
   ClosureComponent,
   Vnode,
 } from "mithril";
@@ -81,7 +81,7 @@ interface MC extends Static {
     <Attrs, State>(
       ctx: Attributes,
       component: ComponentTypes<Attrs, State>,
-      attributes: Attrs & Lifecycle<Attrs, State> & { key?: string | number },
+      attributes: Attrs & CommonAttributes<Attrs, State>,
       ...args: Children[]
     ): Vnode<Attrs, State>;
   };
@@ -116,7 +116,7 @@ function applyContext(vnode, parentContext): void {
       vnodeContext.set(vnode, vc);
       vnode.attrs = Object.assign({}, vc, vnode.attrs);
     }
-    if (vnode.children && vnode.children.length)
+    if (vnode.children?.length)
       for (const c of vnode.children) applyContext(c, vc);
   }
 }
@@ -133,7 +133,7 @@ export function contextifyComponent(component: ComponentTypes): ComponentTypes {
         applyContext(res, context);
         return res;
       };
-    } else if (!component.prototype || !component.prototype.view) {
+    } else if (!component.prototype?.view) {
       c = (initialNode) => {
         const state = (component as ClosureComponent)(initialNode);
         const view = state.view;
@@ -144,7 +144,14 @@ export function contextifyComponent(component: ComponentTypes): ComponentTypes {
             applyContext(res, context);
             return res;
           } catch (err) {
-            return m("p.error", { title: err.message }, "Error!");
+            return m(
+              "p.error",
+              {
+                title: "Click to print stack trace to console",
+                onclick: () => console.error(err),
+              },
+              "Error!"
+            );
           }
         };
         return state;

ui/components/all-parameters.ts

@@ -85,7 +85,7 @@ const component: ClosureComponent = (): Component => {
         let c = 0;
         for (const k of keys) {
           const p = device[k];
-          const str = p.value && p.value[0] ? `${k} ${p.value[0]}` : k;
+          const str = p.value?.[0] ? `${k} ${p.value[0]}` : k;
           if (re && !re.test(str)) continue;
           ++c;
           if (count < limit) filteredKeys.push(k);

ui/components/parameter.ts

@@ -38,7 +38,7 @@ const evaluateParam = memoize((exp, obj, now: number) => {
       ) {
         let v = null;
         const p = obj[e[i][1]];
-        if (p && p.value) {
+        if (p?.value) {
           v = p.value[0];
           timestamp = Math.min(timestamp, p.valueTimestamp);
         }
@@ -58,7 +58,7 @@ const evaluateParam = memoize((exp, obj, now: number) => {
     value = exp;
   } else if (exp[0] === "PARAM") {
     const p = obj[exp[1]];
-    if (p && p.value) {
+    if (p?.value) {
       timestamp = p.valueTimestamp;
       value = p.value[0];
       parameter = exp[1];
@@ -78,13 +78,13 @@ const component: ClosureComponent = (): Component => {
       const { value, timestamp, parameter } = evaluateParam(
         vnode.attrs["parameter"],
         vnode.attrs["device"],
-        store.getTimestamp()
+        store.getTimestamp() + store.getClockSkew()
       );
 
       if (value == null) return null;
 
       let edit;
-      if (device[parameter] && device[parameter].writable) {
+      if (device[parameter]?.writable) {
         edit = m(
           "button",
           {
@@ -117,7 +117,7 @@ const component: ClosureComponent = (): Component => {
             e.redraw = false;
             // Don't update any child element
             if (e.target === (el as VnodeDOM).dom) {
-              const now = Date.now();
+              const now = Date.now() + store.getClockSkew();
               const localeString = new Date(timestamp).toLocaleString();
               e.target.title = `${localeString} (${timeAgo(now - timestamp)})`;
             }

ui/config-functions.ts

@@ -45,7 +45,7 @@ export function flattenConfig(config: Record<string, unknown>): any {
 // Order keys such that nested objects come last
 function orderKeys(config: any): number {
   let res = 1;
-  if (typeof config !== "object") return res;
+  if (config == null || typeof config !== "object") return res;
   if (Array.isArray(config)) {
     for (const c of config) res += orderKeys(c);
     return res;
@@ -79,7 +79,7 @@ export function structureConfig(config: Config[]): any {
     let ref = _config;
     while (keys.length > 1) {
       const k = keys.shift();
-      if (typeof ref[k] !== "object") ref[k] = {};
+      if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
       ref = ref[k];
     }
     ref[keys[0]] = c.value;
@@ -137,7 +137,7 @@ export function diffConfig(
   };
 
   for (const [k, v] of Object.entries(target))
-    if (!current[k] || current[k] !== v) diff.add.push({ _id: k, value: v });
+    if (v && current[k] !== v) diff.add.push({ _id: k, value: v });
 
   for (const k of Object.keys(current)) if (!target[k]) diff.remove.push(k);
 

ui/config-page.ts

@@ -154,7 +154,7 @@ function renderTable(confsResponse, searchString): Children {
               {
                 base: conf,
                 actionHandler: (action, object) => {
-                  return new Promise((resolve) => {
+                  return new Promise<void>((resolve) => {
                     putActionHandler(action, object, false)
                       .then((errors) => {
                         const ErrorList = errors ? Object.values(errors) : [];
@@ -164,11 +164,11 @@ function renderTable(confsResponse, searchString): Children {
                         } else {
                           overlay.close(cb);
                         }
-                        resolve(null);
+                        resolve();
                       })
                       .catch((err) => {
                         notifications.push("error", err.message);
-                        resolve(null);
+                        resolve();
                       });
                   });
                 },
@@ -322,7 +322,7 @@ export const component: ClosureComponent = (): Component => {
                     uiConfigComponent,
                     Object.assign(
                       {
-                        onUpdate: (errs: string[]) => {
+                        onUpdate: (errs: Record<string, string>) => {
                           const errors = errs ? Object.values(errs) : [];
                           if (errors.length) {
                             for (const err of errors)

ui/css/app.css

@@ -787,3 +787,13 @@ span.parameter-value {
     background-size: 100% 100%;
   }
 }
+
+.progress {
+  background-color: #eee;
+  overflow: hidden;
+  height: 3px;
+  & > .progress-bar {
+    height: 100%;
+    background-color: #b12d5c;
+  }
+}

ui/devices-page.ts

@@ -328,10 +328,11 @@ export const component: ClosureComponent = (): Component => {
       if (window.authorizer.hasAccess("devices", 3))
         attrs["actionsCallback"] = renderActions;
 
-      const filterAttrs = {};
-      filterAttrs["resource"] = "devices";
-      filterAttrs["filter"] = vnode.attrs["filter"];
-      filterAttrs["onChange"] = onFilterChanged;
+      const filterAttrs = {
+        resource: "devices",
+        filter: vnode.attrs["filter"],
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing devices"),

ui/faults-page.ts

@@ -200,10 +200,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
-      filterAttrs["resource"] = "faults";
-      filterAttrs["filter"] = vnode.attrs["filter"];
-      filterAttrs["onChange"] = onFilterChanged;
+      const filterAttrs = {
+        resource: "faults",
+        filter: vnode.attrs["filter"],
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing faults"),

ui/files-page.ts

@@ -73,58 +73,32 @@ const unpackSmartQuery = memoize((query) => {
   });
 });
 
-interface ValidationErrors {
-  [prop: string]: string;
-}
-
-function putActionHandler(action, _object): Promise<ValidationErrors> {
-  return new Promise((resolve, reject) => {
-    const object = Object.assign({}, _object);
-    if (action === "save") {
-      const file = object["file"] ? object["file"][0] : null;
-      delete object["file"];
-
-      if (!file) return void resolve({ file: "File not selected" });
-
-      const id = file.name;
-
-      store
-        .resourceExists("files", id)
-        .then((exists) => {
-          if (exists) {
-            store.setTimestamp(Date.now());
-            return void resolve({ file: "File already exists" });
-          }
-          const headers = Object.assign(
-            {
-              "Content-Type": "application/octet-stream",
-              Accept: "application/octet-stream",
-            },
-            object
-          );
-
-          store
-            .xhrRequest({
-              method: "PUT",
-              headers: headers,
-              url: `api/files/${encodeURIComponent(id)}`,
-              serialize: (body) => body, // Identity function to prevent JSON.parse on blob data
-              body: file,
-            })
-            .then(() => {
-              notifications.push(
-                "success",
-                `File ${exists ? "updated" : "created"}`
-              );
-              store.setTimestamp(Date.now());
-              resolve(null);
-            })
-            .catch(reject);
-        })
-        .catch(reject);
-    } else {
-      reject(new Error("Undefined action"));
-    }
+function upload(
+  file: File,
+  headers: Record<string, string>,
+  abortSignal?: AbortSignal,
+  progressListener?: (e: ProgressEvent) => void
+): Promise<void> {
+  headers = Object.assign(
+    {
+      "Content-Type": "application/octet-stream",
+    },
+    headers
+  );
+  return store.xhrRequest({
+    method: "PUT",
+    headers: headers,
+    url: `api/files/${encodeURIComponent(file.name)}`,
+    serialize: (body) => body, // Identity function to prevent JSON.parse on blob data
+    body: file,
+    config: (xhr) => {
+      if (progressListener)
+        xhr.upload.addEventListener("progress", progressListener);
+      if (abortSignal) {
+        if (abortSignal.aborted) xhr.abort();
+        abortSignal.addEventListener("abort", () => xhr.abort());
+      }
+    },
   });
 }
 
@@ -224,42 +198,83 @@ export const component: ClosureComponent = (): Component => {
                 title: "Create new file",
                 onclick: () => {
                   let cb: () => Children = null;
+                  const abortController = new AbortController();
+                  let progress = -1;
                   const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
-                        actionHandler: (action, object) => {
-                          return new Promise<void>((resolve) => {
-                            putActionHandler(action, object)
-                              .then((errors) => {
-                                const errorList = errors
-                                  ? Object.values(errors)
-                                  : [];
-                                if (errorList.length) {
-                                  for (const err of errorList)
-                                    notifications.push("error", err);
-                                } else {
-                                  overlay.close(cb);
-                                }
-                                resolve();
-                              })
-                              .catch((err) => {
-                                notifications.push("error", err.message);
-                                resolve();
-                              });
-                          });
+                        actionHandler: async (action, obj) => {
+                          if (action !== "save")
+                            throw new Error("Undefined action");
+                          const file = obj["file"]?.[0];
+
+                          // nginx strips out headers with dot, so replace with dash
+                          const headers = {
+                            "metadata-fileType": obj["metadata.fileType"] || "",
+                            "metadata-oui": obj["metadata.oui"] || "",
+                            "metadata-productclass":
+                              obj["metadata.productClass"] || "",
+                            "metadata-version": obj["metadata.version"] || "",
+                          };
+
+                          if (!file) {
+                            notifications.push("error", "File not selected");
+                            return;
+                          }
+
+                          if (await store.resourceExists("files", file.name)) {
+                            store.setTimestamp(Date.now());
+                            notifications.push("error", "File already exists");
+                            return;
+                          }
+
+                          const progressListener = (e: ProgressEvent): void => {
+                            progress = e.loaded / e.total;
+                            m.redraw();
+                          };
+
+                          progress = 0;
+                          try {
+                            await upload(
+                              file,
+                              headers,
+                              abortController.signal,
+                              progressListener
+                            );
+                            store.setTimestamp(Date.now());
+                            notifications.push("success", "File created");
+                            overlay.close(cb);
+                          } catch (err) {
+                            notifications.push("error", err.message);
+                          }
+                          progress = -1;
                         },
                       },
                       formData
                     )
                   );
-                  cb = () => comp;
-                  overlay.open(
-                    cb,
-                    () =>
-                      !comp.state["current"]["modified"] ||
-                      confirm("You have unsaved changes. Close anyway?")
-                  );
+                  cb = () => {
+                    if (progress < 0) return [null, comp];
+                    return [
+                      m(
+                        "div.progress",
+                        m("div.progress-bar", {
+                          style: `width: ${Math.trunc(progress * 100)}%`,
+                        })
+                      ),
+                      comp,
+                    ];
+                  };
+                  overlay.open(cb, () => {
+                    if (
+                      comp.state["current"]["modified"] &&
+                      !confirm("You have unsaved changes. Close anyway?")
+                    )
+                      return false;
+                    abortController.abort();
+                    return true;
+                  });
                 },
               },
               "New"
@@ -301,10 +316,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
-      filterAttrs["resource"] = "files";
-      filterAttrs["filter"] = vnode.attrs["filter"];
-      filterAttrs["onChange"] = onFilterChanged;
+      const filterAttrs = {
+        resource: "files",
+        filter: vnode.attrs["filter"],
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing files"),

ui/filter-component.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import m, { ClosureComponent, Component } from "mithril";
+import m, { ClosureComponent } from "mithril";
 import { parse, stringify, map } from "../lib/common/expression-parser";
 import memoize from "../lib/common/memoize";
 import Autocomplete from "./autocomplete-compnent";
@@ -82,9 +82,9 @@ function stringifyFilter(f: Expression): string {
   return stringify(f);
 }
 
-const splitFilter = memoize((filter) => {
+function splitFilter(filter: string): string[] {
   if (!filter) return [""];
-  const list = [];
+  const list: string[] = [];
   const f = parse(filter);
   if (Array.isArray(f) && f[0] === "AND")
     for (const ff of f.slice(1)) list.push(stringifyFilter(ff));
@@ -92,69 +92,82 @@ const splitFilter = memoize((filter) => {
 
   list.push("");
   return list;
-});
+}
+
+interface Attrs {
+  resource: string;
+  filter: string;
+  onChange: (filter: string) => void;
+}
+
+const component: ClosureComponent<Attrs> = (initialVnode) => {
+  let filterList = splitFilter(initialVnode.attrs.filter);
+  let filterInvalid = 0;
+  let filterTouched = false;
+  let attrs: Attrs = initialVnode.attrs;
+
+  function onChange(): void {
+    filterTouched = false;
+    filterInvalid = 0;
+    filterList = filterList.filter((f) => f);
+    const list = filterList.map((f, idx) => {
+      try {
+        return parseFilter(attrs.resource, f);
+      } catch (err) {
+        filterInvalid |= 1 << idx;
+      }
+      return null;
+    });
+    filterList.push("");
+
+    if (filterInvalid) {
+      m.redraw();
+      return;
+    }
+    if (list.length === 0) attrs.onChange("");
+    else if (list.length > 1) attrs.onChange(stringify(["AND", ...list]));
+    else attrs.onChange(stringify(list[0]));
+  }
 
-const component: ClosureComponent = (): Component => {
   return {
+    onupdate: (vnode) => {
+      getAutocomplete(vnode.attrs.resource).reposition();
+    },
     view: (vnode) => {
-      if (
-        !vnode.state["filterList"] ||
-        vnode.attrs["filter"] !== vnode.state["filter"]
-      ) {
-        vnode.state["filterInvalid"] = 0;
-        vnode.state["filter"] = vnode.attrs["filter"];
-        vnode.state["filterList"] = splitFilter(vnode.attrs["filter"]);
+      if (attrs.filter !== vnode.attrs.filter) {
+        filterInvalid = 0;
+        filterList = splitFilter(vnode.attrs.filter);
       }
 
-      function onChange(): void {
-        vnode.state["filterInvalid"] = 0;
-        vnode.state["filterList"] = vnode.state["filterList"].filter((f) => f);
-        let filter = vnode.state["filterList"].map((f, idx) => {
-          try {
-            return parseFilter(vnode.attrs["resource"], f);
-          } catch (err) {
-            vnode.state["filterInvalid"] |= 1 << idx;
-          }
-          return null;
-        });
-        vnode.state["filterList"].push("");
+      attrs = vnode.attrs;
 
-        if (!vnode.state["filterInvalid"]) {
-          delete vnode.state["filter"];
-          if (filter.length === 0) {
-            vnode.attrs["onChange"]("");
-          } else {
-            if (filter.length > 1) filter = ["AND"].concat(filter);
-            else filter = filter[0];
-            vnode.attrs["onChange"](stringify(filter));
-          }
-        }
-      }
+      return m("div.filter", [
+        m("b", "Filter"),
+        ...filterList.map((fltr, idx) => {
+          return m("input", {
+            type: "text",
+            class: `${(filterInvalid >> idx) & 1 ? "error" : ""}`,
+            value: fltr,
+            oninput: (e) => {
+              e.redraw = false;
+              filterList[idx] = e.target.value;
+              filterTouched = true;
+            },
+            oncreate: (vn) => {
+              const el = vn.dom as HTMLInputElement;
+              getAutocomplete(vnode.attrs.resource).attach(el);
 
-      return m(
-        "div.filter",
-        [m("b", "Filter")].concat(
-          vnode.state["filterList"].map((fltr, idx) => {
-            return m("input", {
-              type: "text",
-              class: `${
-                (vnode.state["filterInvalid"] >> idx) & 1 ? "error" : ""
-              }`,
-              value: fltr,
-              onchange: (e) => {
-                vnode.state["filterList"] = vnode.state["filterList"].slice();
-                vnode.state["filterList"][idx] = e.target.value.trim();
-                onChange();
-              },
-              oncreate: (vn) => {
-                getAutocomplete(vnode.attrs["resource"]).attach(
-                  vn.dom as HTMLInputElement
-                );
-              },
-            });
-          })
-        )
-      );
+              el.addEventListener("blur", () => {
+                if (filterTouched) onChange();
+              });
+
+              el.addEventListener("keydown", (e) => {
+                if (e.key === "Enter" && filterTouched) onChange();
+              });
+            },
+          });
+        }),
+      ]);
     },
   };
 };

ui/permissions-page.ts

@@ -365,10 +365,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
-      filterAttrs["resource"] = "permissions";
-      filterAttrs["filter"] = vnode.attrs["filter"];
-      filterAttrs["onChange"] = onFilterChanged;
+      const filterAttrs = {
+        resource: "permissions",
+        filter: vnode.attrs["filter"],
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing permissions"),

ui/presets-page.ts

@@ -323,7 +323,7 @@ export const component: ClosureComponent = (): Component => {
                     return m(
                       "div",
                       { style: "margin:20px" },
-                      "This UI only supports presets with a single 'provision' configuraiton. If this preset was originally created from the old UI (genieacs-gui), you must edit it there."
+                      "This UI only supports presets with a single 'provision' configuration. If this preset was originally created from the old UI (genieacs-gui), you must edit it there."
                     );
                   }
                   return comp;
@@ -331,7 +331,7 @@ export const component: ClosureComponent = (): Component => {
                 overlay.open(
                   cb,
                   () =>
-                    !comp.state["current"]["modified"] ||
+                    !comp.state?.["current"]["modified"] ||
                     confirm("You have unsaved changes. Close anyway?")
                 );
               },
@@ -427,10 +427,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
-      filterAttrs["resource"] = "presets";
-      filterAttrs["filter"] = vnode.attrs["filter"];
-      filterAttrs["onChange"] = onFilterChanged;
+      const filterAttrs = {
+        resource: "presets",
+        filter: vnode.attrs["filter"],
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing presets"),

ui/provisions-page.ts

@@ -346,10 +346,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
-      filterAttrs["resource"] = "provisions";
-      filterAttrs["filter"] = vnode.attrs["filter"];
-      filterAttrs["onChange"] = onFilterChanged;
+      const filterAttrs = {
+        resource: "provisions",
+        filter: vnode.attrs["filter"],
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing provisions"),

ui/put-form-component.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { VnodeDOM, ClosureComponent, Component, Children } from "mithril";
+import { VnodeDOM, ClosureComponent, Children } from "mithril";
 import { m } from "./components";
 import codeEditorComponent from "./code-editor-component";
 import { getDatalistId } from "./datalist";
@@ -175,13 +175,25 @@ function createField(current, attr, focus): Children {
   });
 }
 
-const component: ClosureComponent = (): Component => {
+interface Attrs {
+  base?: Record<string, any>;
+  actionHandler: (action: string, object: any) => Promise<void>;
+  resource: string;
+  attributes: {
+    id: string;
+    label: string;
+    type?: string;
+    options?: string[];
+  }[];
+}
+
+const component: ClosureComponent<Attrs> = () => {
   return {
     view: (vnode) => {
-      const actionHandler = vnode.attrs["actionHandler"];
-      const attributes = vnode.attrs["attributes"];
-      const resource = vnode.attrs["resource"];
-      const base = vnode.attrs["base"] || {};
+      const actionHandler = vnode.attrs.actionHandler;
+      const attributes = vnode.attrs.attributes;
+      const resource = vnode.attrs.resource;
+      const base = vnode.attrs.base || {};
       if (!vnode.state["current"]) {
         vnode.state["current"] = {
           isNew: !base["_id"],
@@ -226,7 +238,7 @@ const component: ClosureComponent = (): Component => {
               onclick: (e) => {
                 e.redraw = false;
                 e.target.disabled = true;
-                actionHandler("delete", current.object).then(() => {
+                actionHandler("delete", current.object).finally(() => {
                   e.target.disabled = false;
                 });
               },
@@ -255,7 +267,7 @@ const component: ClosureComponent = (): Component => {
               // e.target.onsubmit = null;
               (submit.dom as HTMLFormElement).disabled = true;
               // submit.dom.textContent = "Loading ...";
-              actionHandler("save", current.object).then(() => {
+              actionHandler("save", current.object).finally(() => {
                 // submit.dom.textContent = "Save";
                 // e.target.onsubmit = onsubmit;
                 (submit.dom as HTMLFormElement).disabled = false;

ui/smart-query.ts

@@ -145,7 +145,7 @@ function queryTag(tag: string): Expression {
 
 export function getTip(resource: string, label: string): string {
   let tip;
-  if (resources[resource] && resources[resource][label]) {
+  if (resources[resource]?.[label]) {
     const param = resources[resource][label];
     const types =
       resource === "devices" ? param["type"] : param["type"].split(",");

ui/store.ts

@@ -34,6 +34,8 @@ const memoizedEvaluate = memoize(evaluate);
 let fulfillTimestamp = 0;
 let connectionNotification, configNotification, versionNotification;
 
+let clockSkew = 0;
+
 const queries = {
   filter: new WeakMap(),
   bookmark: new WeakMap(),
@@ -93,11 +95,13 @@ class QueryResponse {
 }
 
 function checkConnection(): void {
+  const now1 = Date.now();
   m.request({
     url: "status",
     method: "GET",
     background: true,
     extract: (xhr) => {
+      const now2 = Date.now();
       if (xhr.status !== 200) {
         if (!connectionNotification) {
           connectionNotification = notifications.push(
@@ -112,6 +116,21 @@ function checkConnection(): void {
           connectionNotification = null;
         }
 
+        try {
+          const nowAvg = Math.trunc((now1 + now2) / 2);
+          const skew = Date.parse(xhr.getResponseHeader("Date")) - nowAvg;
+          if (Math.abs(skew - clockSkew) > 5000 && now2 - now1 < 1000) {
+            clockSkew = skew;
+            console.warn(
+              `System and server clocks are out of sync. Adding ${clockSkew}ms offset to any client-side time relative calculations.`
+            );
+            setTimestamp(now2);
+            m.redraw();
+          }
+        } catch (err) {
+          // Ignore in case of missing or invalid Date header
+        }
+
         const configChanged =
           xhr.getResponseHeader("x-config-snapshot") !== configSnapshot;
         const versionChanged =
@@ -209,7 +228,7 @@ export async function xhrRequest(
 
 export function unpackExpression(exp: Expression): Expression {
   if (!Array.isArray(exp)) return exp;
-  const e = memoizedEvaluate(exp, null, fulfillTimestamp);
+  const e = memoizedEvaluate(exp, null, fulfillTimestamp + clockSkew);
   return e;
 }
 
@@ -295,7 +314,7 @@ function compareFunction(sort: {
 function findMatches(resourceType, filter, sort, limit): any[] {
   let value = [];
   for (const obj of resources[resourceType].objects.values())
-    if (evaluate(filter, obj, fulfillTimestamp)) value.push(obj);
+    if (evaluate(filter, obj, fulfillTimestamp + clockSkew)) value.push(obj);
 
   value = value.sort(compareFunction(sort));
   if (limit) value = value.slice(0, limit);
@@ -461,7 +480,7 @@ export function fulfill(accessTimestamp: number): void {
 
         for (const queryResponse of toFetch) {
           let filter = queries.filter.get(queryResponse);
-          filter = memoizedEvaluate(filter, null, fulfillTimestamp);
+          filter = memoizedEvaluate(filter, null, fulfillTimestamp + clockSkew);
           const bookmark = queries.bookmark.get(queryResponse);
           const sort = queries.sort.get(queryResponse);
           if (bookmark) filter = limitFilter(filter, sort, bookmark);
@@ -476,7 +495,11 @@ export function fulfill(accessTimestamp: number): void {
         if (!diff) {
           for (const queryResponse of toFetch) {
             let filter = queries.filter.get(queryResponse);
-            filter = memoizedEvaluate(filter, null, fulfillTimestamp);
+            filter = memoizedEvaluate(
+              filter,
+              null,
+              fulfillTimestamp + clockSkew
+            );
             const limit = queries.limit.get(queryResponse);
             const bookmark = queries.bookmark.get(queryResponse);
             const sort = queries.sort.get(queryResponse);
@@ -521,7 +544,9 @@ export function fulfill(accessTimestamp: number): void {
 
             for (const d of deleted) {
               const obj = resources[resourceType].objects.get(d);
-              if (evaluate(combinedFilterDiff, obj, fulfillTimestamp))
+              if (
+                evaluate(combinedFilterDiff, obj, fulfillTimestamp + clockSkew)
+              )
                 resources[resourceType].objects.delete(d);
             }
 
@@ -563,6 +588,10 @@ export function setTimestamp(t: number): void {
   }
 }
 
+export function getClockSkew(): number {
+  return clockSkew;
+}
+
 export function postTasks(
   deviceId: string,
   tasks: QueueTask[]
@@ -580,9 +609,8 @@ export function postTasks(
       if (xhr.status === 403) throw new Error("Not authorized");
       if (!xhr.status) throw new Error("Server is unreachable");
       if (xhr.status !== 200) throw new Error(xhr.response);
-      const connectionRequestStatus = xhr.getResponseHeader(
-        "Connection-Request"
-      );
+      const connectionRequestStatus =
+        xhr.getResponseHeader("Connection-Request");
       const st = JSON.parse(xhr.response);
       for (const [i, t] of st.entries()) {
         tasks[i]._id = t._id;
@@ -664,7 +692,7 @@ export function evaluateExpression(
   obj: Record<string, unknown>
 ): Expression {
   if (!Array.isArray(exp)) return exp;
-  return memoizedEvaluate(exp, obj, fulfillTimestamp);
+  return memoizedEvaluate(exp, obj, fulfillTimestamp + clockSkew);
 }
 
 export function changePassword(

ui/ui-config-component.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ClosureComponent, Component } from "mithril";
+import { ClosureComponent } from "mithril";
 import { m } from "./components";
 import * as store from "./store";
 import { yaml } from "./dynamic-loader";
@@ -65,7 +65,7 @@ function putActionHandler(prefix: string[], dataYaml: string): Promise<any> {
               store.putResource(
                 "config",
                 obj._id,
-                (obj as unknown) as Record<string, unknown>
+                obj as unknown as Record<string, unknown>
               )
             );
           }
@@ -86,12 +86,20 @@ function putActionHandler(prefix: string[], dataYaml: string): Promise<any> {
   });
 }
 
-const component: ClosureComponent = (): Component => {
+interface Attrs {
+  prefix: string;
+  name: string;
+  data: { _id: string; value: string }[];
+  onUpdate: (errs: Record<string, string>) => void;
+  onError: (err: Error) => void;
+}
+
+const component: ClosureComponent<Attrs> = () => {
   return {
     view: (vnode) => {
-      const prefix = vnode.attrs["prefix"].split(".");
-      const name = vnode.attrs["name"];
-      const data = vnode.attrs["data"];
+      const prefix = vnode.attrs.prefix.split(".");
+      const name = vnode.attrs.name;
+      const data = vnode.attrs.data;
 
       if (prefix[prefix.length - 1] === "") prefix.pop();
 
@@ -135,8 +143,8 @@ const component: ClosureComponent = (): Component => {
                 vnode.state["updatedYaml"] = yamlString;
 
               putActionHandler(prefix, vnode.state["updatedYaml"])
-                .then(vnode.attrs["onUpdate"])
-                .catch(vnode.attrs["onError"]);
+                .then(vnode.attrs.onUpdate)
+                .catch(vnode.attrs.onError);
             },
           },
           [code, m(".actions-bar", [submit])]

ui/users-page.ts

@@ -401,10 +401,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
-      filterAttrs["resource"] = "users";
-      filterAttrs["filter"] = vnode.attrs["filter"];
-      filterAttrs["onChange"] = onFilterChanged;
+      const filterAttrs = {
+        resource: "users",
+        filter: vnode.attrs["filter"],
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing users"),

ui/virtual-parameters-page.ts

@@ -346,10 +346,11 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {};
-      filterAttrs["resource"] = "virtualParameters";
-      filterAttrs["filter"] = vnode.attrs["filter"];
-      filterAttrs["onChange"] = onFilterChanged;
+      const filterAttrs = {
+        resource: "virtualParameters",
+        filter: vnode.attrs["filter"],
+        onChange: onFilterChanged,
+      };
 
       return [
         m("h1", "Listing virtual parameters"),