1.2.3

Some CPEs incorrectly report writable parameters as non-writable. This
config option lets us bypass the check and attempt to modify the
parameter even if it reported as read-only.

.eslintrc.json

@@ -21,6 +21,7 @@
     "@typescript-eslint/no-explicit-any": 0,
     "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
     "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
+    "@typescript-eslint/no-floating-promises": "error",
     "require-atomic-updates": 0,
     "no-prototype-builtins": 0
   }

CHANGELOG.md

@@ -1,5 +1,39 @@
 # Change Log
 
+## 1.2.3 (2020-10-26)
+
+- New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
+report writable parameters as non-writable. When set to true, the scripts will
+no longer respect the 'writable' attribute of the CPE parameters and will send
+a SetParamteerValues, AddObject, or DeleteObject request anyway.
+- Tags no longer restrict what characters are allowed. Any character other than
+alphanumeric characters, hyphen, or underscore is now encoded in the data
+model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
+- Ask for a confirmation before closing a pop-up dialog with unsaved changes.
+- Better XML validation to avoid crashes caused by invalid CPE requests.
+- Fix confusing 404 error message when the user attempts to modify a resource
+when they don't have the necessary permissions.
+- Fix a rare issue where genieacs-cwmp stops accepting new connections after
+running for a few weeks.
+- Fix exception when IS NULL operator is used in certain situations.
+
+## 1.2.2 (2020-10-03)
+
+- Added button to push files to selected devices from device listing page.
+- A few minor UI improvements.
+- Fix exception that can happen and persist after a Download request.
+- Fix validation bug preventing running refreshObject task on data model root.
+- Fix invalid arguments fault in refresh preset configuration when upgrading
+from v1.1.
+
+## 1.2.1 (2020-09-08)
+
+- Fix bug causing faults to not be displayed in the UI.
+- Fix bug where deleting objects does not get reflected immediately in the UI.
+- Improve conversion between filters written in the expression format and
+MongoDB queries. There should now be fewer edge cases where the two are not
+equisatisfiable.
+
 ## 1.2.0 (2020-09-01)
 
 - Support GetParameterAttributes and SetParameterAttributes TR-069 methods.

bin/genieacs-cwmp.ts

@@ -43,7 +43,7 @@ function exitWorkerGracefully(): void {
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -77,9 +77,16 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("CWMP_SSL_KEY") as string,
-    cert: config.get("CWMP_SSL_CERT") as string,
+  const key = config.get("CWMP_SSL_KEY") as string;
+  const cert = config.get("CWMP_SSL_CERT") as string;
+
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    onConnection: cwmp.onConnection,
+    timeout: 30000,
+    keepAliveTimeout: 0,
   };
 
   process.on("uncaughtException", (err) => {
@@ -94,14 +101,7 @@ if (!cluster.worker) {
 
   const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
-      server.start(
-        SERVICE_PORT,
-        SERVICE_ADDRESS,
-        ssl,
-        cwmp.listener,
-        cwmp.onConnection,
-        0
-      );
+      server.start(options, cwmp.listener);
     })
     .catch((err) => {
       setTimeout(() => {

bin/genieacs-fs.ts

@@ -73,9 +73,13 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("FS_SSL_KEY") as string,
-    cert: config.get("FS_SSL_CERT") as string,
+  const key = config.get("FS_SSL_KEY") as string;
+  const cert = config.get("FS_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
   let stopping = false;
@@ -98,7 +102,7 @@ if (!cluster.worker) {
 
   const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+      server.start(options, _listener);
     })
     .catch((err) => {
       setTimeout(() => {

bin/genieacs-nbi.ts

@@ -43,7 +43,7 @@ function exitWorkerGracefully(): void {
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -77,9 +77,13 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("NBI_SSL_KEY") as string,
-    cert: config.get("NBI_SSL_CERT") as string,
+  const key = config.get("NBI_SSL_KEY") as string;
+  const cert = config.get("NBI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
   let stopping = false;
@@ -102,7 +106,7 @@ if (!cluster.worker) {
 
   const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+      server.start(options, _listener);
     })
     .catch((err) => {
       setTimeout(() => {

bin/genieacs-ui.ts

@@ -45,7 +45,7 @@ function exitWorkerGracefully(): void {
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -79,9 +79,13 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("UI_SSL_KEY") as string,
-    cert: config.get("UI_SSL_CERT") as string,
+  const key = config.get("UI_SSL_KEY") as string;
+  const cert = config.get("UI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
   let stopping = false;
@@ -104,7 +108,7 @@ if (!cluster.worker) {
 
   const initPromise = Promise.all([db2.connect(), cache.connect()])
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+      server.start(options, _listener);
     })
     .catch((err) => {
       setTimeout(() => {

build/build.ts

@@ -19,14 +19,13 @@
 
 import * as path from "path";
 import * as fs from "fs";
-import { promisify } from "util";
 import { rollup, WarningHandler } from "rollup";
 import rollupJson from "@rollup/plugin-json";
 import typescript from "@rollup/plugin-typescript";
 import { terser } from "rollup-plugin-terser";
 import nodeResolve from "@rollup/plugin-node-resolve";
 import commonjs from "@rollup/plugin-commonjs";
-import webpack from "webpack";
+import webpack, { Configuration } from "webpack";
 import postcss from "postcss";
 import postcssImport from "postcss-import";
 import postcssPresetEnv from "postcss-preset-env";
@@ -81,7 +80,7 @@ const externals = [
   "codemirror/mode/yaml/yaml",
   "ipaddr.js",
   "jsbi",
-  "yebool",
+  "espresso-iisojs",
 ];
 
 function rmDirSync(dirPath): void {
@@ -297,7 +296,7 @@ async function generateFrontendJs(): Promise<void> {
   const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
   const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
 
-  const inlineDeps = ["mithril", "parsimmon", "jsbi", "yebool"];
+  const inlineDeps = ["mithril", "parsimmon", "jsbi", "espresso-iisojs"];
   const bundle = await rollup({
     input: inputFile,
     external: externals.filter((e) => !inlineDeps.includes(e)),
@@ -306,10 +305,14 @@ async function generateFrontendJs(): Promise<void> {
       {
         name: "",
         resolveId: function (importee, importer) {
-          if (importee.endsWith("/bigint"))
+          if (importee.endsWith("/bigint")) {
             return this.resolve(importee + "-jsbi", importer);
-          else if (importee === "yebool")
-            return this.resolve("yebool/dist/yebool-jsbi.mjs", importer);
+          } else if (importee === "espresso-iisojs") {
+            return this.resolve(
+              "espresso-iisojs/dist/espresso-iisojs-jsbi.mjs",
+              importer
+            );
+          }
           return null;
         },
       },
@@ -336,8 +339,8 @@ async function generateFrontendJs(): Promise<void> {
     file: outputFile,
   });
 
-  const webpackConf = {
-    mode: MODE,
+  const webpackConf: Configuration = {
+    mode: MODE as "development" | "production",
     entry: outputFile,
     resolve: {
       aliasFields: ["module"],
@@ -358,8 +361,13 @@ async function generateFrontendJs(): Promise<void> {
     },
   };
 
-  const stats = await promisify(webpack)(webpackConf);
+  return new Promise((resolve, reject) => {
+    webpack(webpackConf, (err, stats) => {
+      if (err) return reject(err);
       process.stdout.write(stats.toString({ colors: true }) + "\n");
+      resolve();
+    });
+  });
 }
 
 async function generateIconsSprite(): Promise<void> {

lib/api-functions.ts

@@ -236,6 +236,10 @@ function sanitizeTask(task): void {
       break;
 
     case "refreshObject":
+      if (typeof task.objectName !== "string")
+        throw new Error("Missing 'objectName' property");
+      break;
+
     case "deleteObject":
       if (typeof task.objectName !== "string" || !task.objectName.length)
         throw new Error("Missing 'objectName' property");

lib/common.ts

@@ -47,3 +47,17 @@ export function generateDeviceId(
 export function escapeRegExp(str: string): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
+
+export function encodeTag(tag: string): string {
+  return encodeURIComponent(tag)
+    .replace(
+      /[!~*'()]/g,
+      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
+    )
+    .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")
+    .replace(/%/g, "0x");
+}
+
+export function decodeTag(tag: string): string {
+  return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%"));
+}

lib/common/boolean-expression.ts

@@ -18,7 +18,7 @@
  */
 
 import * as BI from "./bigint";
-import { espresso, complement, tautology } from "yebool";
+import { espresso, complement, tautology } from "espresso-iisojs";
 import { Expression } from "../types";
 import { map } from "./expression-parser";
 import { and, evaluateCallback } from "./expression";
@@ -1019,8 +1019,7 @@ function boolExprSynthToExpression(boolExpr: BoolExprSynth): Expression {
   const { dcSet, isNull } = generateDcSetAndIsNull(variables);
   minterms = sanitizeMinterms(minterms, isNull);
 
-  const canRaise = (idx: number, set: Set<number>): boolean =>
-    !isNull.has(idx >> 2) || !(idx & 1) || !set.has(idx ^ 3);
+  const canRaise = getCanRaiseCallback(isNull);
   minterms = espresso(minterms, dcSet, { canRaise });
   return sopToExpression(minterms, variables);
 }

lib/common/expression-parser.ts

@@ -42,7 +42,7 @@ function interpretEscapes(str): string {
 
 export function map(
   exp: Expression,
-  callback: (exp: Expression) => Expression
+  callback: (e: Expression) => Expression
 ): Expression {
   if (!Array.isArray(exp)) return callback(exp);
 
@@ -60,7 +60,7 @@ export function map(
 
 export async function mapAsync(
   exp: Expression,
-  callback: (exp: Expression) => Promise<Expression>
+  callback: (e: Expression) => Promise<Expression>
 ): Promise<Expression> {
   if (!Array.isArray(exp)) return callback(exp);
 

lib/common/expression.ts

@@ -264,21 +264,21 @@ export function evaluateCallback(exp: Expression): Expression {
 
 export function evaluate(
   exp: Expression,
-  obj: Record<string, unknown> | ((exp: string) => Expression),
+  obj: Record<string, unknown> | ((e: string) => Expression),
   now: number,
-  cb?: (exp: Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): string | number | boolean | null;
 export function evaluate(
   exp: Expression,
-  obj?: Record<string, unknown> | ((exp: string) => Expression),
+  obj?: Record<string, unknown> | ((e: string) => Expression),
   now?: number,
-  cb?: (exp: Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): Expression;
 export function evaluate(
   exp: Expression,
-  obj?: Record<string, unknown> | ((exp: string) => Expression),
+  obj?: Record<string, unknown> | ((e: string) => Expression),
   now?: number,
-  cb?: (exp: Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): Expression {
   return map(exp, (e) => {
     if (cb) e = cb(e);
@@ -306,19 +306,19 @@ export async function evaluateAsync(
   exp: Expression,
   obj: Record<string, unknown>,
   now: number,
-  cb?: (exp: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<string | number | boolean | null>;
 export async function evaluateAsync(
   exp: Expression,
   obj?: Record<string, unknown>,
   now?: number,
-  cb?: (exp: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<Expression>;
 export async function evaluateAsync(
   exp: Expression,
   obj?: Record<string, unknown>,
   now?: number,
-  cb?: (exp: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<Expression> {
   return mapAsync(exp, async (e) => {
     if (cb) e = await cb(e);

lib/cwmp.ts

@@ -90,9 +90,7 @@ async function authenticate(
   }
 
   if (authentication && authentication.method === "Digest") {
-    const sessionNonce = sessionsNonces.get(
-      sessionContext.httpRequest.connection
-    );
+    const sessionNonce = sessionsNonces.get(sessionContext.httpRequest.socket);
 
     if (
       !sessionNonce ||
@@ -185,7 +183,7 @@ async function writeResponse(
   }
 
   const httpResponse = sessionContext.httpResponse;
-  const connection = httpResponse.connection;
+  const connection = httpResponse.socket;
 
   httpResponse.setHeader("Content-Length", Buffer.byteLength(data));
   httpResponse.writeHead(res.code, res.headers);
@@ -842,8 +840,11 @@ async function sendAcsRequest(
   return writeResponse(sessionContext, res);
 }
 
-async function getSession(connection, sessionId): Promise<SessionContext> {
-  const sessionContext = currentSessions.get(connection);
+async function getSession(
+  connection: Socket,
+  sessionId: string
+): Promise<SessionContext> {
+  let sessionContext = currentSessions.get(connection);
   if (sessionContext) {
     currentSessions.delete(connection);
     return sessionContext;
@@ -855,7 +856,9 @@ async function getSession(connection, sessionId): Promise<SessionContext> {
 
   const sessionContextString = await cache.pop(`session_${sessionId}`);
   if (!sessionContextString) return null;
-  return session.deserialize(sessionContextString);
+  sessionContext = await session.deserialize(sessionContextString);
+  connection.setTimeout(sessionContext.timeout);
+  return sessionContext;
 }
 
 // Only needed to prevent tree shaking from removing the remoteAddress
@@ -988,7 +991,7 @@ async function reportBadState(sessionContext: SessionContext): Promise<void> {
     sessionContext: sessionContext,
   });
   const httpResponse = sessionContext.httpResponse;
-  currentSessions.delete(httpResponse.connection);
+  currentSessions.delete(httpResponse.socket);
   const body = "Bad session state";
   httpResponse.setHeader("Content-Length", Buffer.byteLength(body));
   httpResponse.writeHead(400, { Connection: "close" });
@@ -1015,14 +1018,14 @@ async function responseUnauthorized(
       resHeaders["WWW-Authenticate"] = `Basic realm="${REALM}"`;
     } else {
       const nonce = crypto.randomBytes(16).toString("hex");
-      sessionsNonces.set(sessionContext.httpRequest.connection, nonce);
+      sessionsNonces.set(sessionContext.httpRequest.socket, nonce);
       let d = `Digest realm="${REALM}"`;
       d += ',qop="auth,auth-int"';
       d += `,nonce="${nonce}"`;
 
       resHeaders["WWW-Authenticate"] = d;
     }
-    currentSessions.set(sessionContext.httpRequest.connection, sessionContext);
+    currentSessions.set(sessionContext.httpRequest.socket, sessionContext);
   }
 
   const httpResponse = sessionContext.httpResponse;
@@ -1042,7 +1045,6 @@ async function processRequest(
 ): Promise<void> {
   for (const w of parseWarnings) {
     w.sessionContext = sessionContext;
-    w.rpc = rpc;
     logger.accessWarn(w);
   }
 
@@ -1070,6 +1072,8 @@ async function processRequest(
       );
     }
 
+    sessionContext.httpRequest.socket.setTimeout(sessionContext.timeout * 1000);
+
     if (sessionContext.debug) {
       debug.incomingHttpRequest(
         sessionContext.httpRequest,
@@ -1210,19 +1214,24 @@ export function listener(
       stats.concurrentRequests -= 1;
     })
     .catch((err) => {
-      currentSessions.delete(httpResponse.connection);
-      httpResponse.writeHead(500, { Connection: "close" });
-      httpResponse.end(`${err.name}: ${err.message}`);
+      currentSessions.delete(httpResponse.socket);
       stats.concurrentRequests -= 1;
       setTimeout(() => {
         throw err;
       });
+      try {
+        httpResponse.socket.unref();
+        httpResponse.writeHead(500, { Connection: "close" });
+        httpResponse.end(`${err.name}: ${err.message}`);
+      } catch (err) {
+        // Ignore
+      }
     });
 }
 
 function decodeString(buffer: Buffer, charset: string): string {
   try {
-    return buffer.toString(charset);
+    return buffer.toString(charset as BufferEncoding);
   } catch (err) {
     if (encodingExists(charset)) return decode(buffer, charset);
   }
@@ -1307,9 +1316,9 @@ async function listenerAsync(
   // Request aborted
   if (!body) return;
 
-  const newConnection = !currentSessions.has(httpRequest.connection);
+  const newConnection = !currentSessions.has(httpRequest.socket);
 
-  const sessionContext = await getSession(httpRequest.connection, sessionId);
+  const sessionContext = await getSession(httpRequest.socket, sessionId);
 
   if (sessionContext) {
     sessionContext.httpRequest = httpRequest;
@@ -1347,7 +1356,7 @@ async function listenerAsync(
     return;
   }
 
-  let charset;
+  let charset: string;
   if (httpRequest.headers["content-type"]) {
     const m = httpRequest.headers["content-type"].match(
       /charset=['"]?([^'"\s]+)/i
@@ -1409,7 +1418,7 @@ async function listenerAsync(
   }
 
   const parseWarnings = [];
-  let rpc;
+  let rpc: SoapMessage;
   try {
     rpc = soap.request(
       bodyStr,
@@ -1514,7 +1523,6 @@ async function listenerAsync(
   _sessionContext.httpRequest = httpRequest;
   _sessionContext.httpResponse = httpResponse;
   _sessionContext.sessionId = crypto.randomBytes(8).toString("hex");
-  httpRequest.connection.setTimeout(_sessionContext.timeout * 1000);
 
   const {
     tasks: dueTasks,

lib/db.ts

@@ -19,7 +19,7 @@
 
 import { MongoClient, ObjectID, Collection } from "mongodb";
 import { get } from "./config";
-import { escapeRegExp } from "./common";
+import { encodeTag, escapeRegExp } from "./common";
 import { parse } from "./common/expression-parser";
 import {
   DeviceData,
@@ -234,10 +234,9 @@ export async function fetchDevice(
           ]);
         }
 
-        for (let t of v as string[]) {
-          t = t.replace(/[^a-zA-Z0-9-]+/g, "_");
+        for (const t of v as string[]) {
           res.push([
-            Path.parse("Tags." + t),
+            Path.parse("Tags." + encodeTag(t)),
             timestamp,
             {
               object: [timestamp, 0],
@@ -691,6 +690,11 @@ export async function getOperations(
   for (const r of res) {
     const commandKey = r._id.slice(deviceId.length + 1);
     delete r._id;
+    // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
+    if (typeof r.provisions !== "string") {
+      operations[commandKey] = r;
+      continue;
+    }
     r.timestamp = +r.timestamp;
     if (r.args) r.args = JSON.parse(r.args);
     r.provisions = JSON.parse(r.provisions);
@@ -712,7 +716,7 @@ export async function saveOperation(
   o["provisions"] = JSON.stringify(operation.provisions);
   o["retries"] = JSON.stringify(operation.retries);
   o["args"] = JSON.stringify(operation.args);
-  await operationsCollection.replaceOne({ _id: id }, operation, {
+  await operationsCollection.replaceOne({ _id: id }, o, {
     upsert: true,
   });
 }

lib/debug.ts

@@ -49,7 +49,7 @@ export function incomingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
   const msg = {
     event: "incoming HTTP request",
     timestamp: now,
@@ -77,7 +77,7 @@ export function outgoingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
   const msg = {
     event: "outgoing HTTP response",
     timestamp: now,
@@ -104,7 +104,7 @@ export function outgoingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
@@ -160,13 +160,13 @@ export function incomingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
   const msg = {
     event: "incoming HTTP response",
     timestamp: now,
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
-    connection: getConnectionTimestamp(httpResponse.connection),
+    connection: getConnectionTimestamp(httpResponse.socket),
     statusCode: httpResponse.statusCode,
     headers: httpResponse.headers,
     body: body,

lib/extensions.ts

@@ -58,6 +58,7 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
             });
           }
 
+          // eslint-disable-next-line @typescript-eslint/no-floating-promises
           kill(processes[scriptName]);
           delete processes[scriptName];
         }

lib/local-cache.ts

@@ -22,7 +22,7 @@ import * as crypto from "crypto";
 import * as config from "./config";
 import * as db from "./db";
 import * as cache from "./cache";
-import { mongoQueryToFilter } from "./mongodb-functions";
+import { convertOldPrecondition } from "./mongodb-functions";
 import * as logger from "./logger";
 import * as scheduling from "./scheduling";
 import * as expression from "./common/expression";
@@ -173,7 +173,9 @@ async function fetchPresets(): Promise<Preset[]> {
       try {
         precondition = parse(preset["precondition"]);
       } catch (error) {
-        precondition = mongoQueryToFilter(JSON.parse(preset["precondition"]));
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"])
+        );
       }
 
       // Simplify expression
@@ -186,7 +188,7 @@ async function fetchPresets(): Promise<Preset[]> {
     for (const c of preset["configurations"]) {
       switch (c.type) {
         case "age":
-          _provisions.push(["refresh", c.name, c.age]);
+          _provisions.push(["refresh", c.name, +c.age]);
           break;
 
         case "value":
@@ -473,7 +475,7 @@ export function getConfig(
   key: string,
   context: Record<string, unknown>,
   now: number,
-  cb?: (Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): string | number | boolean | null {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");

lib/mongodb-functions.ts

@@ -19,8 +19,9 @@
 
 import { ObjectID } from "mongodb";
 import { map, parse, stringify, parseList } from "./common/expression-parser";
-import { likePatternToRegExp, evaluate } from "./common/expression";
+import { likePatternToRegExp } from "./common/expression";
 import { Expression, Fault, Task } from "./types";
+import { decodeTag, encodeTag } from "./common";
 
 const isArray = Array.isArray;
 
@@ -43,13 +44,11 @@ export function processDeviceFilter(filter: Expression): Expression {
       exp[1][0] === "PARAM" &&
       exp[1][1].startsWith("Tags.")
     ) {
-      const t = exp[1][1].slice(5);
-      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
-      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
-      else if (exp[0] === "=" && exp[2] === true)
-        return ["=", ["PARAM", "_tags"], t];
-      else if (exp[0] === "<>" && exp[2] !== true)
-        return ["=", ["PARAM", "_tags"], t];
+      const t = decodeTag(exp[1][1].slice(5));
+      if (exp[0] === "IS NULL") return ["_tags", t, false];
+      else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
+      else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
+      else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
     } else if (
       [
         "=",
@@ -117,140 +116,131 @@ export function processFaultsFilter(filter: Expression): Expression {
 }
 
 export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
-  const ops = {
-    OR: 0,
-    AND: 0,
-    NOT: 0,
-    "=": 1,
-    "<>": 1,
-    ">": 1,
-    ">=": 1,
-    "<": 1,
-    "<=": 1,
-    LIKE: 1,
-    "NOT LIKE": 1,
-    "IS NULL": 1,
-    "IS NOT NULL": 1,
-  };
-
-  function recursive(filter, negate, res = {}): Record<string, unknown> {
+  function recursive(filter, negate): Record<string, unknown> {
     const op = filter[0];
 
-    if (ops[op] === 0) {
-      if (op === "AND") filter = filter.filter((f) => f !== true);
-      if (op === "OR") filter = filter.filter((f) => f !== false);
-      if (filter.length === 2) return recursive(filter[1], negate, res);
-
-      for (let i = 1; i < filter.length; ++i) {
-        if (!isArray(filter[i]) || ops[filter[i][0]] == null)
-          throw new Error(`Invalid expression in ${op} clause`);
+    if (op === "AND") {
+      filter = filter.filter((f) => f !== true);
+      if (filter.length === 2) return recursive(filter[1], negate);
     }
 
-      if ((!negate && op === "AND") || (negate && op === "OR")) {
-        res["$and"] = res["$and"] || [];
-        for (let i = 1; i < filter.length; ++i)
-          res["$and"].push(recursive(filter[i], negate));
-      } else if ((!negate && op === "OR") || (negate && op === "AND")) {
-        res["$or"] = res["$or"] || [];
-
-        for (let i = 1; i < filter.length; ++i)
-          res["$or"].push(recursive(filter[i], negate));
-      } else if (op === "NOT") {
-        recursive(filter[1], !negate, res);
-      }
-    } else if (ops[op] === 1) {
-      if (isArray(filter[2])) {
-        throw new Error(`Right operand of ${op} clause is not a primitive`);
-      } else if (
-        ["LIKE", "NOT LIKE"].includes(op) &&
-        isArray(filter[1]) &&
-        filter[1][0] === "FUNC" &&
-        ["UPPER", "LOWER"].includes(filter[1][1])
-      ) {
-        if (
-          (filter[1][1] === "UPPER" && filter[2] !== filter[2].toUpperCase()) ||
-          (filter[1][1] === "LOWER" && filter[2] !== filter[2].toLowerCase())
-        ) {
-          throw new Error(
-            `Cannot compare ${
-              filter[1][1]
-            }() against non-${filter[1][1].toLowerCase()}case pattern`
-          );
-        }
-      } else if (!isArray(filter[1]) || filter[1][0] !== "PARAM") {
-        throw new Error(`Left operand of ${op} clause is not a parameter`);
+    if (op === "OR") {
+      filter = filter.filter((f) => f !== false);
+      if (filter.length === 2) return recursive(filter[1], negate);
     }
 
-      if (op === "=") {
-        const param = filter[1][1];
-        const p = (res[param] = res[param] || {});
-        if (negate) p["$ne"] = filter[2];
-        else p["$eq"] = filter[2];
-      } else if (op === "<>") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$ne"] = filter[2];
-        if (param !== "_tags") p["$exists"] = true;
-      } else if (op === ">") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gt"] = filter[2];
-      } else if (op === ">=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gte"] = filter[2];
-      } else if (op === "<") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lt"] = filter[2];
-      } else if (op === "<=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lte"] = filter[2];
-      } else if (op === "IS NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: negate };
-      } else if (op === "IS NOT NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: !negate };
-      } else {
+    if ((!negate && op === "AND") || (negate && op === "OR"))
+      return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if ((!negate && op === "OR") || (negate && op === "AND"))
+      return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if (op === "NOT") return recursive(filter[1], !negate);
+
+    if (isArray(filter[2]))
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "LIKE" || op === "NOT LIKE") {
       if (op === "NOT LIKE") negate = !negate;
       let param;
       let flags;
+      if (!Array.isArray(filter[1]))
+        throw new Error(`Invalid LHS operand of ${op} clause`);
       if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
+        if (filter[2] !== filter[2].toUpperCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
         param = filter[1][2][1];
         flags = "i";
       } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
+        if (filter[2] !== filter[2].toLowerCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
         param = filter[1][2][1];
         flags = "i";
-        } else {
+      } else if (filter[1][0] === "PARAM") {
         param = filter[1][1];
         flags = "";
+      } else {
+        throw new Error(`Invalid LHS operand of ${op} clause`);
       }
       const r = likePatternToRegExp(filter[2], filter[3], flags);
-        if (negate) res[param] = { $not: r };
-        else res[param] = r;
+      if (negate) return { [param]: { $nin: [r, null] } };
+      else return { [param]: r };
     }
-    } else {
+
+    if (op === "_tags") {
+      let t = filter[2];
+      if (negate) t = !t;
+      if (t) return { _tags: filter[1] };
+      else return { _tags: { $ne: filter[1] } };
+    }
+
+    if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
+      throw new Error(`Invalid LHS operand of ${op} clause`);
+
+    if (op === "IS NULL") {
+      const p = filter[1][1];
+      return { [p]: null };
+    }
+
+    if (op === "IS NOT NULL") {
+      const p = filter[1][1];
+      return { [p]: { $ne: null } };
+    }
+
+    if (isArray(filter[2]) || filter[2] == null)
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (!negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === "<>") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === ">") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lte: v } };
+      else return { [p]: { $gt: v } };
+    }
+
+    if (op === ">=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lt: v } };
+      else return { [p]: { $gte: v } };
+    }
+
+    if (op === "<") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gte: v } };
+      else return { [p]: { $lt: v } };
+    }
+
+    if (op === "<=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gt: v } };
+      else return { [p]: { $lte: v } };
+    }
+
     throw new Error(`Unrecognized operator ${op}`);
   }
 
-    return res;
-  }
-
-  const _exp = evaluate(exp, null, Date.now());
-
-  if (!isArray(_exp)) {
-    if (_exp === true) return {};
+  if (!isArray(exp)) {
+    if (exp === true) return {};
     throw new Error("Primitives are not valid queries");
   }
 
-  return recursive(_exp, false);
+  return recursive(exp, false);
 }
 
 export function processDeviceProjection(
@@ -417,7 +407,7 @@ export function flattenDevice(device: Record<string, unknown>): FlatDevice {
           };
 
           for (const t of tree as string[]) {
-            output[`Tags.${t}`] = {
+            output[`Tags.${encodeTag(t)}`] = {
               value: [true, "xsd:boolean"],
               valueTimestamp: timestamp,
               writable: true,
@@ -495,7 +485,9 @@ export function flattenTask(task: unknown): Task {
   return t;
 }
 
-export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
+export function convertOldPrecondition(
+  query: Record<string, unknown>
+): Expression {
   function recursive(_query): Expression {
     const expressions: Expression[] = [];
     for (const [k, v] of Object.entries(_query)) {
@@ -514,14 +506,27 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
       } else if (k === "_tags") {
         if (typeof v === "object") {
           if (isArray(v)) throw new Error(`Invalid type`);
-
-          if (v.hasOwnProperty("$ne"))
-            expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]);
-          else if (v.hasOwnProperty("$eq"))
-            expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v["$eq"]}`]]);
-          else throw new Error(`Invalid tag query`);
+          const conjs: Expression[] = [];
+          for (const [op, val] of Object.entries(v)) {
+            if (op === "$ne") {
+              if (typeof v["$ne"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
             } else {
-          expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
+              throw new Error(`Invalid tag query`);
+            }
+          }
+          if (conjs.length === 1) expressions.push(conjs[0]);
+          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
+        } else {
+          expressions.push([
+            "IS NOT NULL",
+            ["PARAM", `Tags.${encodeTag(v as string)}`],
+          ]);
         }
       } else if (k.startsWith("Tags.")) {
         let exists: boolean;
@@ -537,34 +542,31 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
 
         const exps: Expression[] = [];
         for (const [kk, vv] of Object.entries(v)) {
-          let op;
-          switch (kk) {
-            case "$eq":
-              op = "=";
-              break;
-            case "$ne":
-              op = "<>";
-              break;
-            case "$lt":
-              op = "<";
-              break;
-            case "$lte":
-              op = "<=";
-              break;
-            case "$gt":
-              op = ">";
-              break;
-            case "$gte":
-              op = ">=";
-              break;
-            default:
+          if (kk === "$eq") {
+            exps.push(["=", ["PARAM", k], vv]);
+          } else if (kk === "$ne") {
+            exps.push([
+              "OR",
+              ["<>", ["PARAM", k], vv],
+              ["IS NULL", ["PARAM", k]],
+            ]);
+          } else if (kk === "$lt") {
+            exps.push(["<", ["PARAM", k], vv]);
+          } else if (kk === "$lte") {
+            exps.push(["<=", ["PARAM", k], vv]);
+          } else if (kk === "$gt") {
+            exps.push([">", ["PARAM", k], vv]);
+          } else if (kk === "$gte") {
+            exps.push([">=", ["PARAM", k], vv]);
+          } else {
             throw new Error(`Operator ${kk} not supported`);
           }
-          exps.push([op, ["PARAM", k], vv]);
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
         }
         if (exps.length === 1) {
           expressions.push(exps[0]);
-        } else {
+        } else if (exps.length > 1) {
           const and: Expression = ["AND"];
           expressions.push(and.concat(exps));
         }
@@ -573,8 +575,8 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
       }
     }
     if (expressions.length === 1) return expressions[0];
-    const and: Expression = ["AND"];
-    return and.concat(expressions);
+    if (expressions.length === 0) return true;
+    return ["AND", ...expressions];
   }
 
   // empty filter
@@ -592,7 +594,9 @@ export function flattenPreset(
       // Try parse to check expression validity
       parse(p.precondition as string);
     } catch (error) {
-      p.precondition = mongoQueryToFilter(JSON.parse(p.precondition as string));
+      p.precondition = convertOldPrecondition(
+        JSON.parse(p.precondition as string)
+      );
       p.precondition = (p.precondition as string).length
         ? stringify(p.precondition as Expression)
         : "";

lib/server.ts

@@ -52,35 +52,43 @@ function closeServer(timeout, callback): void {
   });
 }
 
+interface ServerOptions {
+  port?: number;
+  host?: string;
+  ssl?: { key: string; cert: string };
+  timeout?: number;
+  keepAliveTimeout?: number;
+  onConnection?: (socket: Socket) => void;
+}
+
 export function start(
-  port: number,
-  networkInterface: string,
-  ssl: { key: string; cert: string },
-  _listener: http.RequestListener,
-  onConnection?: (socket: Socket) => void,
-  keepAliveTimeout = -1
+  options: ServerOptions,
+  _listener: http.RequestListener
 ): void {
   listener = _listener;
 
-  if (ssl && ssl.key && ssl.cert) {
-    const options = {
-      key: ssl.key
+  if (options.ssl) {
+    const opts = {
+      key: options.ssl.key
         .split(":")
         .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
-      cert: ssl.cert
+      cert: options.ssl.cert
         .split(":")
         .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
     };
 
-    server = https.createServer(options, listener);
-    if (onConnection != null) server.on("secureConnection", onConnection);
+    server = https.createServer(opts, listener);
+    if (options.onConnection)
+      server.on("secureConnection", options.onConnection);
   } else {
     server = http.createServer(listener);
-    if (onConnection != null) server.on("connection", onConnection);
+    if (options.onConnection) server.on("connection", options.onConnection);
   }
 
-  if (keepAliveTimeout >= 0) server.keepAliveTimeout = keepAliveTimeout;
-  server.listen(port, networkInterface);
+  server.timeout = options.timeout || 0;
+  if (options.keepAliveTimeout != null)
+    server.keepAliveTimeout = options.keepAliveTimeout;
+  server.listen({ port: options.port, host: options.host });
 }
 
 export function stop(): Promise<void> {

lib/session.ts

@@ -299,7 +299,7 @@ export async function transferComplete(
   sessionContext.operationsTouched[commandKey] = 1;
 
   if (rpcReq.faultStruct && rpcReq.faultStruct.faultCode !== "0") {
-    await revertDownloadParameters(sessionContext, operation.args.instance);
+    revertDownloadParameters(sessionContext, operation.args.instance);
 
     const fault: Fault = {
       code: `cwmp.${rpcReq.faultStruct.faultCode}`,
@@ -398,10 +398,10 @@ export async function transferComplete(
   };
 }
 
-async function revertDownloadParameters(
+function revertDownloadParameters(
   sessionContext: SessionContext,
   instance
-): Promise<void> {
+): void {
   const timestamp = sessionContext.timestamp + sessionContext.iteration + 1;
 
   let p;
@@ -1734,20 +1734,35 @@ function generateSetRpcRequest(
 
   const deviceData = sessionContext.deviceData;
 
+  const SKIP_WRITABLE_CHECK = !!localCache.getConfig(
+    sessionContext.cacheSnapshot,
+    "cwmp.skipWritableCheck",
+    {},
+    sessionContext.timestamp,
+    (e) => configContextCallback(sessionContext, e)
+  );
+
+  const canWrite = (attrs: Attributes): boolean =>
+    SKIP_WRITABLE_CHECK || (attrs.writable && !!attrs.writable[1]);
+
   // Delete instance
   for (const instances of syncState.instancesToDelete.values()) {
-    const instance = instances.values().next().value;
-    if (instance && sessionContext.deviceData.attributes.has(instance)) {
+    for (const instance of instances) {
+      const attrs = sessionContext.deviceData.attributes.get(instance);
+      if (attrs && canWrite(attrs)) {
+        instances.delete(instance);
         return {
           name: "DeleteObject",
           objectName: instance.toString() + ".",
         };
       }
     }
+  }
 
   // Create instance
   for (const [param, instances] of syncState.instancesToCreate) {
-    if (sessionContext.deviceData.attributes.has(param)) {
+    const attrs = sessionContext.deviceData.attributes.get(param);
+    if (attrs && canWrite(attrs)) {
       const instance = instances.values().next().value;
       if (instance) {
         instances.delete(instance);
@@ -1791,7 +1806,7 @@ function generateSetRpcRequest(
     syncState.spv.delete(k);
     const attrs = sessionContext.deviceData.attributes.get(k);
     const curVal = attrs.value ? attrs.value[1] : null;
-    if (curVal && attrs.writable && attrs.writable[1]) {
+    if (curVal && canWrite(attrs)) {
       const val = v.slice() as [string | number | boolean, string];
       if (!val[1]) val[1] = curVal[1];
       device.sanitizeParameterValue(val);
@@ -2609,7 +2624,6 @@ export async function rpcResponse(
         timestamp + 1,
         {
           object: [timestamp + 1, 0],
-          writable: [timestamp + 1, 1],
           value: [
             timestamp + 1,
             p.slice(1) as [string | number | boolean, string],

lib/soap.ts

@@ -127,7 +127,8 @@ function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
 
       if (parsed == null) {
         warnings.push({
-          message: "Invalid writable attribute",
+          message: "Missing or invalid XML node",
+          element: "Writable",
           parameter: param,
         });
         parsed = false;
@@ -167,7 +168,8 @@ function parameterValueList(
       let valueType = getValueType(valueElement.attrs);
       if (!valueType) {
         warnings.push({
-          message: "Invalid value type attribute",
+          message: "Missing or invalid XML node",
+          attribute: "type",
           parameter: param,
         });
         valueType = "xsd:string";
@@ -179,7 +181,8 @@ function parameterValueList(
         parsed = parseBool(value);
         if (parsed == null) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
+            element: "Value",
             parameter: param,
           });
           parsed = value;
@@ -188,7 +191,8 @@ function parameterValueList(
         parsed = parseInt(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
+            element: "Value",
             parameter: param,
           });
           parsed = value;
@@ -197,7 +201,8 @@ function parameterValueList(
         parsed = Date.parse(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
+            element: "Value",
             parameter: param,
           });
           parsed = value;
@@ -232,7 +237,8 @@ function parameterAttributeList(xml: Element): [Path, number, string[]][] {
       let notification = parseInt(notificationElement.text);
       if (isNaN(notification)) {
         warnings.push({
-          message: "Invalid notification attribute",
+          message: "Missing or invalid XML node",
+          element: "Notification",
           parameter: param,
         });
         notification = 0;
@@ -331,7 +337,10 @@ function SetParameterValuesResponse(xml: Element): SetParameterValuesResponse {
   }
 
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid SetParameterValuesResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
@@ -392,8 +401,14 @@ function AddObjectResponse(xml: Element): AddObjectResponse {
     }
   }
 
+  if (!/^[0-9]+$/.test(instanceNumber))
+    throw new Error("Missing or invalid instance number");
+
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid AddObjectResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
@@ -424,7 +439,10 @@ function DeleteObjectResponse(xml: Element): DeleteObjectResponse {
   }
 
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid DeleteObjectResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
@@ -495,10 +513,29 @@ function DownloadResponse(xml: Element): DownloadResponse {
   }
 
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid DownloadResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "DownloadResponse",
     status: status,
@@ -537,6 +574,30 @@ function Inform(xml: Element): InformRequest {
     }
   }
 
+  if (!deviceId || !deviceId.SerialNumber || !deviceId.OUI)
+    throw new Error("Missing or invalid DeviceId element");
+
+  if (!parameterList) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "ParameterList",
+    });
+    parameterList = [];
+  }
+
+  if (!evnt) {
+    warnings.push({ message: "Missing or invalid XML node", element: "Event" });
+    evnt = [];
+  }
+
+  if (retryCount == null || isNaN(retryCount)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "RetryCount",
+    });
+    retryCount = 0;
+  }
+
   return {
     name: "Inform",
     parameterList: parameterList,
@@ -563,7 +624,10 @@ function GetRPCMethodsResponse(methodResponse): string {
 }
 
 function TransferComplete(xml: Element): TransferCompleteRequest {
-  let commandKey, _faultStruct, startTime, completeTime;
+  let commandKey: string,
+    _faultStruct: FaultStruct,
+    startTime: number,
+    completeTime: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "CommandKey":
@@ -581,6 +645,38 @@ function TransferComplete(xml: Element): TransferCompleteRequest {
     }
   }
 
+  if (commandKey == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CommandKey",
+    });
+    commandKey = "";
+  }
+
+  if (!_faultStruct) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultStruct",
+    });
+    _faultStruct = { faultCode: "0", faultString: "" };
+  }
+
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "TransferComplete",
     commandKey: commandKey,
@@ -606,7 +702,12 @@ function RequestDownloadResponse(): string {
 }
 
 function faultStruct(xml: Element): FaultStruct {
-  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
+  let faultCode: string,
+    faultString: string,
+    setParameterValuesFault: SpvFault[],
+    pn: string,
+    fc: string,
+    fs: string;
   for (const c of xml.children) {
     switch (c.localName) {
       case "FaultCode":
@@ -617,6 +718,7 @@ function faultStruct(xml: Element): FaultStruct {
         break;
       case "SetParameterValuesFault":
         setParameterValuesFault = setParameterValuesFault || [];
+        pn = fc = fs = null;
         for (const cc of c.children) {
           switch (cc.localName) {
             case "ParameterName":
@@ -638,11 +740,27 @@ function faultStruct(xml: Element): FaultStruct {
     }
   }
 
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultCode",
+    });
+    faultCode = "";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultString",
+    });
+    faultString = "";
+  }
+
   return { faultCode, faultString, setParameterValuesFault };
 }
 
 function fault(xml: Element): CpeFault {
-  let faultCode, faultString, detail;
+  let faultCode: string, faultString: string, detail: FaultStruct;
   for (const c of xml.children) {
     switch (c.localName) {
       case "faultcode":
@@ -657,7 +775,25 @@ function fault(xml: Element): CpeFault {
     }
   }
 
-  return { faultCode, faultString, detail };
+  if (!detail) throw new Error("Missing detail element");
+
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultcode",
+    });
+    faultCode = "Client";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultstring",
+    });
+    faultString = "CWMP fault";
+  }
+
+  return { faultCode, faultString, detail } as CpeFault;
 }
 
 export function request(

lib/types.ts

@@ -289,8 +289,8 @@ export interface FaultStruct {
 }
 
 export interface CpeFault {
-  faultCode: string;
-  faultString: string;
+  faultCode: "Client" | "Server";
+  faultString: "CWMP fault";
   detail?: FaultStruct;
 }
 

lib/ui.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Z_SYNC_FLUSH } from "zlib";
+import { constants } from "zlib";
 import Koa from "koa";
 import Router from "koa-router";
 import * as jwt from "jsonwebtoken";
@@ -60,7 +60,9 @@ const getAuthorizer = memoize(
 );
 
 koa.on("error", async (err) => {
+  setTimeout(() => {
     throw err;
+  });
 });
 
 koa.use(async (ctx, next) => {
@@ -252,10 +254,10 @@ router.get("/", async (ctx) => {
 koa.use(
   koaCompress({
     gzip: {
-      flush: Z_SYNC_FLUSH,
+      flush: constants.Z_SYNC_FLUSH,
     },
     deflate: {
-      flush: Z_SYNC_FLUSH,
+      flush: constants.Z_SYNC_FLUSH,
     },
   })
 );

lib/ui/api-functions.ts

@@ -110,7 +110,7 @@ export async function postTasks(
   try {
     await connectionRequest(deviceId, device);
   } catch (err) {
-    for (const t of statuses) db.deleteTask(new ObjectID(t._id));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
     return {
       connectionRequest: err.message,
       tasks: statuses,
@@ -138,20 +138,11 @@ export async function postTasks(
       r.status = "fault";
       r["fault"] = res[i * 2 + 1][0];
     }
-    db.deleteTask(new ObjectID(r._id));
   }
 
-  return { connectionRequest: "OK", tasks: statuses };
-}
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
 
-interface PingResponse {
-  packetsTransmitted: number;
-  packetsReceived: number;
-  packetLoss: string;
-  min: number;
-  avg: number;
-  max: number;
-  mdev: number;
+  return { connectionRequest: "OK", tasks: statuses };
 }
 
 export async function putResource(

lib/ui/api.ts

@@ -31,6 +31,7 @@ import { del } from "../cache";
 import Authorizer from "../common/authorizer";
 import { ping } from "../ping";
 import * as url from "url";
+import { decodeTag } from "../common";
 
 const router = new Router();
 export default router;
@@ -85,7 +86,7 @@ router.get(`/devices/:id.csv`, async (ctx) => {
 
   if (!authorizer.hasAccess("devices", 2)) {
     logUnauthorizedWarning(log);
-    return void (ctx.status = 404);
+    return void (ctx.status = 403);
   }
 
   const res = await db.query("devices", filter);
@@ -126,7 +127,7 @@ router.get(`/devices/:id.csv`, async (ctx) => {
 });
 
 for (const [resource, flags] of Object.entries(resources)) {
-  router.head(`/${resource}`, async (ctx, next) => {
+  router.head(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     let filter: Expression = authorizer.getFilter(resource, 1);
     if (ctx.request.query.filter)
@@ -141,14 +142,15 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     if (!authorizer.hasAccess(resource, 1)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -161,7 +163,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.get(`/${resource}`, async (ctx, next) => {
+  router.get(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = {};
     let filter: Expression = authorizer.getFilter(resource, 2);
@@ -189,14 +191,15 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -214,7 +217,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   });
 
   // CSV download
-  router.get(`/${resource}.csv`, async (ctx, next) => {
+  router.get(`/${resource}.csv`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = { projection: {} };
     let filter: Expression = authorizer.getFilter(resource, 2);
@@ -236,7 +239,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const columns: Record<string, Expression> = JSON.parse(
@@ -254,8 +257,9 @@ for (const [resource, flags] of Object.entries(resources)) {
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -277,7 +281,7 @@ for (const [resource, flags] of Object.entries(resources)) {
                 if (e[1] === "Tags") {
                   const tags = [];
                   for (const p in obj)
-                    if (p.startsWith("Tags.")) tags.push(p.slice(5));
+                    if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
 
                   return tags.join(", ");
                 }
@@ -304,7 +308,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.head(`/${resource}/:id`, async (ctx, next) => {
+  router.head(`/${resource}/:id`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Count ${resource}`,
@@ -319,18 +323,18 @@ for (const [resource, flags] of Object.entries(resources)) {
     ]);
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void next();
+    if (!res.length) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = "";
   });
 
-  router.get(`/${resource}/:id`, async (ctx, next) => {
+  router.get(`/${resource}/:id`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Query ${resource}`,
@@ -345,19 +349,19 @@ for (const [resource, flags] of Object.entries(resources)) {
     ]);
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void next();
+    if (!res.length) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = res[0];
   });
 
   if (flags & RESOURCE_DELETE) {
-    router.delete(`/${resource}/:id`, async (ctx, next) => {
+    router.delete(`/${resource}/:id`, async (ctx) => {
       const authorizer: Authorizer = ctx.state.authorizer;
       const log = {
         message: `Delete ${resource}`,
@@ -372,10 +376,10 @@ for (const [resource, flags] of Object.entries(resources)) {
       ]);
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
       const res = await db.query(resource, filter);
-      if (!res.length) return void next();
+      if (!res.length) return void (ctx.status = 404);
 
       const validate = authorizer.getValidator(resource, res[0]);
       if (!validate("delete")) {
@@ -392,7 +396,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 
   if (flags & RESOURCE_PUT) {
-    router.put(`/${resource}/:id`, async (ctx, next) => {
+    router.put(`/${resource}/:id`, async (ctx) => {
       const authorizer: Authorizer = ctx.state.authorizer;
       const id = ctx.params.id;
 
@@ -404,7 +408,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
 
       const obj = ctx.request.body;
@@ -431,7 +435,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 }
 
-router.put("/files/:id", async (ctx, next) => {
+router.put("/files/:id", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const resource = "files";
   const id = ctx.params.id;
@@ -445,7 +449,7 @@ router.put("/files/:id", async (ctx, next) => {
 
   if (!authorizer.hasAccess(resource, 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
   const metadata = {
@@ -474,7 +478,7 @@ router.put("/files/:id", async (ctx, next) => {
   ctx.body = "";
 });
 
-router.post("/devices/:id/tasks", async (ctx, next) => {
+router.post("/devices/:id/tasks", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Commit tasks",
@@ -490,10 +494,10 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
   ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
   const devices = await db.query("devices", filter);
-  if (!devices.length) return void next();
+  if (!devices.length) return void (ctx.status = 404);
   const device = devices[0];
 
   const validate = authorizer.getValidator("devices", device);
@@ -542,7 +546,7 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
   ctx.body = res.tasks;
 });
 
-router.post("/devices/:id/tags", async (ctx, next) => {
+router.post("/devices/:id/tags", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Update tags",
@@ -558,10 +562,10 @@ router.post("/devices/:id/tags", async (ctx, next) => {
   ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
   const res = await db.query("devices", filter);
-  if (!res.length) return void next();
+  if (!res.length) return void (ctx.status = 404);
 
   const validate = authorizer.getValidator("devices", res[0]);
   if (!validate("tags", ctx.request.body)) {
@@ -597,7 +601,7 @@ router.get("/ping/:host", async (ctx) => {
   });
 });
 
-router.put("/users/:id/password", async (ctx, next) => {
+router.put("/users/:id/password", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const username = ctx.params.id;
   const log = {
@@ -622,7 +626,7 @@ router.put("/users/:id/password", async (ctx, next) => {
     }
   } else if (!authorizer.hasAccess("users", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
   const filter = and(authorizer.getFilter("users", 3), [
@@ -631,7 +635,7 @@ router.put("/users/:id/password", async (ctx, next) => {
     username,
   ]);
   const res = await db.query("users", filter);
-  if (!res.length) return void next();
+  if (!res.length) return void (ctx.status = 404);
 
   const newPassword = ctx.request.body.newPassword;
   if (ctx.state.user) {

lib/ui/db.ts

@@ -128,7 +128,8 @@ export function query(
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then((client) => {
+    getClient()
+      .then((client) => {
         const collection = client
           .db()
           .collection(RESOURCE_COLLECTION[resource] || resource);
@@ -199,7 +200,8 @@ export function query(
             }
           );
         }
-    });
+      })
+      .catch(reject);
   });
 }
 
@@ -221,7 +223,8 @@ export function count(resource: string, filter: Expression): Promise<number> {
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then((client) => {
+    getClient()
+      .then((client) => {
         const collection = client
           .db()
           .collection(RESOURCE_COLLECTION[resource] || resource);
@@ -229,45 +232,34 @@ export function count(resource: string, filter: Expression): Promise<number> {
           if (err) reject(err);
           else resolve(c);
         });
-    });
-  });
-}
-
-export function updateDeviceTags(
-  deviceId: string,
-  tags: Record<string, boolean>
-): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const add = [];
-    const pull = [];
-
-    const regex = /^[0-9a-zA-Z_]+$/;
-    for (let [tag, onOff] of Object.entries(tags)) {
-      tag = tag.trim();
-      if (onOff) {
-        if (!tag.match(regex))
-          return void reject(new Error(`Invalid tag '${tag}'`));
-        add.push(tag);
-      } else {
-        pull.push(tag);
-      }
-    }
-    getClient()
-      .then((client) => {
-        const collection = client.db().collection("devices");
-
-        const object = {};
-        if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
-        if (pull && pull.length) object["$pullAll"] = { _tags: pull };
-        collection.updateOne({ _id: deviceId }, object, (err) => {
-          if (err) return void reject(err);
-          resolve();
-        });
       })
       .catch(reject);
   });
 }
 
+export async function updateDeviceTags(
+  deviceId: string,
+  tags: Record<string, boolean>
+): Promise<void> {
+  const add = [];
+  const pull = [];
+
+  for (let [tag, onOff] of Object.entries(tags)) {
+    tag = tag.trim();
+    if (onOff) add.push(tag);
+    else pull.push(tag);
+  }
+
+  const client = await getClient();
+  const collection = client.db().collection("devices");
+  const object = {};
+
+  if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
+  if (pull && pull.length) object["$pullAll"] = { _tags: pull };
+
+  await collection.updateOne({ _id: deviceId }, object);
+}
+
 function putResource(resource, id, object): Promise<void> {
   return new Promise((resolve, reject) => {
     getClient()

lib/xml-parser.ts

@@ -53,7 +53,8 @@ export interface Element {
 }
 
 export function parseXmlDeclaration(buffer: Buffer): Attribute[] {
-  for (const enc of ["utf16le", "utf8", "latin1", "ascii"]) {
+  const encodings: BufferEncoding[] = ["utf16le", "utf8", "latin1", "ascii"];
+  for (const enc of encodings) {
     let str = buffer.toString(enc, 0, 150);
     if (str.startsWith("<?xml")) {
       str = str.split("\n")[0].trim();

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.0",
+  "version": "1.2.3",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -20,6 +20,7 @@
     "tools": "tools"
   },
   "dependencies": {
+    "espresso-iisojs": "^1.0.3",
     "iconv-lite": "^0.6.2",
     "ipaddr.js": "^2.0.0",
     "jsonwebtoken": "^8.5.1",
@@ -30,48 +31,48 @@
     "koa-router": "^9.4.0",
     "koa-static": "^5.0.0",
     "later": "^1.2.0",
-    "mongodb": "^3.6.0",
+    "mongodb": "^3.6.2",
     "parsimmon": "^1.16.0",
     "seedrandom": "^3.0.5",
-    "yaml": "^1.10.0",
-    "yebool": "^1.0.2"
+    "yaml": "^1.10.0"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^15.0.0",
+    "@rollup/plugin-commonjs": "^16.0.0",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^9.0.0",
-    "@rollup/plugin-typescript": "^5.0.2",
+    "@rollup/plugin-node-resolve": "^10.0.0",
+    "@rollup/plugin-typescript": "^6.0.0",
     "@types/jsonwebtoken": "^8.5.0",
-    "@types/koa": "^2.11.4",
-    "@types/koa-compress": "^4.0.0",
+    "@types/koa": "^2.11.6",
+    "@types/koa-compress": "^4.0.1",
     "@types/koa-router": "^7.4.1",
     "@types/koa-static": "^4.0.1",
-    "@types/mithril": "^2.0.3",
-    "@types/mongodb": "^3.5.26",
-    "@types/parsimmon": "^1.10.2",
+    "@types/mithril": "^2.0.5",
+    "@types/mongodb": "^3.5.31",
+    "@types/node": "^14.14.5",
+    "@types/parsimmon": "^1.10.3",
     "@types/seedrandom": "^2.4.28",
     "@types/svgo": "^1.3.3",
-    "@typescript-eslint/eslint-plugin": "^3.10.1",
-    "@typescript-eslint/parser": "^3.10.1",
-    "ava": "^3.12.1",
-    "codemirror": "^5.57.0",
+    "@typescript-eslint/eslint-plugin": "^4.6.0",
+    "@typescript-eslint/parser": "^4.6.0",
+    "ava": "^3.13.0",
+    "codemirror": "^5.58.2",
     "cssnano": "^4.1.10",
-    "eslint": "^7.8.0",
-    "eslint-config-prettier": "^6.11.0",
+    "eslint": "^7.12.1",
+    "eslint-config-prettier": "^6.14.0",
     "esm": "^3.2.25",
-    "jsbi": "^3.1.3",
+    "jsbi": "^3.1.4",
     "mithril": "^2.0.4",
-    "postcss": "^7.0.32",
-    "postcss-import": "^12.0.1",
+    "postcss": "^8.1.4",
+    "postcss-import": "^13.0.0",
     "postcss-preset-env": "^6.7.0",
-    "prettier": "^2.1.1",
-    "rollup": "^2.26.9",
-    "rollup-plugin-terser": "^7.0.1",
-    "source-map-loader": "^1.1.0",
+    "prettier": "^2.1.2",
+    "rollup": "^2.32.1",
+    "rollup-plugin-terser": "^7.0.2",
+    "source-map-loader": "^1.1.1",
     "svgo": "^1.3.2",
     "ts-node": "^9.0.0",
-    "typescript": "^4.0.2",
-    "webpack": "^4.44.1"
+    "typescript": "^4.0.5",
+    "webpack": "^5.2.0"
   },
   "engines": {
     "node": ">=12.13.0"

test/mongodb-functions.ts

@@ -1,13 +1,13 @@
 import ava from "ava";
 import * as mongodbFunctions from "../lib/mongodb-functions";
-import { stringify } from "../lib/common/expression-parser";
+import { stringify, parse } from "../lib/common/expression-parser";
 
-ava("mongoQueryToFilter", (t) => {
+ava("convertOldPrecondition", (t) => {
   const tests = [
     [{}, "true"],
     [{ test: "test" }, 'test = "test"'],
     [{ test: { $eq: "test" } }, 'test = "test"'],
-    [{ test: { $ne: "test" } }, 'test <> "test"'],
+    [{ test: { $ne: "test" } }, 'test <> "test" OR test IS NULL'],
     [{ test: { $gte: "test" } }, 'test >= "test"'],
     [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
     [{ "Tags.test": false }, "Tags.test IS NULL"],
@@ -22,20 +22,20 @@ ava("mongoQueryToFilter", (t) => {
     [{ _tags: { $eq: "test" } }, "Tags.test IS NOT NULL"],
     [
       { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" AND test <> "test"',
+      'test = "test" AND (test <> "test" OR test IS NULL)',
     ],
     [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
     [
       { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" OR test <> "test"',
+      'test = "test" OR (test <> "test" OR test IS NULL)',
     ],
     [
       { test: { $gte: "test1", $ne: "test2" } },
-      'test >= "test1" AND test <> "test2"',
+      'test >= "test1" AND (test <> "test2" OR test IS NULL)',
     ],
     [
       { test: "test", test2: { $ne: "test2" } },
-      'test = "test" AND test2 <> "test2"',
+      'test = "test" AND (test2 <> "test2" OR test2 IS NULL)',
     ],
   ];
 
@@ -50,13 +50,83 @@ ava("mongoQueryToFilter", (t) => {
 
   t.plan(tests.length + 2 * shouldFailTests.length);
   for (const test of tests)
-    t.is(stringify(mongodbFunctions.mongoQueryToFilter(test[0])), test[1]);
+    t.is(stringify(mongodbFunctions.convertOldPrecondition(test[0])), test[1]);
 
   for (const test of shouldFailTests) {
     const func = (): void => {
-      mongodbFunctions.mongoQueryToFilter(test[0] as Record<string, unknown>);
+      mongodbFunctions.convertOldPrecondition(
+        test[0] as Record<string, unknown>
+      );
     };
     const error = t.throws(func, { instanceOf: Error });
     t.is(error.message, test[1]);
   }
 });
+
+ava("filterToMongoQuery", async (t) => {
+  const queries: [string, Record<string, unknown>][] = [
+    ["true", {}],
+    ["Tags.tag1 = true", { _tags: "tag1" }],
+    ["Tags.tag1 <> false", { _tags: "tag1" }],
+    ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }],
+    ["Tags.tag1 = 123", { "Tags.tag1": 123 }],
+    ["Param1 = 'value1'", { "Param1._value": "value1" }],
+    ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }],
+    ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
+    ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
+    ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }],
+    ["LOWER(Param1) LIKE 'value'", { "Param1._value": /^value$/i }],
+    [
+      "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          {
+            $and: [
+              { "Param2._value": { $nin: ["value1", null] } },
+              { "Param1._value": { $gte: "value2" } },
+            ],
+          },
+        ],
+      },
+    ],
+    [
+      "Param1 <> 'value2' OR Param1 IS NULL",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          { "Param1._value": null },
+        ],
+      },
+    ],
+  ];
+
+  for (const [expStr, expect] of queries) {
+    const exp = parse(expStr);
+    const query = mongodbFunctions.filterToMongoQuery(
+      mongodbFunctions.processDeviceFilter(exp)
+    );
+    t.deepEqual(query, expect);
+  }
+
+  const failQueries: [any, string][] = [
+    ["Param1 = Param2", "Invalid RHS operand of = clause"],
+    ["Param1 LIKE Param2", "Invalid RHS operand of LIKE clause"],
+    ["1 LIKE '1'", "Invalid LHS operand of LIKE clause"],
+    ["1 = 2", "Invalid LHS operand of = clause"],
+    ["1 IS NULL", "Invalid LHS operand of IS NULL clause"],
+    ["false", "Primitives are not valid queries"],
+    ["UPPER(Param1) LIKE 'value'", "Invalid RHS operand of LIKE clause"],
+  ];
+
+  for (const [expStr, err] of failQueries) {
+    const exp = parse(expStr);
+    t.throws(
+      () =>
+        mongodbFunctions.filterToMongoQuery(
+          mongodbFunctions.processDeviceFilter(exp)
+        ),
+      { message: err }
+    );
+  }
+});

ui/app.ts

@@ -71,7 +71,7 @@ let fulfillTimeout = null;
 function fulfill(): void {
   clearTimeout(fulfillTimeout);
   fulfillTimeout = setTimeout(() => {
-    store.fulfill(lastRenderTimestamp, pageVisitTimestamp).then((updated) => {
+    store.fulfill(lastRenderTimestamp, pageVisitTimestamp, (updated) => {
       if (updated) m.redraw();
     });
   }, 100);

ui/components/device-actions.ts

@@ -70,7 +70,7 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageDownload({
                 name: "download",
-                device: device["DeviceID.ID"].value[0],
+                devices: [device["DeviceID.ID"].value[0]],
               });
             },
           },

ui/components/parameter.ts

@@ -90,7 +90,7 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageSpv({
                 name: "setParameterValues",
-                device: device["DeviceID.ID"].value[0],
+                devices: [device["DeviceID.ID"].value[0]],
                 parameterValues: [
                   [
                     parameter,

ui/components/ping.ts

@@ -27,23 +27,28 @@ const component: ClosureComponent = (vn): Component => {
   let interval: ReturnType<typeof setInterval>;
   let host: string;
 
-  const refresh = async (): Promise<void> => {
+  const refresh = (): void => {
+    if (!host) {
+      const dom = (vn as VnodeDOM).dom;
+      if (dom) dom.innerHTML = "";
+      return;
+    }
+
     let status = "";
-    if (host) {
-      try {
-        const res = await store.ping(host);
+    store
+      .ping(host)
+      .then((res) => {
         if (res["avg"] != null) status = `${Math.trunc(res["avg"])} ms`;
         else status = "Unreachable";
-      } catch (err) {
-        setTimeout(() => {
-          throw err;
-        }, 0);
-      }
-    }
-
+      })
+      .catch(() => {
+        status = "Error!";
+        clearInterval(interval);
+      })
+      .finally(() => {
         const dom = (vn as VnodeDOM).dom;
-
         if (dom) dom.innerHTML = `Pinging ${host}: ${status}`;
+      });
   };
 
   return {

ui/components/summon-button.ts

@@ -79,6 +79,10 @@ const component: ClosureComponent = (): Component => {
               .then(() => {
                 e.target.disabled = false;
                 store.fulfill(0, Date.now());
+              })
+              .catch((err) => {
+                e.target.disabled = false;
+                notifications.push("error", err.message);
               });
           },
         },

ui/components/tags.ts

@@ -22,6 +22,7 @@ import { m } from "../components";
 import * as notifications from "../notifications";
 import * as store from "../store";
 import { getIcon } from "../icons";
+import { decodeTag } from "../../lib/common";
 
 const component: ClosureComponent = (): Component => {
   return {
@@ -32,7 +33,7 @@ const component: ClosureComponent = (): Component => {
 
       const tags = [];
       for (const p of Object.keys(device))
-        if (p.startsWith("Tags.")) tags.push(p.slice(5));
+        if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
 
       tags.sort();
 

ui/config-functions.ts

@@ -42,6 +42,35 @@ export function flattenConfig(config: Record<string, unknown>): any {
   return flatten;
 }
 
+// Order keys such that nested objects come last
+function orderKeys(config: any): number {
+  let res = 1;
+  if (typeof config !== "object") return res;
+  if (Array.isArray(config)) {
+    for (const c of config) res += orderKeys(c);
+    return res;
+  }
+
+  const weights: [string, number][] = Object.entries(config).map(([k, v]) => [
+    k,
+    orderKeys(v),
+  ]);
+
+  weights.sort((a, b) => {
+    if (a[1] !== b[1]) return a[1] - b[1];
+    if (b[0] > a[0]) return -1;
+    else return 1;
+  });
+
+  for (const [k, w] of weights) {
+    res += w;
+    const v = config[k];
+    delete config[k];
+    config[k] = v;
+  }
+  return res;
+}
+
 export function structureConfig(config: Config[]): any {
   config.sort((a, b) => (a._id > b._id ? 1 : a._id < b._id ? -1 : 0));
   const _config = {};
@@ -93,7 +122,9 @@ export function structureConfig(config: Config[]): any {
     return object;
   };
 
-  return toArray(_config);
+  const res = toArray(_config);
+  orderKeys(res);
+  return res;
 }
 
 export function diffConfig(

ui/config-page.ts

@@ -147,8 +147,8 @@ function renderTable(confsResponse, searchString): Children {
       {
         title: "Edit config value",
         onclick: () => {
-          const cb = (): Children => {
-            return m(
+          let cb: () => Children = null;
+          const comp = m(
             putFormComponent,
             Object.assign(
               {
@@ -176,8 +176,13 @@ function renderTable(confsResponse, searchString): Children {
               formData
             )
           );
-          };
-          overlay.open(cb);
+          cb = () => comp;
+          overlay.open(
+            cb,
+            () =>
+              !comp.state["current"]["modified"] ||
+              confirm("You have unsaved changes. Close anyway?")
+          );
         },
       },
       getIcon("edit")
@@ -244,8 +249,8 @@ export const component: ClosureComponent = (): Component => {
           {
             title: "Create new config",
             onclick: () => {
-              const cb = (): Children => {
-                return m(
+              let cb: () => Children = null;
+              const comp = m(
                 putFormComponent,
                 Object.assign(
                   {
@@ -274,8 +279,13 @@ export const component: ClosureComponent = (): Component => {
                   formData
                 )
               );
-              };
-              overlay.open(cb);
+              cb = () => comp;
+              overlay.open(
+                cb,
+                () =>
+                  !comp.state["current"]["modified"] ||
+                  confirm("You have unsaved changes. Close anyway?")
+              );
             },
           },
           "New config"
@@ -307,8 +317,8 @@ export const component: ClosureComponent = (): Component => {
               "button",
               {
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     uiConfigComponent,
                     Object.assign(
                       {
@@ -338,8 +348,13 @@ export const component: ClosureComponent = (): Component => {
                       attrs
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               `Edit ${sub.name}`

ui/css/app.css

@@ -178,6 +178,8 @@ body {
       }
 
       & > .queue {
+        max-height: 400px;
+        overflow-y: auto;
         & > .pending {
           animation: 0.5s linear 0s infinite alternate pending-blink;
           border-radius: 0.4em;

ui/devices-page.ts

@@ -23,7 +23,7 @@ import config from "./config";
 import indexTableComponent from "./index-table-component";
 import filterComponent from "./filter-component";
 import * as store from "./store";
-import * as taskQueue from "./task-queue";
+import { queueTask, stageDownload } from "./task-queue";
 import * as notifications from "./notifications";
 import { parse, stringify } from "../lib/common/expression-parser";
 import { evaluate, extractParams } from "../lib/common/expression";
@@ -91,12 +91,11 @@ function renderActions(selected: Set<string>): Children {
         title: "Reboot selected devices",
         disabled: !selected.size,
         onclick: () => {
-          for (const d of selected) {
-            taskQueue.queueTask({
+          const tasks = [...selected].map((s) => ({
             name: "reboot",
-              device: d,
-            });
-          }
+            device: s,
+          }));
+          queueTask(...tasks);
         },
       },
       "Reboot"
@@ -110,18 +109,34 @@ function renderActions(selected: Set<string>): Children {
         title: "Factory reset selected devices",
         disabled: !selected.size,
         onclick: () => {
-          for (const d of selected) {
-            taskQueue.queueTask({
+          const tasks = [...selected].map((s) => ({
             name: "factoryReset",
-              device: d,
-            });
-          }
+            device: s,
+          }));
+          queueTask(...tasks);
         },
       },
       "Reset"
     )
   );
 
+  buttons.push(
+    m(
+      "button.critical",
+      {
+        title: "Push a firmware or a config file",
+        disabled: !selected.size,
+        onclick: () => {
+          stageDownload({
+            name: "download",
+            devices: [...selected],
+          });
+        },
+      },
+      "Push file"
+    )
+  );
+
   buttons.push(
     m(
       "button.primary",

ui/drawer-component.ts

@@ -24,10 +24,21 @@ import m, {
   Component,
   VnodeDOM,
 } from "mithril";
-import * as taskQueue from "./task-queue";
 import * as store from "./store";
 import * as notifications from "./notifications";
 import { getIcon } from "./icons";
+import {
+  clear,
+  commit,
+  deleteTask,
+  getQueue,
+  getStaging,
+  QueueTask,
+  queueTask,
+  StageTask,
+} from "./task-queue";
+
+const invalid: WeakSet<StageTask> = new WeakSet();
 
 function mparam(param): Children {
   return m("span.parameter", { title: param }, `${param}`);
@@ -37,11 +48,11 @@ function mval(val): Children {
   return m("span.value", { title: val }, `${val}`);
 }
 
-function renderStagingSpv(task, queueFunc, cancelFunc): Children {
-  function keydown(e): void {
-    if (e.keyCode === 13) queueFunc();
-    else if (e.keyCode === 27) cancelFunc();
-    else e.redraw = false;
+function renderStagingSpv(task: StageTask, queueFunc, cancelFunc): Children {
+  function keydown(e: KeyboardEvent): void {
+    if (e.key === "Enter") queueFunc();
+    else if (e.key === "Escape") cancelFunc();
+    else e["redraw"] = false;
   }
 
   let input;
@@ -90,13 +101,24 @@ function renderStagingSpv(task, queueFunc, cancelFunc): Children {
   return [m("span", "Editing ", mparam(task.parameterValues[0][0])), input];
 }
 
-function renderStagingDownload(task): Children {
-  task.invalid = !task.fileName || !task.fileType;
+function renderStagingDownload(task: StageTask): Children {
+  if (!task.fileName || !task.fileType) invalid.add(task);
+  else invalid.delete(task);
   const files = store.fetch("files", true);
-  const idParts = task.device.split("-");
-  const oui = decodeURIComponent(idParts[0]);
-  const productClass =
-    idParts.length === 3 ? decodeURIComponent(idParts[1]) : "";
+  let oui = "";
+  let productClass = "";
+  for (const d of task.devices) {
+    const parts = d.split("-");
+    if (oui === "") oui = parts[0];
+    else if (oui !== parts[0]) oui = null;
+    if (parts.length === 3) {
+      if (productClass === "") productClass = parts[1];
+      else if (productClass !== parts[1]) productClass = null;
+    }
+  }
+
+  if (oui) oui = decodeURIComponent(oui);
+  if (productClass) productClass = decodeURIComponent(productClass);
 
   const typesList = [
     "",
@@ -162,13 +184,17 @@ function renderStagingDownload(task): Children {
   ];
 }
 
-function renderStaging(staging): Child[] {
+function renderStaging(staging: Set<StageTask>): Child[] {
   const elements: Child[] = [];
 
   for (const s of staging) {
     const queueFunc = (): void => {
       staging.delete(s);
-      taskQueue.queueTask(s);
+      for (const d of s.devices) {
+        const t = Object.assign({ device: d }, s);
+        delete t.devices;
+        queueTask(t);
+      }
     };
     const cancelFunc = (): void => {
       staging.delete(s);
@@ -181,7 +207,7 @@ function renderStaging(staging): Child[] {
 
     const queue = m(
       "button.primary",
-      { title: "Queue task", onclick: queueFunc, disabled: s.invalid },
+      { title: "Queue task", onclick: queueFunc, disabled: invalid.has(s) },
       "Queue"
     );
     const cancel = m(
@@ -195,7 +221,7 @@ function renderStaging(staging): Child[] {
   return elements;
 }
 
-function renderQueue(queue): Child[] {
+function renderQueue(queue: Set<QueueTask>): Child[] {
   const details: Child[] = [];
   const devices: { [deviceId: string]: any[] } = {};
   for (const t of queue) {
@@ -215,7 +241,7 @@ function renderQueue(queue): Child[] {
             {
               title: "Retry this task",
               onclick: () => {
-                taskQueue.queueTask(t);
+                queueTask(t);
               },
             },
             getIcon("retry")
@@ -229,7 +255,7 @@ function renderQueue(queue): Child[] {
           {
             title: "Remove this task",
             onclick: () => {
-              taskQueue.deleteTask(t);
+              deleteTask(t);
             },
           },
           getIcon("remove")
@@ -347,8 +373,8 @@ function renderNotifications(notifs): Child[] {
 const component: ClosureComponent = (): Component => {
   return {
     view: (vnode) => {
-      const queue = taskQueue.getQueue();
-      const staging = taskQueue.getStaging();
+      const queue = getQueue();
+      const staging = getStaging();
       const notifs = notifications.getNotifications();
 
       let drawerElement, statusElement;
@@ -394,11 +420,10 @@ const component: ClosureComponent = (): Component => {
               title: "Commit queued tasks",
               disabled: !statusCount.queued,
               onclick: () => {
-                const tasks = Array.from(taskQueue.getQueue()).filter(
+                const tasks = Array.from(getQueue()).filter(
                   (t) => t["status"] === "queued"
                 );
-                taskQueue
-                  .commit(
+                commit(
                   tasks,
                   (deviceId, err, connectionRequestStatus, tasks2) => {
                     if (err) {
@@ -441,6 +466,9 @@ const component: ClosureComponent = (): Component => {
                 )
                   .then(() => {
                     store.fulfill(0, Date.now());
+                  })
+                  .catch((err) => {
+                    notifications.push("error", err.message);
                   });
               },
             },
@@ -450,7 +478,7 @@ const component: ClosureComponent = (): Component => {
             "button",
             {
               title: "Clear tasks",
-              onclick: taskQueue.clear,
+              onclick: clear,
               disabled: !queueElements.length,
             },
             "Clear"

ui/dynamic-loader.ts

@@ -51,7 +51,7 @@ export function loadCodeMirror(): Promise<void> {
     ];
     Promise.all(promises)
       .then((modules) => {
-        codeMirror = modules[0];
+        codeMirror = modules[0].default;
         resolve();
       })
       .catch((err) => {

ui/files-page.ts

@@ -221,8 +221,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new file",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -251,8 +251,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/index-table-component.ts

@@ -27,6 +27,8 @@ interface Attribute {
   type?: string;
 }
 
+const MAX_PAGE_SIZE = 200;
+
 function renderTable(
   attributes: Attribute[],
   data: Record<string, any>[],
@@ -37,7 +39,7 @@ function renderTable(
   onSortChange: (obj: Record<string, number>) => void,
   downloadUrl?: string,
   valueCallback?: (attr: Attribute, record: Record<string, any>) => Children,
-  actionsCallback?: Children | ((selected: Set<string>) => Children),
+  actionsCallback?: Children | ((sel: Set<string>) => Children),
   recordActionsCallback?: Children | ((record: Record<string, any>) => Children)
 ): Children {
   const records = data || [];
@@ -180,7 +182,8 @@ function renderTable(
       {
         title: "Show more records",
         onclick: showMoreCallback,
-        disabled: !data.length || records.length >= total,
+        disabled:
+          !data.length || records.length >= Math.min(MAX_PAGE_SIZE, total),
       },
       "More"
     )

ui/login-page.ts

@@ -81,7 +81,7 @@ export const component: ClosureComponent = (): Component => {
                       location.reload();
                     })
                     .catch((err) => {
-                      notifications.push("error", err.message);
+                      notifications.push("error", err.response || err.message);
                       e.target.disabled = false;
                     });
                   return false;

ui/overlay.ts

@@ -20,16 +20,24 @@
 import m, { Children } from "mithril";
 
 type OverlayCallback = () => Children;
+type CloseCallback = () => boolean;
 
 let overlayCallback: OverlayCallback = null;
+let closeCallback: CloseCallback = null;
 
-export function open(callback: OverlayCallback): void {
+export function open(
+  callback: OverlayCallback,
+  closeCb: CloseCallback = null
+): void {
   overlayCallback = callback;
+  closeCallback = closeCb;
 }
 
-export function close(callback: OverlayCallback): boolean {
+export function close(callback: OverlayCallback, force = true): boolean {
   if (callback === overlayCallback) {
+    if (!force && closeCallback && !closeCallback()) return false;
     overlayCallback = null;
+    closeCallback = null;
     return true;
   }
 
@@ -43,7 +51,7 @@ export function render(): Children {
       {
         tabindex: 0,
         onclick: () => {
-          close(overlayCallback);
+          close(overlayCallback, false);
         },
         style: "opacity: 0",
         oncreate: (vnode) => {
@@ -75,9 +83,10 @@ export function render(): Children {
 }
 
 document.addEventListener("keydown", (e) => {
-  if (overlayCallback && e.keyCode === 27 && close(overlayCallback)) m.redraw();
+  if (overlayCallback && e.key === "Escape" && close(overlayCallback, false))
+    m.redraw();
 });
 
 window.addEventListener("popstate", () => {
-  if (close(overlayCallback)) m.redraw();
+  if (close(overlayCallback, false)) m.redraw();
 });

ui/permissions-page.ts

@@ -285,8 +285,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new permission",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -315,8 +315,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/presets-page.ts

@@ -287,16 +287,8 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
-                  if (!preset.provision) {
-                    return m(
-                      "div",
-                      { style: "margin:20px" },
-                      "This UI only supports presets with a single 'provision' configuraiton. If this preset was originally created from the old UI (genieacs-gui), you must edit it there."
-                    );
-                  }
-
-                  return m(
+                let cb: () => Children = null;
+                const comp = m(
                   putFormComponent,
                   Object.assign(
                     {
@@ -326,8 +318,22 @@ export const component: ClosureComponent = (): Component => {
                     formData
                   )
                 );
+                cb = (): Children => {
+                  if (!preset.provision) {
+                    return m(
+                      "div",
+                      { style: "margin:20px" },
+                      "This UI only supports presets with a single 'provision' configuraiton. If this preset was originally created from the old UI (genieacs-gui), you must edit it there."
+                    );
+                  }
+                  return comp;
                 };
-                overlay.open(cb);
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
               },
             },
             "Show"
@@ -343,8 +349,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new preset",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -373,8 +379,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/provisions-page.ts

@@ -207,8 +207,8 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
-                  return m(
+                let cb: () => Children = null;
+                const comp = m(
                   putFormComponent,
                   Object.assign(
                     {
@@ -238,8 +238,13 @@ export const component: ClosureComponent = (): Component => {
                     formData
                   )
                 );
-                };
-                overlay.open(cb);
+                cb = () => comp;
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
               },
             },
             "Show"
@@ -255,8 +260,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new provision",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -285,8 +290,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/put-form-component.ts

@@ -56,6 +56,7 @@ function createField(current, attr, focus): Children {
           : null,
         onchange: (e) => {
           current.object[attr.id] = e.target.value;
+          current.modified = true;
           e.redraw = false;
         },
       },
@@ -80,6 +81,7 @@ function createField(current, attr, focus): Children {
           if (e.target.checked) currentSelected.add(op);
           else currentSelected.delete(op);
           current.object[attr.id] = Array.from(currentSelected);
+          current.modified = true;
           e.redraw = false;
         },
       };
@@ -98,6 +100,7 @@ function createField(current, attr, focus): Children {
       },
       onChange: (value) => {
         current.object[attr.id] = value;
+        current.modified = true;
       },
     };
     return m(codeEditorComponent, attrs);
@@ -112,6 +115,7 @@ function createField(current, attr, focus): Children {
         : null,
       onchange: (e) => {
         current.object[attr.id] = e.target.files;
+        current.modified = true;
         e.redraw = false;
       },
     });
@@ -132,6 +136,7 @@ function createField(current, attr, focus): Children {
         : null,
       oninput: (e) => {
         current.object[attr.id] = e.target.value;
+        current.modified = true;
         e.redraw = false;
       },
       onkeypress: (e) => {
@@ -158,6 +163,7 @@ function createField(current, attr, focus): Children {
       : null,
     oninput: (e) => {
       current.object[attr.id] = e.target.value;
+      current.modified = true;
       e.redraw = false;
     },
   });
@@ -174,6 +180,7 @@ const component: ClosureComponent = (): Component => {
         vnode.state["current"] = {
           isNew: !base["_id"],
           object: Object.assign({}, base),
+          modified: false,
         };
       }
 

ui/smart-query.ts

@@ -19,6 +19,7 @@
 
 import config from "./config";
 import { Expression } from "../lib/types";
+import { encodeTag } from "../lib/common";
 
 const resources = {
   devices: {},
@@ -138,7 +139,7 @@ function queryMac(param, value): Expression {
 }
 
 function queryTag(tag: string): Expression {
-  const t = tag.replace(/[^a-zA-Z0-9-]+/g, "_");
+  const t = encodeTag(tag);
   return ["IS NOT NULL", ["PARAM", `Tags.${t}`]];
 }
 

ui/store.ts

@@ -152,6 +152,8 @@ function checkConnection(): void {
         }
       }
     },
+  }).catch((err) => {
+    notifications.push("error", err.message);
   });
 }
 
@@ -169,11 +171,26 @@ export async function xhrRequest(
   ): any => {
     if (typeof extract === "function") return extract(xhr, _options);
 
+    // https://mithril.js.org/request.html#error-handling
+    if (xhr.status !== 304 && Math.floor(xhr.status / 100) !== 2) {
+      if (xhr.status === 403) throw new Error("Not authorized");
+      const err = new Error();
+      err["message"] =
+        xhr.status === 0
+          ? "Server is unreachable"
+          : `Unexpected response status code ${xhr.status}`;
+      err["code"] = xhr.status;
+      err["response"] = xhr.responseText;
+      throw err;
+    }
+
     let response: any;
     if (typeof deserialize === "function") {
       response = deserialize(xhr.responseText);
     } else if (
-      xhr.getResponseHeader("content-type").startsWith("application/json")
+      (xhr.getResponseHeader("content-type") || "").startsWith(
+        "application/json"
+      )
     ) {
       try {
         response = xhr.responseText ? JSON.parse(xhr.responseText) : null;
@@ -184,15 +201,6 @@ export async function xhrRequest(
       response = xhr.responseText;
     }
 
-    // https://mithril.js.org/request.html#error-handling
-    if (xhr.status !== 304 && Math.floor(xhr.status / 100) !== 2) {
-      const err = new Error();
-      err["message"] = xhr.responseText;
-      err["code"] = xhr.status;
-      err["response"] = response;
-      throw err;
-    }
-
     return response;
   };
 
@@ -340,7 +348,7 @@ export function fetch(
   return queryResponse;
 }
 
-export function fulfill(
+function _fulfill(
   accessTimestamp: number,
   _fulfillTimestamp: number
 ): Promise<boolean> {
@@ -377,7 +385,17 @@ export function fulfill(
                 m.buildQueryString({
                   filter: memoizedStringify(filter),
                 }),
-              extract: (xhr) => +xhr.getResponseHeader("x-total-count"),
+              extract: (xhr) => {
+                if (xhr.status === 403) throw new Error("Not authorized");
+                if (!xhr.status) {
+                  throw new Error("Server is unreachable");
+                } else if (xhr.status !== 200) {
+                  throw new Error(
+                    `Unexpected response status code ${xhr.status}`
+                  );
+                }
+                return +xhr.getResponseHeader("x-total-count");
+              },
               background: true,
             })
               .then((c) => {
@@ -497,14 +515,14 @@ export function fulfill(
             continue;
           }
 
-          const combinedFilterDiff = diff;
-          resources[resourceType].combinedFilter = union;
-
           updated = true;
           let deleted = new Set<string>();
           if (!resources[resourceType].combinedFilter)
             deleted = new Set(resources[resourceType].objects.keys());
 
+          const combinedFilterDiff = diff;
+          resources[resourceType].combinedFilter = union;
+
           allPromises2.push(
             new Promise((resolve2, reject2) => {
               xhrRequest({
@@ -560,6 +578,17 @@ export function fulfill(
   });
 }
 
+export function fulfill(
+  accessTimestamp: number,
+  _fulfillTimestamp: number,
+  callback?: (updated: boolean) => void
+): void {
+  _fulfill(accessTimestamp, _fulfillTimestamp).then(callback, (err) => {
+    notifications.push("error", err.message);
+    if (callback) callback(false);
+  });
+}
+
 export function getTimestamp(): number {
   return fulfillTimestamp;
 }
@@ -578,6 +607,8 @@ export function postTasks(
     url: `api/devices/${encodeURIComponent(deviceId)}/tasks`,
     body: tasks,
     extract: (xhr) => {
+      if (xhr.status === 403) throw new Error("Not authorized");
+      if (!xhr.status) throw new Error("Server is unreachable");
       if (xhr.status !== 200) throw new Error(xhr.response);
       const connectionRequestStatus = xhr.getResponseHeader(
         "Connection-Request"
@@ -647,7 +678,13 @@ export function resourceExists(resource: string, id: string): Promise<number> {
       m.buildQueryString({
         filter: memoizedStringify(filter),
       }),
-    extract: (xhr) => +xhr.getResponseHeader("x-total-count"),
+    extract: (xhr) => {
+      if (xhr.status === 403) throw new Error("Not authorized");
+      if (!xhr.status) throw new Error("Server is unreachable");
+      else if (xhr.status !== 200)
+        throw new Error(`Unexpected response status code ${xhr.status}`);
+      return +xhr.getResponseHeader("x-total-count");
+    },
     background: true,
   });
 }

ui/task-queue.ts

@@ -19,23 +19,41 @@
 
 import * as store from "./store";
 import { Task } from "../lib/types";
+import * as notifications from "./notifications";
 
 export interface QueueTask extends Task {
   status?: string;
-  device?: string;
+  device: string;
 }
 
-const queue: Set<QueueTask> = new Set();
-const staging: Set<QueueTask> = new Set();
+export interface StageTask extends Task {
+  devices: string[];
+}
 
-export function queueTask(task: QueueTask): void {
-  staging.delete(task);
+const MAX_QUEUE = 100;
+
+const queue: Set<QueueTask> = new Set();
+const staging: Set<StageTask> = new Set();
+
+function canQueue(tasks: QueueTask[]): boolean {
+  let count = queue.size;
+  for (const task of tasks) if (!queue.has(task)) ++count;
+  return count <= MAX_QUEUE;
+}
+
+export function queueTask(...tasks: QueueTask[]): void {
+  if (!canQueue(tasks)) {
+    notifications.push("error", "Too many tasks in queue");
+    return;
+  }
+
+  for (const task of tasks) {
     task.status = "queued";
     queue.add(task);
+  }
 }
 
 export function deleteTask(task: QueueTask): void {
-  staging.delete(task);
   queue.delete(task);
 }
 
@@ -47,7 +65,7 @@ export function clear(): void {
   queue.clear();
 }
 
-export function getStaging(): Set<QueueTask> {
+export function getStaging(): Set<StageTask> {
   return staging;
 }
 
@@ -55,11 +73,19 @@ export function clearStaging(): void {
   staging.clear();
 }
 
-export function stageSpv(task: QueueTask): void {
+export function stageSpv(task: StageTask): void {
+  if (queue.size + task.devices.length > MAX_QUEUE) {
+    notifications.push("error", "Too many tasks in queue");
+    return;
+  }
   staging.add(task);
 }
 
-export function stageDownload(task: QueueTask): void {
+export function stageDownload(task: StageTask): void {
+  if (queue.size + task.devices.length > MAX_QUEUE) {
+    notifications.push("error", "Too many tasks in queue");
+    return;
+  }
   staging.add(task);
 }
 
@@ -69,14 +95,19 @@ export function commit(
     deviceId: string,
     err: Error,
     conReqStatus: string,
-    tasks: QueueTask[]
+    _tasks: QueueTask[]
   ) => void
 ): Promise<void> {
   const devices: { [deviceId: string]: any[] } = {};
+
+  if (!canQueue(tasks))
+    return Promise.reject(new Error("Too many tasks in queue"));
+
   for (const t of tasks) {
     devices[t.device] = devices[t.device] || [];
     devices[t.device].push(t);
-    queueTask(t);
+    t.status = "queued";
+    queue.add(t);
   }
 
   return new Promise((resolve) => {

ui/ui-config-component.ts

@@ -102,12 +102,7 @@ const component: ClosureComponent = (): Component => {
 
       const yamlString =
         config && Object.values(config).length
-          ? yaml.stringify(config, {
-              schema: "failsafe",
-              tags: (args) => {
-                return args;
-              },
-            })
+          ? yaml.stringify(config, { schema: "failsafe" })
           : "";
 
       const attrs = {
@@ -120,6 +115,7 @@ const component: ClosureComponent = (): Component => {
         },
         onChange: (value) => {
           vnode.state["updatedYaml"] = value;
+          vnode.state["modified"] = true;
         },
       };
 

ui/users-page.ts

@@ -237,9 +237,8 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
-                  const children = [
-                    m(
+                let cb: () => Children = null;
+                const comp = m(
                   putFormComponent,
                   Object.assign(
                     {
@@ -274,9 +273,10 @@ export const component: ClosureComponent = (): Component => {
                       attributes: attributes,
                     }
                   )
-                    ),
-                  ];
+                );
 
+                cb = () => {
+                  const children: Children = [comp];
                   if (canWrite) {
                     children.push(m("hr"));
                     const _attrs = {
@@ -292,7 +292,13 @@ export const component: ClosureComponent = (): Component => {
 
                   return children;
                 };
-                overlay.open(cb);
+
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
               },
             },
             "Show"
@@ -317,8 +323,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new user",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -347,8 +353,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/virtual-parameters-page.ts

@@ -207,8 +207,8 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
-                  return m(
+                let cb: () => Children = null;
+                const comp = m(
                   putFormComponent,
                   Object.assign(
                     {
@@ -238,8 +238,13 @@ export const component: ClosureComponent = (): Component => {
                     formData
                   )
                 );
-                };
-                overlay.open(cb);
+                cb = () => comp;
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
               },
             },
             "Show"
@@ -255,8 +260,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new virtual parameter",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -285,8 +290,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/wizard-page.ts

@@ -87,15 +87,23 @@ export const component: ClosureComponent = (vnode): Component => {
               })
                 .then(() => {
                   setTimeout(() => {
-                    m.request({ url: "init" }).then((o) => {
+                    m.request({ url: "init" })
+                      .then((o) => {
                         e.target.disabled = false;
                         options = o;
-                      notifications.push("success", "Initialization complete", {
+                        notifications.push(
+                          "success",
+                          "Initialization complete",
+                          {
                             "Open Sesame!": () => {
                               m.route.set("/login");
                               window.location.reload();
                             },
-                      });
+                          }
+                        );
+                      })
+                      .catch((err) => {
+                        notifications.push("error", err.message);
                       });
                   }, 3000);
                   if (opts["users"]) {