Fix XMPP stream not getting closed properly

The XMPP server may not flag the bind feature as required but we're
still required to bind resource to the stream in order to be able to
send IQ stanzas to other clients.

.eslintrc.json

@@ -21,6 +21,7 @@
     "@typescript-eslint/no-explicit-any": 0,
     "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
     "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
+    "@typescript-eslint/no-floating-promises": "error",
     "require-atomic-updates": 0,
     "no-prototype-builtins": 0
   }

CHANGELOG.md

@@ -1,5 +1,39 @@
 # Change Log
 
+## 1.2.3 (2020-10-26)
+
+- New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
+report writable parameters as non-writable. When set to true, the scripts will
+no longer respect the 'writable' attribute of the CPE parameters and will send
+a SetParamteerValues, AddObject, or DeleteObject request anyway.
+- Tags no longer restrict what characters are allowed. Any character other than
+alphanumeric characters, hyphen, or underscore is now encoded in the data
+model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
+- Ask for a confirmation before closing a pop-up dialog with unsaved changes.
+- Better XML validation to avoid crashes caused by invalid CPE requests.
+- Fix confusing 404 error message when the user attempts to modify a resource
+when they don't have the necessary permissions.
+- Fix a rare issue where genieacs-cwmp stops accepting new connections after
+running for a few weeks.
+- Fix exception when IS NULL operator is used in certain situations.
+
+## 1.2.2 (2020-10-03)
+
+- Added button to push files to selected devices from device listing page.
+- A few minor UI improvements.
+- Fix exception that can happen and persist after a Download request.
+- Fix validation bug preventing running refreshObject task on data model root.
+- Fix invalid arguments fault in refresh preset configuration when upgrading
+from v1.1.
+
+## 1.2.1 (2020-09-08)
+
+- Fix bug causing faults to not be displayed in the UI.
+- Fix bug where deleting objects does not get reflected immediately in the UI.
+- Improve conversion between filters written in the expression format and
+MongoDB queries. There should now be fewer edge cases where the two are not
+equisatisfiable.
+
 ## 1.2.0 (2020-09-01)
 
 - Support GetParameterAttributes and SetParameterAttributes TR-069 methods.

bin/genieacs-cwmp.ts

@@ -24,7 +24,6 @@ import * as server from "../lib/server";
 import * as cwmp from "../lib/cwmp";
 import * as db from "../lib/db";
 import * as extensions from "../lib/extensions";
-import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("cwmp", VERSION);
@@ -36,14 +35,13 @@ function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
-    cache.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -77,9 +75,16 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("CWMP_SSL_KEY") as string,
-    cert: config.get("CWMP_SSL_CERT") as string,
+  const key = config.get("CWMP_SSL_KEY") as string;
+  const cert = config.get("CWMP_SSL_CERT") as string;
+
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    onConnection: cwmp.onConnection,
+    timeout: 30000,
+    keepAliveTimeout: 0,
   };
 
   process.on("uncaughtException", (err) => {
@@ -92,16 +97,10 @@ if (!cluster.worker) {
     server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
-  const initPromise = Promise.all([db.connect(), cache.connect()])
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(
-        SERVICE_PORT,
-        SERVICE_ADDRESS,
-        ssl,
-        cwmp.listener,
-        cwmp.onConnection,
-        0
-      );
+      server.start(options, cwmp.listener);
     })
     .catch((err) => {
       setTimeout(() => {

bin/genieacs-fs.ts

@@ -23,7 +23,6 @@ import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import { listener } from "../lib/fs";
 import * as db from "../lib/db";
-import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("fs", VERSION);
@@ -33,11 +32,9 @@ const SERVICE_PORT = config.get("FS_PORT") as number;
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
-  Promise.all([
-    db.disconnect(),
-    cache.disconnect(),
-    cluster.worker.disconnect(),
-  ]).catch(exitWorkerUngracefully);
+  Promise.all([db.disconnect(), cluster.worker.disconnect()]).catch(
+    exitWorkerUngracefully
+  );
 }
 
 function exitWorkerUngracefully(): void {
@@ -73,9 +70,13 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("FS_SSL_KEY") as string,
-    cert: config.get("FS_SSL_CERT") as string,
+  const key = config.get("FS_SSL_KEY") as string;
+  const cert = config.get("FS_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
   let stopping = false;
@@ -96,9 +97,10 @@ if (!cluster.worker) {
     listener(req, res);
   };
 
-  const initPromise = Promise.all([db.connect(), cache.connect()])
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+      server.start(options, _listener);
     })
     .catch((err) => {
       setTimeout(() => {

bin/genieacs-nbi.ts

@@ -24,7 +24,6 @@ import * as server from "../lib/server";
 import { listener } from "../lib/nbi";
 import * as db from "../lib/db";
 import * as extensions from "../lib/extensions";
-import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("nbi", VERSION);
@@ -36,14 +35,13 @@ function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
-    cache.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -77,9 +75,13 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("NBI_SSL_KEY") as string,
-    cert: config.get("NBI_SSL_CERT") as string,
+  const key = config.get("NBI_SSL_KEY") as string;
+  const cert = config.get("NBI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
   let stopping = false;
@@ -100,9 +102,10 @@ if (!cluster.worker) {
     listener(req, res);
   };
 
-  const initPromise = Promise.all([db.connect(), cache.connect()])
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+      server.start(options, _listener);
     })
     .catch((err) => {
       setTimeout(() => {

bin/genieacs-ui.ts

@@ -23,9 +23,7 @@ import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import { listener } from "../lib/ui";
 import * as extensions from "../lib/extensions";
-import * as db from "../lib/ui/db";
-import * as db2 from "../lib/db";
-import * as cache from "../lib/cache";
+import * as db from "../lib/db";
 import { version as VERSION } from "../package.json";
 
 logger.init("ui", VERSION);
@@ -37,15 +35,13 @@ function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
-    db2.disconnect(),
-    cache.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -79,9 +75,13 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("UI_SSL_KEY") as string,
-    cert: config.get("UI_SSL_CERT") as string,
+  const key = config.get("UI_SSL_KEY") as string;
+  const cert = config.get("UI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
   let stopping = false;
@@ -102,9 +102,10 @@ if (!cluster.worker) {
     listener(req, res);
   };
 
-  const initPromise = Promise.all([db2.connect(), cache.connect()])
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+      server.start(options, _listener);
     })
     .catch((err) => {
       setTimeout(() => {

build/build.ts

@@ -19,14 +19,13 @@
 
 import * as path from "path";
 import * as fs from "fs";
-import { promisify } from "util";
 import { rollup, WarningHandler } from "rollup";
 import rollupJson from "@rollup/plugin-json";
 import typescript from "@rollup/plugin-typescript";
 import { terser } from "rollup-plugin-terser";
 import nodeResolve from "@rollup/plugin-node-resolve";
 import commonjs from "@rollup/plugin-commonjs";
-import webpack from "webpack";
+import webpack, { Configuration } from "webpack";
 import postcss from "postcss";
 import postcssImport from "postcss-import";
 import postcssPresetEnv from "postcss-preset-env";
@@ -52,6 +51,8 @@ const externals = [
   "tls",
   "http",
   "https",
+  "net",
+  "events",
   "zlib",
   "crypto",
   "util",
@@ -81,7 +82,7 @@ const externals = [
   "codemirror/mode/yaml/yaml",
   "ipaddr.js",
   "jsbi",
-  "yebool",
+  "espresso-iisojs",
 ];
 
 function rmDirSync(dirPath): void {
@@ -297,7 +298,7 @@ async function generateFrontendJs(): Promise<void> {
   const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
   const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
 
-  const inlineDeps = ["mithril", "parsimmon", "jsbi", "yebool"];
+  const inlineDeps = ["mithril", "parsimmon", "jsbi", "espresso-iisojs"];
   const bundle = await rollup({
     input: inputFile,
     external: externals.filter((e) => !inlineDeps.includes(e)),
@@ -306,10 +307,14 @@ async function generateFrontendJs(): Promise<void> {
       {
         name: "",
         resolveId: function (importee, importer) {
-          if (importee.endsWith("/bigint"))
+          if (importee.endsWith("/bigint")) {
             return this.resolve(importee + "-jsbi", importer);
-          else if (importee === "yebool")
-            return this.resolve("yebool/dist/yebool-jsbi.mjs", importer);
+          } else if (importee === "espresso-iisojs") {
+            return this.resolve(
+              "espresso-iisojs/dist/espresso-iisojs-jsbi.mjs",
+              importer
+            );
+          }
           return null;
         },
       },
@@ -336,8 +341,8 @@ async function generateFrontendJs(): Promise<void> {
     file: outputFile,
   });
 
-  const webpackConf = {
-    mode: MODE,
+  const webpackConf: Configuration = {
+    mode: MODE as "development" | "production",
     entry: outputFile,
     resolve: {
       aliasFields: ["module"],
@@ -358,8 +363,13 @@ async function generateFrontendJs(): Promise<void> {
     },
   };
 
-  const stats = await promisify(webpack)(webpackConf);
+  return new Promise((resolve, reject) => {
+    webpack(webpackConf, (err, stats) => {
+      if (err) return reject(err);
       process.stdout.write(stats.toString({ colors: true }) + "\n");
+      resolve();
+    });
+  });
 }
 
 async function generateIconsSprite(): Promise<void> {

lib/api-functions.ts

@@ -29,22 +29,31 @@ import {
 import {
   httpConnectionRequest,
   udpConnectionRequest,
+  xmppConnectionRequest,
 } from "./connection-request";
 import { Expression, Task } from "./types";
 import { flattenDevice } from "./mongodb-functions";
 import { evaluate } from "./common/expression";
+import * as config from "../lib/config";
+
+const XMPP_CONFIGURED = !!config.get("XMPP_JID");
 
 export async function connectionRequest(
   deviceId: string,
   device?: Record<string, { value?: [boolean | number | string, string] }>
-): Promise<void> {
+): Promise<string> {
   if (!device) {
     const res = await db.devicesCollection.findOne({ _id: deviceId });
     if (!res) throw new Error("No such device");
     device = flattenDevice(res);
   }
 
-  let connectionRequestUrl, udpConnectionRequestAddress, username, password;
+  let connectionRequestUrl,
+    udpConnectionRequestAddress,
+    stunEnable,
+    connReqJabberId,
+    username,
+    password;
 
   if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
     connectionRequestUrl = (device[
@@ -55,6 +64,12 @@ export async function connectionRequest(
         "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
       ] || {}
     ).value || [""])[0];
+    stunEnable = ((
+      device["InternetGatewayDevice.ManagementServer.STUNEnable"] || {}
+    ).value || [""])[0];
+    connReqJabberId = ((
+      device["InternetGatewayDevice.ManagementServer.ConnReqJabberID"] || {}
+    ).value || [""])[0];
     username = ((
       device[
         "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
@@ -72,6 +87,10 @@ export async function connectionRequest(
     udpConnectionRequestAddress = ((
       device["Device.ManagementServer.UDPConnectionRequestAddress"] || {}
     ).value || [""])[0];
+    stunEnable = ((device["Device.ManagementServer.STUNEnable"] || {})
+      .value || [""])[0];
+    connReqJabberId = ((device["Device.ManagementServer.ConnReqJabberID"] || {})
+      .value || [""])[0];
     username = ((
       device["Device.ManagementServer.ConnectionRequestUsername"] || {}
     ).value || [""])[0];
@@ -146,7 +165,7 @@ export async function connectionRequest(
   const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
 
   let udpProm;
-  if (udpConnectionRequestAddress) {
+  if (udpConnectionRequestAddress && +stunEnable) {
     udpProm = udpConnectionRequest(
       udpConnectionRequestAddress,
       authExp,
@@ -156,8 +175,18 @@ export async function connectionRequest(
     );
   }
 
-  try {
-    await httpConnectionRequest(
+  let status;
+
+  if (connReqJabberId && XMPP_CONFIGURED) {
+    status = await xmppConnectionRequest(
+      connReqJabberId,
+      authExp,
+      CONNECTION_REQUEST_TIMEOUT,
+      debug,
+      deviceId
+    );
+  } else {
+    status = await httpConnectionRequest(
       connectionRequestUrl,
       authExp,
       CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
@@ -165,34 +194,47 @@ export async function connectionRequest(
       debug,
       deviceId
     );
-  } catch (err) {
-    if (!udpProm) throw err;
+
+    if (udpProm) {
       await udpProm;
+      return "";
     }
+  }
+
+  return status;
 }
 
-export async function watchTask(
+export async function awaitSessionStart(
   deviceId: string,
-  taskId: string,
+  lastInform: number,
   timeout: number
-): Promise<string> {
-  await new Promise((resolve) => setTimeout(resolve, 500));
-
-  const task = await db.tasksCollection.findOne(
-    { _id: taskId },
-    { projection: { _id: 1 } }
+): Promise<boolean> {
+  const now = Date.now();
+  const device = await db.devicesCollection.findOne(
+    { _id: deviceId },
+    { projection: { _lastInform: 1 } }
   );
-  if (!task) return "completed";
+  const li = (device["_lastInform"] as Date).getTime();
+  if (li > lastInform) return true;
+  const token = await cache.get(`cwmp_session_${deviceId}`);
+  if (token) return true;
+  if (timeout < 500) return false;
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionStart(deviceId, lastInform, timeout);
+}
 
-  const q = { _id: `${deviceId}:task_${taskId}` };
-  const fault = await db.faultsCollection.findOne(q, {
-    projection: { _id: 1 },
-  });
-  if (fault) return "fault";
-
-  if ((timeout -= 500) <= 0) return "timeout";
-
-  return watchTask(deviceId, taskId, timeout);
+export async function awaitSessionEnd(
+  deviceId: string,
+  timeout: number
+): Promise<boolean> {
+  const now = Date.now();
+  const token = await cache.get(`cwmp_session_${deviceId}`);
+  if (!token) return true;
+  if (timeout < 500) return false;
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionEnd(deviceId, timeout);
 }
 
 function sanitizeTask(task): void {
@@ -236,6 +278,10 @@ function sanitizeTask(task): void {
       break;
 
     case "refreshObject":
+      if (typeof task.objectName !== "string")
+        throw new Error("Missing 'objectName' property");
+      break;
+
     case "deleteObject":
       if (typeof task.objectName !== "string" || !task.objectName.length)
         throw new Error("Missing 'objectName' property");

lib/cache.ts

@@ -17,126 +17,102 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, Collection } from "mongodb";
+import { Collection } from "mongodb";
+import { onConnect } from "./db";
 import * as config from "./config";
 
+const CLOCK_SKEW_TOLERANCE = 30000;
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let clientPromise: Promise<MongoClient>;
-let mongoCollection: Collection;
-let mongoTimeOffset = 0;
+let cacheCollection: Collection;
 
-export async function connect(): Promise<void> {
-  const MONGODB_CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
-  clientPromise = MongoClient.connect(MONGODB_CONNECTION_URL, {
-    useNewUrlParser: true,
-    useUnifiedTopology: true,
-  });
-  const db = (await clientPromise).db();
-  mongoCollection = db.collection("cache");
-  await mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
-  const now = Date.now();
-  const res = await db.command({ hostInfo: 1 });
-  mongoTimeOffset = res.system.currentTime.getTime() - now;
-}
+onConnect(async (db) => {
+  cacheCollection = db.collection("cache");
+  await cacheCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+});
 
-export async function disconnect(): Promise<void> {
-  if (clientPromise) await (await clientPromise).close();
-}
-
-export async function get(key: string | string[]): Promise<any> {
-  const expire = new Date(Date.now() - mongoTimeOffset);
-  if (Array.isArray(key)) {
-    const res = await mongoCollection.find({ _id: { $in: key } }).toArray();
-
-    const indices = {};
-    key.forEach((v, i) => {
-      indices[v] = i;
-    });
-
-    const values = [];
-    res.forEach((r) => {
-      if (r["expire"] > expire) values[indices[r["_id"]]] = r["value"];
-    });
-
-    return values;
-  } else {
-    const res = await mongoCollection.findOne({ _id: { $in: [key] } });
-    if (res && res["expire"] > expire) return res["value"];
+export async function get(key: string): Promise<string> {
+  const res = await cacheCollection.findOne({ _id: key });
+  if (res) return res["value"];
   return null;
-  }
 }
 
-export async function del(key: string | string[]): Promise<void> {
-  if (Array.isArray(key))
-    await mongoCollection.deleteMany({ _id: { $in: key } });
-  else await mongoCollection.deleteOne({ _id: key });
+export async function del(key: string): Promise<void> {
+  await cacheCollection.deleteOne({ _id: key });
 }
 
 export async function set(
   key: string,
-  value: string | number,
+  value: string,
   ttl: number = MAX_CACHE_TTL
 ): Promise<void> {
-  const expire = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
-  await mongoCollection.replaceOne(
+  const timestamp = new Date();
+  const expire = new Date(
+    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000
+  );
+  await cacheCollection.replaceOne(
     { _id: key },
-    { _id: key, value: value, expire: expire },
+    { _id: key, value, expire, timestamp },
     { upsert: true }
   );
 }
 
 export async function pop(key: string): Promise<any> {
-  const res = await mongoCollection.findOneAndDelete({ _id: key });
-
-  if (
-    res &&
-    res["value"] &&
-    +res["value"]["expire"] - (Date.now() - mongoTimeOffset)
-  )
-    return res["value"]["value"];
-
+  const res = await cacheCollection.findOneAndDelete({ _id: key });
+  if (res && res["value"]) return res["value"]["value"];
   return null;
 }
 
-export async function lock(
+export async function acquireLock(
   lockName: string,
-  ttl: number
-): Promise<(extendTtl?: number) => Promise<void>> {
-  const token = Math.random().toString(36).slice(2);
+  ttl: number,
+  timeout = 0,
+  token = Math.random().toString(36).slice(2)
+): Promise<string> {
+  try {
+    const now = Date.now();
+    const r = await cacheCollection.findOneAndUpdate(
+      { _id: lockName, token },
+      {
+        $set: {
+          value: token,
+          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
+        },
+        $currentDate: { timestamp: true },
+      },
+      { upsert: true, returnOriginal: false }
+    );
+    const v = r.value;
+    if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
+      throw new Error("Database clock skew too great");
+  } catch (err) {
+    if (err.code === 11000) {
+      if (timeout > 0) {
+        return new Promise((resolve, reject) => {
+          const w = 50 + Math.random() * 50;
+          setTimeout(() => {
+            acquireLock(lockName, ttl, timeout - w, token).then(
+              resolve,
+              reject
+            );
+          }, w);
+        });
+      }
+      throw new Error("Failed to acquire lock");
+    }
+    throw err;
+  }
 
-  async function unlockOrExtend(extendTtl?: number): Promise<void> {
-    if (!extendTtl) {
-      const res = await mongoCollection.deleteOne({
+  return token;
+}
+
+export async function releaseLock(
+  lockName: string,
+  token: string
+): Promise<void> {
+  const res = await cacheCollection.deleteOne({
     _id: lockName,
     value: token,
   });
   if (res["result"]["n"] !== 1) throw new Error("Lock expired");
-    } else {
-      const expire = new Date(Date.now() - mongoTimeOffset + extendTtl * 1000);
-      const res = await mongoCollection.updateOne(
-        { _id: lockName, value: token },
-        { expire: expire }
-      );
-      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
-    }
-  }
-
-  const expireTest = new Date(Date.now() - mongoTimeOffset);
-  const expireSet = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
-
-  try {
-    await mongoCollection.updateOne(
-      { _id: lockName, expire: { $lte: expireTest } },
-      { $set: { value: token, expire: expireSet } },
-      { upsert: true }
-    );
-  } catch (err) {
-    if (err && err.code === 11000) {
-      await new Promise((resolve) => setTimeout(resolve, 200));
-      return lock(lockName, ttl);
-    }
-  }
-
-  return unlockOrExtend;
 }

lib/common.ts

@@ -47,3 +47,17 @@ export function generateDeviceId(
 export function escapeRegExp(str: string): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
+
+export function encodeTag(tag: string): string {
+  return encodeURIComponent(tag)
+    .replace(
+      /[!~*'()]/g,
+      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
+    )
+    .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")
+    .replace(/%/g, "0x");
+}
+
+export function decodeTag(tag: string): string {
+  return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%"));
+}

lib/common/boolean-expression.ts

@@ -18,7 +18,7 @@
  */
 
 import * as BI from "./bigint";
-import { espresso, complement, tautology } from "yebool";
+import { espresso, complement, tautology } from "espresso-iisojs";
 import { Expression } from "../types";
 import { map } from "./expression-parser";
 import { and, evaluateCallback } from "./expression";
@@ -362,6 +362,10 @@ class VarSynth extends BoolExprSynth {
         exp = exp.slice();
         exp[0] = ">";
         this.negate = true;
+      } else if (op === "NOT LIKE") {
+        exp = exp.slice();
+        exp[0] = "LIKE";
+        this.negate = true;
       }
     }
     this.expStr = JSON.stringify(exp);
@@ -809,6 +813,7 @@ function sopToExpression(
       if (Array.isArray(expr) && expr[0] === "NOT" && Array.isArray(expr[1])) {
         const e: Expression[] = expr[1];
         if (e[0] === "IS NULL") expr = ["IS NOT NULL", ...e.slice(1)];
+        else if (e[0] === "LIKE") expr = ["NOT LIKE", ...e.slice(1)];
         else if (e[0] === "=") expr = ["<>", ...e.slice(1)];
         else if (e[0] === "<>") expr = ["=", ...e.slice(1)];
         else if (e[0] === ">") expr = ["<=", ...e.slice(1)];
@@ -830,7 +835,6 @@ function sopToExpression(
 function generateDcSetAndIsNull(
   variables: Map<number, Expression>
 ): { dcSet: number[][]; isNull: Map<number, number> } {
-  // TODO support LIKE and NOT LIKE operators
   const relations: Map<
     string,
     { op: string; rhs: boolean | number | string; var: number }[]
@@ -843,6 +847,11 @@ function generateDcSetAndIsNull(
       let r = relations.get(lhs);
       if (!r) relations.set(lhs, (r = []));
       r.push({ op: op, rhs: e[2], var: n });
+    } else if (op === "LIKE" && !Array.isArray(e[2]) && !Array.isArray(e[3])) {
+      const lhs = JSON.stringify(e[1]);
+      let r = relations.get(lhs);
+      if (!r) relations.set(lhs, (r = []));
+      r.push({ op: "", rhs: null, var: n });
     } else {
       const lhs = JSON.stringify(e);
       let r = relations.get(lhs);
@@ -1019,8 +1028,7 @@ function boolExprSynthToExpression(boolExpr: BoolExprSynth): Expression {
   const { dcSet, isNull } = generateDcSetAndIsNull(variables);
   minterms = sanitizeMinterms(minterms, isNull);
 
-  const canRaise = (idx: number, set: Set<number>): boolean =>
-    !isNull.has(idx >> 2) || !(idx & 1) || !set.has(idx ^ 3);
+  const canRaise = getCanRaiseCallback(isNull);
   minterms = espresso(minterms, dcSet, { canRaise });
   return sopToExpression(minterms, variables);
 }

lib/common/expression-parser.ts

@@ -42,7 +42,7 @@ function interpretEscapes(str): string {
 
 export function map(
   exp: Expression,
-  callback: (exp: Expression) => Expression
+  callback: (e: Expression) => Expression
 ): Expression {
   if (!Array.isArray(exp)) return callback(exp);
 
@@ -60,7 +60,7 @@ export function map(
 
 export async function mapAsync(
   exp: Expression,
-  callback: (exp: Expression) => Promise<Expression>
+  callback: (e: Expression) => Promise<Expression>
 ): Promise<Expression> {
   if (!Array.isArray(exp)) return callback(exp);
 

lib/common/expression.ts

@@ -264,21 +264,21 @@ export function evaluateCallback(exp: Expression): Expression {
 
 export function evaluate(
   exp: Expression,
-  obj: Record<string, unknown> | ((exp: string) => Expression),
+  obj: Record<string, unknown> | ((e: string) => Expression),
   now: number,
-  cb?: (exp: Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): string | number | boolean | null;
 export function evaluate(
   exp: Expression,
-  obj?: Record<string, unknown> | ((exp: string) => Expression),
+  obj?: Record<string, unknown> | ((e: string) => Expression),
   now?: number,
-  cb?: (exp: Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): Expression;
 export function evaluate(
   exp: Expression,
-  obj?: Record<string, unknown> | ((exp: string) => Expression),
+  obj?: Record<string, unknown> | ((e: string) => Expression),
   now?: number,
-  cb?: (exp: Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): Expression {
   return map(exp, (e) => {
     if (cb) e = cb(e);
@@ -306,19 +306,19 @@ export async function evaluateAsync(
   exp: Expression,
   obj: Record<string, unknown>,
   now: number,
-  cb?: (exp: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<string | number | boolean | null>;
 export async function evaluateAsync(
   exp: Expression,
   obj?: Record<string, unknown>,
   now?: number,
-  cb?: (exp: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<Expression>;
 export async function evaluateAsync(
   exp: Expression,
   obj?: Record<string, unknown>,
   now?: number,
-  cb?: (exp: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<Expression> {
   return mapAsync(exp, async (e) => {
     if (cb) e = await cb(e);

lib/config.ts

@@ -101,6 +101,9 @@ const options = {
 
   // Should probably never be changed
   DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
+
+  XMPP_JID: { type: "string", default: "" },
+  XMPP_PASSWORD: { type: "string", default: "" },
 };
 
 const allConfig: { [name: string]: string | number } = {};

lib/connection-request.ts

@@ -26,6 +26,10 @@ import { Expression } from "./types";
 import * as auth from "./auth";
 import * as extensions from "./extensions";
 import * as debug from "./debug";
+import XmppClient from "./xmpp-client";
+import * as config from "../lib/config";
+import { encodeEntities, parseAttrs, Element } from "./xml-parser";
+import * as logger from "../lib/logger";
 
 async function extractAuth(
   exp: Expression,
@@ -62,7 +66,6 @@ async function extractAuth(
 
 function httpGet(
   options: http.RequestOptions,
-  timeout,
   _debug: boolean,
   deviceId: string
 ): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
@@ -77,15 +80,12 @@ function httpGet(
         }
       })
       .on("error", (err) => {
-        req.abort();
-        reject(new Error("Device is offline"));
+        req.destroy();
+        reject(err);
         if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
       })
-      .on("socket", (socket) => {
-        socket.setTimeout(timeout);
-        socket.on("timeout", () => {
-          req.abort();
-        });
+      .on("timeout", () => {
+        req.destroy();
       });
   });
 }
@@ -97,14 +97,15 @@ export async function httpConnectionRequest(
   timeout: number,
   _debug: boolean,
   deviceId: string
-): Promise<void> {
+): Promise<string> {
   const options: http.RequestOptions = parse(address);
   if (options.protocol !== "http:")
-    throw new Error("Invalid connection request URL or protocol");
+    return "Invalid connection request URL or protocol";
 
   options.agent = new http.Agent({
     maxSockets: 1,
     keepAlive: true,
+    timeout: timeout,
   });
 
   let authHeader: Record<string, string>;
@@ -115,8 +116,7 @@ export async function httpConnectionRequest(
     let opts = options;
     if (authHeader) {
       if (authHeader["method"] === "Basic") {
-        if (!allowBasicAuth)
-          throw new Error("Basic HTTP authentication not allowed");
+        if (!allowBasicAuth) return "Basic HTTP authentication not allowed";
 
         opts = Object.assign(
           {
@@ -143,17 +143,34 @@ export async function httpConnectionRequest(
           options
         );
       } else {
-        throw new Error("Unrecognized auth method");
+        return "Unrecognized auth method";
       }
     }
 
-    let res = await httpGet(opts, timeout, _debug, deviceId);
-
+    let res: { statusCode: number; headers: http.IncomingHttpHeaders };
+    try {
+      res = await httpGet(opts, _debug, deviceId);
+    } catch (err) {
       // Workaround for some devices unexpectedly closing the connection
-    if (res.statusCode === 0 && authHeader)
-      res = await httpGet(opts, timeout, _debug, deviceId);
-    if (res.statusCode === 0) throw new Error("Device is offline");
-    if (res.statusCode === 200 || res.statusCode === 204) return;
+      if (authHeader) {
+        try {
+          res = await httpGet(opts, _debug, deviceId);
+        } catch (err) {
+          return `Connection request error: ${err.message}`;
+        }
+      }
+
+      if (err["code"] === "ECONNRESET" || err["code"] === "ECONNREFUSED")
+        return "Device is offline";
+
+      return `Connection request error: ${err.message}`;
+    }
+
+    if (res.statusCode === 200 || res.statusCode === 204) return "";
+
+    // When a Connection Request is received for the Virtual CWMP Device and the Proxied
+    // Device is offline the CPE Proxier MUST respond with an HTTP 503 failure
+    if (res.statusCode === 503) return "Device is offline";
 
     if (res.statusCode === 401 && res.headers["www-authenticate"]) {
       authHeader = auth.parseWwwAuthenticateHeader(
@@ -161,12 +178,11 @@ export async function httpConnectionRequest(
       );
       [username, password, authExp] = await extractAuth(authExp, false);
     } else {
-      throw new Error(
-        `Unexpected response code from device: ${res.statusCode}`
-      );
+      return `Connection request error: Unexpected status code ${res.statusCode}`;
     }
   }
-  throw new Error("Incorrect connection request credentials");
+
+  return "Connection request error: Incorrect connection request credentials";
 }
 
 export async function udpConnectionRequest(
@@ -222,3 +238,84 @@ export async function udpConnectionRequest(
   }
   client.close();
 }
+
+const XMPP_JID = config.get("XMPP_JID") as string;
+const XMPP_PASSWORD = config.get("XMPP_PASSWORD") as string;
+const XMPP_RESOURCE = crypto.randomBytes(8).toString("hex");
+
+let xmppClient: XmppClient;
+
+function xmppClientOnError(err: Error): void {
+  xmppClient = null;
+  logger.error({
+    message: "XMPP exception",
+    exception: err,
+    pid: process.pid,
+  });
+}
+
+function xmppClientOnClose(): void {
+  xmppClient = null;
+}
+
+export async function xmppConnectionRequest(
+  jid: string,
+  authExp: Expression,
+  timeout: number,
+  _debug: boolean,
+  deviceId: string
+): Promise<string> {
+  if (!xmppClient) {
+    const [host, username] = XMPP_JID.split("@").reverse();
+    xmppClient = await XmppClient.connect({
+      host,
+      username,
+      resource: XMPP_RESOURCE,
+      password: XMPP_PASSWORD,
+      timeout: 120000,
+    });
+    xmppClient.on("error", xmppClientOnError);
+    xmppClient.on("close", xmppClientOnClose);
+    xmppClient.unref();
+  }
+
+  let username: string;
+  let password: string;
+
+  [username, password, authExp] = await extractAuth(authExp, null);
+  while (username != null && password != null) {
+    const msg = `<connectionRequest xmlns="urn:broadband-forum-org:cwmp:xmppConnReq-1-0"><username>${encodeEntities(
+      username
+    )}</username><password>${encodeEntities(
+      password
+    )}</password></connectionRequest>`;
+    let res: Element, rawRes: string, rawReq: string;
+    try {
+      ({ res, rawRes, rawReq } = await xmppClient.sendIqStanza(
+        XMPP_JID,
+        jid,
+        "get",
+        msg,
+        timeout
+      ));
+    } catch (err) {
+      return err.message;
+    }
+    if (_debug) {
+      debug.outgoingXmppStanza(deviceId, rawReq);
+      debug.incomingXmppStanza(deviceId, rawRes);
+    }
+    const attrs = parseAttrs(res.attrs);
+    const type = attrs.find((a) => a.name === "type");
+    if (type && type.value === "result") return "";
+    const error = res.children.find((c) => c.name === "error");
+    if (!error || !error.children[0])
+      return "Unexpected XMPP connection request response";
+    if (error.children[0].name === "service-unavailable")
+      return "Device is offline";
+    if (error.children[0].name !== "not-authorized")
+      return "Unexpected XMPP connection request response";
+    [username, password, authExp] = await extractAuth(authExp, null);
+  }
+  return "Incorrect connection request credentials";
+}

lib/cwmp.ts

@@ -90,9 +90,7 @@ async function authenticate(
   }
 
   if (authentication && authentication.method === "Digest") {
-    const sessionNonce = sessionsNonces.get(
-      sessionContext.httpRequest.connection
-    );
+    const sessionNonce = sessionsNonces.get(sessionContext.httpRequest.socket);
 
     if (
       !sessionNonce ||
@@ -185,7 +183,7 @@ async function writeResponse(
   }
 
   const httpResponse = sessionContext.httpResponse;
-  const connection = httpResponse.connection;
+  const connection = httpResponse.socket;
 
   httpResponse.setHeader("Content-Length", Buffer.byteLength(data));
   httpResponse.writeHead(res.code, res.headers);
@@ -198,17 +196,23 @@ async function writeResponse(
       sessionContext: sessionContext,
       message: "Connection dropped",
     });
+    await endSession(sessionContext);
   } else if (close) {
-    const isNew = await endSession(sessionContext);
-    if (isNew) {
-      logger.accessInfo({
-        sessionContext: sessionContext,
-        message: "New device registered",
-      });
-    }
+    session.clearProvisions(sessionContext);
+    await endSession(sessionContext);
   } else {
-    sessionContext.lastActivity = Date.now();
+    const now = Date.now();
+    sessionContext.lastActivity = now;
     currentSessions.set(connection, sessionContext);
+    if (now >= sessionContext.extendLock) {
+      sessionContext.extendLock = now + 10000;
+      await cache.acquireLock(
+        `cwmp_session_${sessionContext.deviceId}`,
+        sessionContext.timeout + 10000,
+        0,
+        sessionContext.sessionId
+      );
+    }
   }
 }
 
@@ -728,9 +732,20 @@ async function nextRpc(sessionContext: SessionContext): Promise<void> {
   return nextRpc(sessionContext);
 }
 
-async function endSession(sessionContext: SessionContext): Promise<boolean> {
+async function endSession(sessionContext: SessionContext): Promise<void> {
   let saveCache = sessionContext.cacheUntil != null;
 
+  if (sessionContext.provisions.length) {
+    const fault = {
+      code: "session_terminated",
+      message: "The TR-069 session was unsuccessfully terminated",
+      timestamp: sessionContext.timestamp,
+    };
+    recordFault(sessionContext, fault);
+    // No need to save retryNow
+    for (const f of Object.values(sessionContext.faults)) delete f.retryNow;
+  }
+
   const promises = [];
 
   promises.push(
@@ -793,7 +808,16 @@ async function endSession(sessionContext: SessionContext): Promise<boolean> {
   }
 
   await Promise.all(promises);
-  return sessionContext.new;
+  await cache.releaseLock(
+    `cwmp_session_${sessionContext.deviceId}`,
+    sessionContext.sessionId
+  );
+  if (sessionContext.new) {
+    logger.accessInfo({
+      sessionContext: sessionContext,
+      message: "New device registered",
+    });
+  }
 }
 
 async function sendAcsRequest(
@@ -842,8 +866,11 @@ async function sendAcsRequest(
   return writeResponse(sessionContext, res);
 }
 
-async function getSession(connection, sessionId): Promise<SessionContext> {
-  const sessionContext = currentSessions.get(connection);
+async function getSession(
+  connection: Socket,
+  sessionId: string
+): Promise<SessionContext> {
+  let sessionContext = currentSessions.get(connection);
   if (sessionContext) {
     currentSessions.delete(connection);
     return sessionContext;
@@ -855,7 +882,9 @@ async function getSession(connection, sessionId): Promise<SessionContext> {
 
   const sessionContextString = await cache.pop(`session_${sessionId}`);
   if (!sessionContextString) return null;
-  return session.deserialize(sessionContextString);
+  sessionContext = await session.deserialize(sessionContextString);
+  connection.setTimeout(sessionContext.timeout);
+  return sessionContext;
 }
 
 // Only needed to prevent tree shaking from removing the remoteAddress
@@ -875,15 +904,21 @@ export function onConnection(socket: Socket): void {
     const now = Date.now();
 
     const lastActivity = sessionContext.lastActivity;
-    const timeoutMsg = logger.flatten({
+    const timeoutMsg = {
       sessionContext: sessionContext,
       message: "Session timeout",
       sessionTimestamp: sessionContext.timestamp,
-    });
+    };
 
     const timeout =
       sessionContext.lastActivity + sessionContext.timeout * 1000 - now;
-    if (timeout <= 0) return void logger.accessError(timeoutMsg);
+
+    if (timeout <= 0) {
+      logger.accessError(timeoutMsg);
+      // TODO it's possible that lock would have already been expired
+      await endSession(sessionContext);
+      return;
+    }
 
     setTimeout(async () => {
       const sessionContextString = await cache.get(
@@ -891,8 +926,10 @@ export function onConnection(socket: Socket): void {
       );
       if (!sessionContextString) return;
       const _sessionContext = await session.deserialize(sessionContextString);
-      if (_sessionContext.lastActivity === lastActivity)
+      if (_sessionContext.lastActivity === lastActivity) {
         logger.accessError(timeoutMsg);
+        await endSession(sessionContext);
+      }
     }, timeout + 1000).unref();
 
     if (sessionContext.state === 0) return;
@@ -988,7 +1025,7 @@ async function reportBadState(sessionContext: SessionContext): Promise<void> {
     sessionContext: sessionContext,
   });
   const httpResponse = sessionContext.httpResponse;
-  currentSessions.delete(httpResponse.connection);
+  currentSessions.delete(httpResponse.socket);
   const body = "Bad session state";
   httpResponse.setHeader("Content-Length", Buffer.byteLength(body));
   httpResponse.writeHead(400, { Connection: "close" });
@@ -1015,14 +1052,14 @@ async function responseUnauthorized(
       resHeaders["WWW-Authenticate"] = `Basic realm="${REALM}"`;
     } else {
       const nonce = crypto.randomBytes(16).toString("hex");
-      sessionsNonces.set(sessionContext.httpRequest.connection, nonce);
+      sessionsNonces.set(sessionContext.httpRequest.socket, nonce);
       let d = `Digest realm="${REALM}"`;
       d += ',qop="auth,auth-int"';
       d += `,nonce="${nonce}"`;
 
       resHeaders["WWW-Authenticate"] = d;
     }
-    currentSessions.set(sessionContext.httpRequest.connection, sessionContext);
+    currentSessions.set(sessionContext.httpRequest.socket, sessionContext);
   }
 
   const httpResponse = sessionContext.httpResponse;
@@ -1042,7 +1079,6 @@ async function processRequest(
 ): Promise<void> {
   for (const w of parseWarnings) {
     w.sessionContext = sessionContext;
-    w.rpc = rpc;
     logger.accessWarn(w);
   }
 
@@ -1070,6 +1106,8 @@ async function processRequest(
       );
     }
 
+    sessionContext.httpRequest.socket.setTimeout(sessionContext.timeout * 1000);
+
     if (sessionContext.debug) {
       debug.incomingHttpRequest(
         sessionContext.httpRequest,
@@ -1088,6 +1126,37 @@ async function processRequest(
       }
     }
 
+    try {
+      sessionContext.extendLock = sessionContext.timestamp + 10000;
+      await cache.acquireLock(
+        `cwmp_session_${sessionContext.deviceId}`,
+        sessionContext.timeout + 10000,
+        0,
+        sessionContext.sessionId
+      );
+    } catch (err) {
+      logger.accessError({
+        message: "CPE already in session",
+        sessionContext: sessionContext,
+      });
+
+      const _body = "CPE already in session";
+      sessionContext.httpResponse.setHeader(
+        "Content-Length",
+        Buffer.byteLength(_body)
+      );
+      sessionContext.httpResponse.writeHead(400, { Connection: "close" });
+      if (sessionContext.debug) {
+        debug.outgoingHttpResponse(
+          sessionContext.httpResponse,
+          sessionContext.deviceId,
+          _body
+        );
+      }
+      sessionContext.httpResponse.end(_body);
+      return;
+    }
+
     sessionContext.state = 1;
     sessionContext.authState = 2;
 
@@ -1116,6 +1185,7 @@ async function processRequest(
         sessionContext.authState = 1;
         return responseUnauthorized(sessionContext, false);
       } else {
+        await endSession(sessionContext);
         return responseUnauthorized(sessionContext, true);
       }
     }
@@ -1210,19 +1280,24 @@ export function listener(
       stats.concurrentRequests -= 1;
     })
     .catch((err) => {
-      currentSessions.delete(httpResponse.connection);
-      httpResponse.writeHead(500, { Connection: "close" });
-      httpResponse.end(`${err.name}: ${err.message}`);
+      currentSessions.delete(httpResponse.socket);
       stats.concurrentRequests -= 1;
       setTimeout(() => {
         throw err;
       });
+      try {
+        httpResponse.socket.unref();
+        httpResponse.writeHead(500, { Connection: "close" });
+        httpResponse.end(`${err.name}: ${err.message}`);
+      } catch (err) {
+        // Ignore
+      }
     });
 }
 
 function decodeString(buffer: Buffer, charset: string): string {
   try {
-    return buffer.toString(charset);
+    return buffer.toString(charset as BufferEncoding);
   } catch (err) {
     if (encodingExists(charset)) return decode(buffer, charset);
   }
@@ -1307,9 +1382,9 @@ async function listenerAsync(
   // Request aborted
   if (!body) return;
 
-  const newConnection = !currentSessions.has(httpRequest.connection);
+  const newConnection = !currentSessions.has(httpRequest.socket);
 
-  const sessionContext = await getSession(httpRequest.connection, sessionId);
+  const sessionContext = await getSession(httpRequest.socket, sessionId);
 
   if (sessionContext) {
     sessionContext.httpRequest = httpRequest;
@@ -1334,6 +1409,7 @@ async function listenerAsync(
         );
       }
       httpResponse.end(_body);
+      await endSession(sessionContext);
       return;
     }
     if (newConnection && sessionContext.authState !== 1)
@@ -1347,7 +1423,7 @@ async function listenerAsync(
     return;
   }
 
-  let charset;
+  let charset: string;
   if (httpRequest.headers["content-type"]) {
     const m = httpRequest.headers["content-type"].match(
       /charset=['"]?([^'"\s]+)/i
@@ -1384,6 +1460,7 @@ async function listenerAsync(
         );
         debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, msg);
       }
+      await endSession(sessionContext);
     } else {
       const cacheSnapshot = await localCache.getCurrentSnapshot();
       const d = !!localCache.getConfig(
@@ -1409,7 +1486,7 @@ async function listenerAsync(
   }
 
   const parseWarnings = [];
-  let rpc;
+  let rpc: SoapMessage;
   try {
     rpc = soap.request(
       bodyStr,
@@ -1440,6 +1517,7 @@ async function listenerAsync(
           error.message
         );
       }
+      await endSession(sessionContext);
     } else {
       const cacheSnapshot = await localCache.getCurrentSnapshot();
       const d = !!localCache.getConfig(
@@ -1514,7 +1592,6 @@ async function listenerAsync(
   _sessionContext.httpRequest = httpRequest;
   _sessionContext.httpResponse = httpResponse;
   _sessionContext.sessionId = crypto.randomBytes(8).toString("hex");
-  httpRequest.connection.setTimeout(_sessionContext.timeout * 1000);
 
   const {
     tasks: dueTasks,

lib/db.ts

@@ -17,9 +17,9 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, ObjectID, Collection } from "mongodb";
+import { MongoClient, ObjectID, Collection, Db } from "mongodb";
 import { get } from "./config";
-import { escapeRegExp } from "./common";
+import { encodeTag, escapeRegExp } from "./common";
 import { parse } from "./common/expression-parser";
 import {
   DeviceData,
@@ -45,7 +45,6 @@ export let tasksCollection: Collection,
   configCollection: Collection;
 
 let clientPromise: Promise<MongoClient>;
-export let client: MongoClient;
 
 function compareAccessLists(list1: string[], list2: string[]): boolean {
   if (list1.length !== list2.length) return false;
@@ -53,15 +52,24 @@ function compareAccessLists(list1: string[], list2: string[]): boolean {
   return true;
 }
 
+const onConnectCallbacks: ((db: Db) => Promise<void>)[] = [];
+
+export function onConnect(callback: (db: Db) => Promise<void>): void {
+  onConnectCallbacks.push(callback);
+}
+
 export async function connect(): Promise<void> {
   clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
     useNewUrlParser: true,
     useUnifiedTopology: true,
   });
 
-  client = await clientPromise;
+  const client = await clientPromise;
   const db = client.db();
+  await Promise.all(onConnectCallbacks.map((c) => c(db)));
+}
 
+onConnect(async (db) => {
   tasksCollection = db.collection("tasks");
   await tasksCollection.createIndex({ device: 1, timestamp: 1 });
 
@@ -76,7 +84,7 @@ export async function connect(): Promise<void> {
   permissionsCollection = db.collection("permissions");
   usersCollection = db.collection("users");
   configCollection = db.collection("config");
-}
+});
 
 export async function disconnect(): Promise<void> {
   if (clientPromise) await (await clientPromise).close();
@@ -234,10 +242,9 @@ export async function fetchDevice(
           ]);
         }
 
-        for (let t of v as string[]) {
-          t = t.replace(/[^a-zA-Z0-9-]+/g, "_");
+        for (const t of v as string[]) {
           res.push([
-            Path.parse("Tags." + t),
+            Path.parse("Tags." + encodeTag(t)),
             timestamp,
             {
               object: [timestamp, 0],
@@ -691,6 +698,11 @@ export async function getOperations(
   for (const r of res) {
     const commandKey = r._id.slice(deviceId.length + 1);
     delete r._id;
+    // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
+    if (typeof r.provisions !== "string") {
+      operations[commandKey] = r;
+      continue;
+    }
     r.timestamp = +r.timestamp;
     if (r.args) r.args = JSON.parse(r.args);
     r.provisions = JSON.parse(r.provisions);
@@ -712,7 +724,7 @@ export async function saveOperation(
   o["provisions"] = JSON.stringify(operation.provisions);
   o["retries"] = JSON.stringify(operation.retries);
   o["args"] = JSON.stringify(operation.args);
-  await operationsCollection.replaceOne({ _id: id }, operation, {
+  await operationsCollection.replaceOne({ _id: id }, o, {
     upsert: true,
   });
 }

lib/debug.ts

@@ -49,7 +49,7 @@ export function incomingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
   const msg = {
     event: "incoming HTTP request",
     timestamp: now,
@@ -77,7 +77,7 @@ export function outgoingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
   const msg = {
     event: "outgoing HTTP response",
     timestamp: now,
@@ -104,7 +104,7 @@ export function outgoingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
@@ -160,13 +160,13 @@ export function incomingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
   const msg = {
     event: "incoming HTTP response",
     timestamp: now,
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
-    connection: getConnectionTimestamp(httpResponse.connection),
+    connection: getConnectionTimestamp(httpResponse.socket),
     statusCode: httpResponse.statusCode,
     headers: httpResponse.headers,
     body: body,
@@ -202,3 +202,37 @@ export function outgoingUdpMessage(
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
 }
+
+export function outgoingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function incomingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "incoming XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}

lib/extensions.ts

@@ -58,6 +58,7 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
             });
           }
 
+          // eslint-disable-next-line @typescript-eslint/no-floating-promises
           kill(processes[scriptName]);
           delete processes[scriptName];
         }

lib/fs.ts

@@ -20,11 +20,19 @@
 import * as url from "url";
 import * as querystring from "querystring";
 import { IncomingMessage, ServerResponse } from "http";
-import { GridFSBucket } from "mongodb";
-import * as db from "./db";
+import { Collection, GridFSBucket } from "mongodb";
+import { onConnect } from "./db";
 import * as logger from "./logger";
 import { getRequestOrigin } from "./forwarded";
 
+let filesCollection: Collection;
+let filesBucket: GridFSBucket;
+
+onConnect(async (db) => {
+  filesCollection = db.collection("fs.files");
+  filesBucket = new GridFSBucket(db);
+});
+
 export function listener(
   request: IncomingMessage,
   response: ServerResponse
@@ -39,7 +47,7 @@ export function listener(
       remoteAddress: getRequestOrigin(request).remoteAddress,
     };
 
-    db.filesCollection.findOne({ _id: filename }, (err, file) => {
+    filesCollection.findOne({ _id: filename }, (err, file) => {
       if (err) throw err;
 
       if (!file) {
@@ -55,8 +63,7 @@ export function listener(
         "Content-Length": file.length,
       });
 
-      const bucket = new GridFSBucket(db.client.db());
-      const downloadStream = bucket.openDownloadStreamByName(filename);
+      const downloadStream = filesBucket.openDownloadStreamByName(filename);
       downloadStream.pipe(response);
 
       logger.accessInfo(log);

lib/local-cache.ts

@@ -22,7 +22,7 @@ import * as crypto from "crypto";
 import * as config from "./config";
 import * as db from "./db";
 import * as cache from "./cache";
-import { mongoQueryToFilter } from "./mongodb-functions";
+import { convertOldPrecondition } from "./mongodb-functions";
 import * as logger from "./logger";
 import * as scheduling from "./scheduling";
 import * as expression from "./common/expression";
@@ -173,7 +173,9 @@ async function fetchPresets(): Promise<Preset[]> {
       try {
         precondition = parse(preset["precondition"]);
       } catch (error) {
-        precondition = mongoQueryToFilter(JSON.parse(preset["precondition"]));
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"])
+        );
       }
 
       // Simplify expression
@@ -186,7 +188,7 @@ async function fetchPresets(): Promise<Preset[]> {
     for (const c of preset["configurations"]) {
       switch (c.type) {
         case "age":
-          _provisions.push(["refresh", c.name, c.age]);
+          _provisions.push(["refresh", c.name, +c.age]);
           break;
 
         case "value":
@@ -396,7 +398,7 @@ async function refresh(): Promise<void> {
     return;
   }
 
-  const unlockOrExtend = await cache.lock("presets_hash_lock", 3);
+  const lockToken = await cache.acquireLock("presets_hash_lock", 4000, 4000);
 
   const res = await Promise.all([
     fetchPresets(),
@@ -430,7 +432,7 @@ async function refresh(): Promise<void> {
   currentSnapshot = computeHash(snapshot);
   snapshots.set(currentSnapshot, snapshot);
   await cache.set("presets_hash", currentSnapshot, 300);
-  await unlockOrExtend(0);
+  await cache.releaseLock("presets_hash_lock", lockToken);
 
   nextRefresh = now + (REFRESH - (now % REFRESH));
 }
@@ -473,7 +475,7 @@ export function getConfig(
   key: string,
   context: Record<string, unknown>,
   now: number,
-  cb?: (Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): string | number | boolean | null {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");

lib/mongodb-functions.ts

@@ -19,8 +19,9 @@
 
 import { ObjectID } from "mongodb";
 import { map, parse, stringify, parseList } from "./common/expression-parser";
-import { likePatternToRegExp, evaluate } from "./common/expression";
+import { likePatternToRegExp } from "./common/expression";
 import { Expression, Fault, Task } from "./types";
+import { decodeTag, encodeTag } from "./common";
 
 const isArray = Array.isArray;
 
@@ -43,13 +44,11 @@ export function processDeviceFilter(filter: Expression): Expression {
       exp[1][0] === "PARAM" &&
       exp[1][1].startsWith("Tags.")
     ) {
-      const t = exp[1][1].slice(5);
-      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
-      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
-      else if (exp[0] === "=" && exp[2] === true)
-        return ["=", ["PARAM", "_tags"], t];
-      else if (exp[0] === "<>" && exp[2] !== true)
-        return ["=", ["PARAM", "_tags"], t];
+      const t = decodeTag(exp[1][1].slice(5));
+      if (exp[0] === "IS NULL") return ["_tags", t, false];
+      else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
+      else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
+      else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
     } else if (
       [
         "=",
@@ -117,140 +116,131 @@ export function processFaultsFilter(filter: Expression): Expression {
 }
 
 export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
-  const ops = {
-    OR: 0,
-    AND: 0,
-    NOT: 0,
-    "=": 1,
-    "<>": 1,
-    ">": 1,
-    ">=": 1,
-    "<": 1,
-    "<=": 1,
-    LIKE: 1,
-    "NOT LIKE": 1,
-    "IS NULL": 1,
-    "IS NOT NULL": 1,
-  };
-
-  function recursive(filter, negate, res = {}): Record<string, unknown> {
+  function recursive(filter, negate): Record<string, unknown> {
     const op = filter[0];
 
-    if (ops[op] === 0) {
-      if (op === "AND") filter = filter.filter((f) => f !== true);
-      if (op === "OR") filter = filter.filter((f) => f !== false);
-      if (filter.length === 2) return recursive(filter[1], negate, res);
-
-      for (let i = 1; i < filter.length; ++i) {
-        if (!isArray(filter[i]) || ops[filter[i][0]] == null)
-          throw new Error(`Invalid expression in ${op} clause`);
+    if (op === "AND") {
+      filter = filter.filter((f) => f !== true);
+      if (filter.length === 2) return recursive(filter[1], negate);
     }
 
-      if ((!negate && op === "AND") || (negate && op === "OR")) {
-        res["$and"] = res["$and"] || [];
-        for (let i = 1; i < filter.length; ++i)
-          res["$and"].push(recursive(filter[i], negate));
-      } else if ((!negate && op === "OR") || (negate && op === "AND")) {
-        res["$or"] = res["$or"] || [];
-
-        for (let i = 1; i < filter.length; ++i)
-          res["$or"].push(recursive(filter[i], negate));
-      } else if (op === "NOT") {
-        recursive(filter[1], !negate, res);
-      }
-    } else if (ops[op] === 1) {
-      if (isArray(filter[2])) {
-        throw new Error(`Right operand of ${op} clause is not a primitive`);
-      } else if (
-        ["LIKE", "NOT LIKE"].includes(op) &&
-        isArray(filter[1]) &&
-        filter[1][0] === "FUNC" &&
-        ["UPPER", "LOWER"].includes(filter[1][1])
-      ) {
-        if (
-          (filter[1][1] === "UPPER" && filter[2] !== filter[2].toUpperCase()) ||
-          (filter[1][1] === "LOWER" && filter[2] !== filter[2].toLowerCase())
-        ) {
-          throw new Error(
-            `Cannot compare ${
-              filter[1][1]
-            }() against non-${filter[1][1].toLowerCase()}case pattern`
-          );
-        }
-      } else if (!isArray(filter[1]) || filter[1][0] !== "PARAM") {
-        throw new Error(`Left operand of ${op} clause is not a parameter`);
+    if (op === "OR") {
+      filter = filter.filter((f) => f !== false);
+      if (filter.length === 2) return recursive(filter[1], negate);
     }
 
-      if (op === "=") {
-        const param = filter[1][1];
-        const p = (res[param] = res[param] || {});
-        if (negate) p["$ne"] = filter[2];
-        else p["$eq"] = filter[2];
-      } else if (op === "<>") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$ne"] = filter[2];
-        if (param !== "_tags") p["$exists"] = true;
-      } else if (op === ">") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gt"] = filter[2];
-      } else if (op === ">=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gte"] = filter[2];
-      } else if (op === "<") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lt"] = filter[2];
-      } else if (op === "<=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lte"] = filter[2];
-      } else if (op === "IS NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: negate };
-      } else if (op === "IS NOT NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: !negate };
-      } else {
+    if ((!negate && op === "AND") || (negate && op === "OR"))
+      return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if ((!negate && op === "OR") || (negate && op === "AND"))
+      return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if (op === "NOT") return recursive(filter[1], !negate);
+
+    if (isArray(filter[2]))
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "LIKE" || op === "NOT LIKE") {
       if (op === "NOT LIKE") negate = !negate;
       let param;
       let flags;
+      if (!Array.isArray(filter[1]))
+        throw new Error(`Invalid LHS operand of ${op} clause`);
       if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
+        if (filter[2] !== filter[2].toUpperCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
         param = filter[1][2][1];
         flags = "i";
       } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
+        if (filter[2] !== filter[2].toLowerCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
         param = filter[1][2][1];
         flags = "i";
-        } else {
+      } else if (filter[1][0] === "PARAM") {
         param = filter[1][1];
         flags = "";
+      } else {
+        throw new Error(`Invalid LHS operand of ${op} clause`);
       }
       const r = likePatternToRegExp(filter[2], filter[3], flags);
-        if (negate) res[param] = { $not: r };
-        else res[param] = r;
+      if (negate) return { [param]: { $nin: [r, null] } };
+      else return { [param]: r };
     }
-    } else {
+
+    if (op === "_tags") {
+      let t = filter[2];
+      if (negate) t = !t;
+      if (t) return { _tags: filter[1] };
+      else return { _tags: { $ne: filter[1] } };
+    }
+
+    if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
+      throw new Error(`Invalid LHS operand of ${op} clause`);
+
+    if (op === "IS NULL") {
+      const p = filter[1][1];
+      return { [p]: null };
+    }
+
+    if (op === "IS NOT NULL") {
+      const p = filter[1][1];
+      return { [p]: { $ne: null } };
+    }
+
+    if (isArray(filter[2]) || filter[2] == null)
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (!negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === "<>") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === ">") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lte: v } };
+      else return { [p]: { $gt: v } };
+    }
+
+    if (op === ">=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lt: v } };
+      else return { [p]: { $gte: v } };
+    }
+
+    if (op === "<") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gte: v } };
+      else return { [p]: { $lt: v } };
+    }
+
+    if (op === "<=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gt: v } };
+      else return { [p]: { $lte: v } };
+    }
+
     throw new Error(`Unrecognized operator ${op}`);
   }
 
-    return res;
-  }
-
-  const _exp = evaluate(exp, null, Date.now());
-
-  if (!isArray(_exp)) {
-    if (_exp === true) return {};
+  if (!isArray(exp)) {
+    if (exp === true) return {};
     throw new Error("Primitives are not valid queries");
   }
 
-  return recursive(_exp, false);
+  return recursive(exp, false);
 }
 
 export function processDeviceProjection(
@@ -417,7 +407,7 @@ export function flattenDevice(device: Record<string, unknown>): FlatDevice {
           };
 
           for (const t of tree as string[]) {
-            output[`Tags.${t}`] = {
+            output[`Tags.${encodeTag(t)}`] = {
               value: [true, "xsd:boolean"],
               valueTimestamp: timestamp,
               writable: true,
@@ -495,7 +485,9 @@ export function flattenTask(task: unknown): Task {
   return t;
 }
 
-export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
+export function convertOldPrecondition(
+  query: Record<string, unknown>
+): Expression {
   function recursive(_query): Expression {
     const expressions: Expression[] = [];
     for (const [k, v] of Object.entries(_query)) {
@@ -514,14 +506,27 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
       } else if (k === "_tags") {
         if (typeof v === "object") {
           if (isArray(v)) throw new Error(`Invalid type`);
-
-          if (v.hasOwnProperty("$ne"))
-            expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]);
-          else if (v.hasOwnProperty("$eq"))
-            expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v["$eq"]}`]]);
-          else throw new Error(`Invalid tag query`);
+          const conjs: Expression[] = [];
+          for (const [op, val] of Object.entries(v)) {
+            if (op === "$ne") {
+              if (typeof v["$ne"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
             } else {
-          expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
+              throw new Error(`Invalid tag query`);
+            }
+          }
+          if (conjs.length === 1) expressions.push(conjs[0]);
+          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
+        } else {
+          expressions.push([
+            "IS NOT NULL",
+            ["PARAM", `Tags.${encodeTag(v as string)}`],
+          ]);
         }
       } else if (k.startsWith("Tags.")) {
         let exists: boolean;
@@ -537,34 +542,31 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
 
         const exps: Expression[] = [];
         for (const [kk, vv] of Object.entries(v)) {
-          let op;
-          switch (kk) {
-            case "$eq":
-              op = "=";
-              break;
-            case "$ne":
-              op = "<>";
-              break;
-            case "$lt":
-              op = "<";
-              break;
-            case "$lte":
-              op = "<=";
-              break;
-            case "$gt":
-              op = ">";
-              break;
-            case "$gte":
-              op = ">=";
-              break;
-            default:
+          if (kk === "$eq") {
+            exps.push(["=", ["PARAM", k], vv]);
+          } else if (kk === "$ne") {
+            exps.push([
+              "OR",
+              ["<>", ["PARAM", k], vv],
+              ["IS NULL", ["PARAM", k]],
+            ]);
+          } else if (kk === "$lt") {
+            exps.push(["<", ["PARAM", k], vv]);
+          } else if (kk === "$lte") {
+            exps.push(["<=", ["PARAM", k], vv]);
+          } else if (kk === "$gt") {
+            exps.push([">", ["PARAM", k], vv]);
+          } else if (kk === "$gte") {
+            exps.push([">=", ["PARAM", k], vv]);
+          } else {
             throw new Error(`Operator ${kk} not supported`);
           }
-          exps.push([op, ["PARAM", k], vv]);
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
         }
         if (exps.length === 1) {
           expressions.push(exps[0]);
-        } else {
+        } else if (exps.length > 1) {
           const and: Expression = ["AND"];
           expressions.push(and.concat(exps));
         }
@@ -573,8 +575,8 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
       }
     }
     if (expressions.length === 1) return expressions[0];
-    const and: Expression = ["AND"];
-    return and.concat(expressions);
+    if (expressions.length === 0) return true;
+    return ["AND", ...expressions];
   }
 
   // empty filter
@@ -592,7 +594,9 @@ export function flattenPreset(
       // Try parse to check expression validity
       parse(p.precondition as string);
     } catch (error) {
-      p.precondition = mongoQueryToFilter(JSON.parse(p.precondition as string));
+      p.precondition = convertOldPrecondition(
+        JSON.parse(p.precondition as string)
+      );
       p.precondition = (p.precondition as string).length
         ? stringify(p.precondition as Expression)
         : "";

lib/nbi.ts

@@ -18,11 +18,11 @@
  */
 
 import * as url from "url";
-import * as mongodb from "mongodb";
+import { Collection, GridFSBucket, ObjectId } from "mongodb";
 import * as querystring from "querystring";
 import * as vm from "vm";
 import * as config from "./config";
-import * as db from "./db";
+import { onConnect } from "./db";
 import * as query from "./query";
 import * as apiFunctions from "./api-functions";
 import { IncomingMessage, ServerResponse } from "http";
@@ -43,6 +43,23 @@ const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
 const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
 
+const collections = {
+  tasks: null as Collection,
+  devices: null as Collection,
+  presets: null as Collection,
+  objects: null as Collection,
+  "fs.files": null as Collection,
+  provisions: null as Collection,
+  virtualParameters: null as Collection,
+  faults: null as Collection,
+};
+let filesBucket: GridFSBucket;
+
+onConnect(async (db) => {
+  for (const k of Object.keys(collections)) collections[k] = db.collection(k);
+  filesBucket = new GridFSBucket(db);
+});
+
 function throwError(err: Error, httpResponse?: ServerResponse): never {
   if (httpResponse) {
     httpResponse.writeHead(500, { Connection: "close" });
@@ -85,7 +102,7 @@ export function listener(
       if (request.method === "PUT") {
         const preset = JSON.parse(body.toString());
         preset._id = presetName;
-        db.presetsCollection.replaceOne(
+        collections.presets.replaceOne(
           { _id: presetName },
           preset,
           { upsert: true },
@@ -106,7 +123,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        db.presetsCollection.deleteOne({ _id: presetName }, (err) => {
+        collections.presets.deleteOne({ _id: presetName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -132,7 +149,7 @@ export function listener(
       if (request.method === "PUT") {
         const object = JSON.parse(body.toString());
         object._id = objectName;
-        db.objectsCollection.replaceOne(
+        collections.objects.replaceOne(
           { _id: objectName },
           object,
           { upsert: true },
@@ -153,7 +170,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        db.objectsCollection.deleteOne({ _id: objectName }, (err) => {
+        collections.objects.deleteOne({ _id: objectName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -190,7 +207,7 @@ export function listener(
           return;
         }
 
-        db.provisionsCollection.replaceOne(
+        collections.provisions.replaceOne(
           { _id: provisionName },
           object,
           { upsert: true },
@@ -211,7 +228,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        db.provisionsCollection.deleteOne({ _id: provisionName }, (err) => {
+        collections.provisions.deleteOne({ _id: provisionName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -248,7 +265,7 @@ export function listener(
           return;
         }
 
-        db.virtualParametersCollection.replaceOne(
+        collections.virtualParameters.replaceOne(
           { _id: virtualParameterName },
           object,
           { upsert: true },
@@ -269,7 +286,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        db.virtualParametersCollection.deleteOne(
+        collections.virtualParameters.deleteOne(
           { _id: virtualParameterName },
           (err) => {
             if (err) return void throwError(err, response);
@@ -296,7 +313,7 @@ export function listener(
       const deviceId = querystring.unescape(r[1]);
       const tag = querystring.unescape(r[2]);
       if (request.method === "POST") {
-        db.devicesCollection.updateOne(
+        collections.devices.updateOne(
           { _id: deviceId },
           { $addToSet: { _tags: tag } },
           (err) => {
@@ -306,7 +323,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        db.devicesCollection.updateOne(
+        collections.devices.updateOne(
           { _id: deviceId },
           { $pull: { _tags: tag } },
           (err) => {
@@ -327,12 +344,12 @@ export function listener(
         );
         const deviceId = faultId.split(":", 1)[0];
         const channel = faultId.slice(deviceId.length + 1);
-        db.faultsCollection.deleteOne({ _id: faultId }, (err) => {
+        collections.faults.deleteOne({ _id: faultId }, (err) => {
           if (err) return void throwError(err, response);
 
           if (channel.startsWith("task_")) {
-            const objId = new mongodb.ObjectID(channel.slice(5));
-            return void db.tasksCollection.deleteOne({ _id: objId }, (err) => {
+            const objId = new ObjectId(channel.slice(5));
+            return void collections.tasks.deleteOne({ _id: objId }, (err) => {
               if (err) return void throwError(err, response);
 
               cache
@@ -375,47 +392,106 @@ export function listener(
           task.device = deviceId;
           apiFunctions
             .insertTasks(task)
-            .then(() => {
-              cache
-                .del(`${deviceId}_tasks_faults_operations`)
-                .then(() => {
-                  if (urlParts.query.connection_request != null) {
-                    apiFunctions
-                      .connectionRequest(deviceId)
-                      .then(() => {
-                        const taskTimeout =
+            .then(async () => {
+              const lastInform = Date.now();
+
+              const socketTimeout: number = request.socket["timeout"];
+
+              // Disable socket timeout while waiting for session
+              if (socketTimeout) request.socket.setTimeout(0);
+
+              const notInSession = await apiFunctions.awaitSessionEnd(
+                deviceId,
+                30000
+              );
+              await cache.del(`${deviceId}_tasks_faults_operations`);
+              if (urlParts.query.connection_request == null) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              if (!notInSession) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const status = await apiFunctions.connectionRequest(deviceId);
+
+              if (status) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, status, {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const onlineThreshold =
                 (urlParts.query.timeout &&
                   parseInt(urlParts.query.timeout as string)) ||
-                          (config.get(
-                            "DEVICE_ONLINE_THRESHOLD",
-                            deviceId
-                          ) as number);
+                (config.get("DEVICE_ONLINE_THRESHOLD", deviceId) as number);
 
-                        apiFunctions
-                          .watchTask(deviceId, task._id, taskTimeout)
-                          .then((status) => {
-                            if (status === "timeout") {
-                              response.writeHead(
-                                202,
-                                "Task queued but not processed",
-                                {
+              const sessionStarted = await apiFunctions.awaitSessionStart(
+                deviceId,
+                lastInform,
+                onlineThreshold
+              );
+              if (!sessionStarted) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
                   "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const sessionEnded = await apiFunctions.awaitSessionEnd(
+                deviceId,
+                120000
+              );
+              if (!sessionEnded) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const prom1 = collections.tasks.findOne(
+                { _id: task._id },
+                { projection: { _id: 1 } }
+              );
+              const prom2 = collections.faults.findOne(
+                { _id: `${deviceId}:task_${task._id}` },
+                {
+                  projection: { _id: 1 },
                 }
               );
-                              response.end(JSON.stringify(task));
-                            } else if (status === "fault") {
-                              db.tasksCollection.findOne(
-                                { _id: task._id },
-                                (err, task2) => {
-                                  if (err)
-                                    return void throwError(err, response);
 
+              const [t, f] = await Promise.all([prom1, prom2]);
+
+              // Restore socket timeout
+              if (socketTimeout) request.socket.setTimeout(socketTimeout);
+
+              if (f) {
                 response.writeHead(202, "Task faulted", {
                   "Content-Type": "application/json",
                 });
-                                  response.end(JSON.stringify(task2));
-                                }
-                              );
+                response.end(JSON.stringify(t || task));
+              } else if (t) {
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(t));
               } else {
                 response.writeHead(200, {
                   "Content-Type": "application/json",
@@ -428,31 +504,6 @@ export function listener(
                 throwError(err, response);
               });
             });
-                      })
-                      .catch((err) => {
-                        response.writeHead(202, err.message, {
-                          "Content-Type": "application/json",
-                        });
-                        response.end(JSON.stringify(task));
-                      });
-                  } else {
-                    response.writeHead(202, {
-                      "Content-Type": "application/json",
-                    });
-                    response.end(JSON.stringify(task));
-                  }
-                })
-                .catch((err) => {
-                  setTimeout(() => {
-                    throwError(err, response);
-                  });
-                });
-            })
-            .catch((err) => {
-              setTimeout(() => {
-                throwError(err, response);
-              });
-            });
         } else if (urlParts.query.connection_request != null) {
           // No task, send connection request only
           apiFunctions
@@ -479,8 +530,8 @@ export function listener(
       const action = r[2];
       if (!action || action === "/") {
         if (request.method === "DELETE") {
-          db.tasksCollection.findOne(
-            { _id: new mongodb.ObjectID(taskId) },
+          collections.tasks.findOne(
+            { _id: new ObjectId(taskId) },
             { projection: { device: 1 } },
             (err, task) => {
               if (err) return void throwError(err, response);
@@ -492,12 +543,12 @@ export function listener(
               }
 
               const deviceId = task.device;
-              db.tasksCollection.deleteOne(
-                { _id: new mongodb.ObjectID(taskId) },
+              collections.tasks.deleteOne(
+                { _id: new ObjectId(taskId) },
                 (err) => {
                   if (err) return void throwError(err, response);
 
-                  db.faultsCollection.deleteOne(
+                  collections.faults.deleteOne(
                     { _id: `${deviceId}:task_${taskId}` },
                     (err) => {
                       if (err) return void throwError(err, response);
@@ -525,14 +576,14 @@ export function listener(
         }
       } else if (action === "/retry") {
         if (request.method === "POST") {
-          db.tasksCollection.findOne(
-            { _id: new mongodb.ObjectID(taskId) },
+          collections.tasks.findOne(
+            { _id: new ObjectId(taskId) },
             { projection: { device: 1 } },
             (err, task) => {
               if (err) return void throwError(err, response);
 
               const deviceId = task.device;
-              db.faultsCollection.deleteOne(
+              collections.faults.deleteOne(
                 { _id: `${deviceId}:task_${taskId}` },
                 (err) => {
                   if (err) return void throwError(err, response);
@@ -571,9 +622,8 @@ export function listener(
           productClass: request.headers.productclass,
           version: request.headers.version,
         };
-        const bucket = new mongodb.GridFSBucket(db.client.db());
-        bucket.delete((filename as unknown) as mongodb.ObjectId, () => {
-          const uploadStream = bucket.openUploadStreamWithId(
+        filesBucket.delete((filename as unknown) as ObjectId, () => {
+          const uploadStream = filesBucket.openUploadStreamWithId(
             filename,
             filename,
             {
@@ -591,8 +641,7 @@ export function listener(
           });
         });
       } else if (request.method === "DELETE") {
-        const bucket = new mongodb.GridFSBucket(db.client.db());
-        bucket.delete((filename as unknown) as mongodb.ObjectId, (err) => {
+        filesBucket.delete((filename as unknown) as ObjectId, (err) => {
           if (err) {
             if (err.message.startsWith("FileNotFound")) {
               response.writeHead(404);
@@ -669,7 +718,7 @@ export function listener(
         return;
       }
 
-      const collection = db[`${collectionName}Collection`];
+      const collection = collections[collectionName];
       if (!collection) {
         response.writeHead(404);
         response.end("404 Not Found");
@@ -693,7 +742,7 @@ export function listener(
           break;
         case "tasks":
           q = query.sanitizeQueryTypes(q, {
-            _id: (v) => new mongodb.ObjectID(v as string),
+            _id: (v) => new ObjectId(v as string),
             timestamp: (v) => new Date(v as number),
             retries: Number,
           });

lib/server.ts

@@ -52,35 +52,43 @@ function closeServer(timeout, callback): void {
   });
 }
 
+interface ServerOptions {
+  port?: number;
+  host?: string;
+  ssl?: { key: string; cert: string };
+  timeout?: number;
+  keepAliveTimeout?: number;
+  onConnection?: (socket: Socket) => void;
+}
+
 export function start(
-  port: number,
-  networkInterface: string,
-  ssl: { key: string; cert: string },
-  _listener: http.RequestListener,
-  onConnection?: (socket: Socket) => void,
-  keepAliveTimeout = -1
+  options: ServerOptions,
+  _listener: http.RequestListener
 ): void {
   listener = _listener;
 
-  if (ssl && ssl.key && ssl.cert) {
-    const options = {
-      key: ssl.key
+  if (options.ssl) {
+    const opts = {
+      key: options.ssl.key
         .split(":")
         .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
-      cert: ssl.cert
+      cert: options.ssl.cert
         .split(":")
         .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
     };
 
-    server = https.createServer(options, listener);
-    if (onConnection != null) server.on("secureConnection", onConnection);
+    server = https.createServer(opts, listener);
+    if (options.onConnection)
+      server.on("secureConnection", options.onConnection);
   } else {
     server = http.createServer(listener);
-    if (onConnection != null) server.on("connection", onConnection);
+    if (options.onConnection) server.on("connection", options.onConnection);
   }
 
-  if (keepAliveTimeout >= 0) server.keepAliveTimeout = keepAliveTimeout;
-  server.listen(port, networkInterface);
+  server.timeout = options.timeout || 0;
+  if (options.keepAliveTimeout != null)
+    server.keepAliveTimeout = options.keepAliveTimeout;
+  server.listen({ port: options.port, host: options.host });
 }
 
 export function stop(): Promise<void> {

lib/session.ts

@@ -299,7 +299,7 @@ export async function transferComplete(
   sessionContext.operationsTouched[commandKey] = 1;
 
   if (rpcReq.faultStruct && rpcReq.faultStruct.faultCode !== "0") {
-    await revertDownloadParameters(sessionContext, operation.args.instance);
+    revertDownloadParameters(sessionContext, operation.args.instance);
 
     const fault: Fault = {
       code: `cwmp.${rpcReq.faultStruct.faultCode}`,
@@ -398,10 +398,10 @@ export async function transferComplete(
   };
 }
 
-async function revertDownloadParameters(
+function revertDownloadParameters(
   sessionContext: SessionContext,
   instance
-): Promise<void> {
+): void {
   const timestamp = sessionContext.timestamp + sessionContext.iteration + 1;
 
   let p;
@@ -1734,20 +1734,35 @@ function generateSetRpcRequest(
 
   const deviceData = sessionContext.deviceData;
 
+  const SKIP_WRITABLE_CHECK = !!localCache.getConfig(
+    sessionContext.cacheSnapshot,
+    "cwmp.skipWritableCheck",
+    {},
+    sessionContext.timestamp,
+    (e) => configContextCallback(sessionContext, e)
+  );
+
+  const canWrite = (attrs: Attributes): boolean =>
+    SKIP_WRITABLE_CHECK || (attrs.writable && !!attrs.writable[1]);
+
   // Delete instance
   for (const instances of syncState.instancesToDelete.values()) {
-    const instance = instances.values().next().value;
-    if (instance && sessionContext.deviceData.attributes.has(instance)) {
+    for (const instance of instances) {
+      const attrs = sessionContext.deviceData.attributes.get(instance);
+      if (attrs && canWrite(attrs)) {
+        instances.delete(instance);
         return {
           name: "DeleteObject",
           objectName: instance.toString() + ".",
         };
       }
     }
+  }
 
   // Create instance
   for (const [param, instances] of syncState.instancesToCreate) {
-    if (sessionContext.deviceData.attributes.has(param)) {
+    const attrs = sessionContext.deviceData.attributes.get(param);
+    if (attrs && canWrite(attrs)) {
       const instance = instances.values().next().value;
       if (instance) {
         instances.delete(instance);
@@ -1791,7 +1806,7 @@ function generateSetRpcRequest(
     syncState.spv.delete(k);
     const attrs = sessionContext.deviceData.attributes.get(k);
     const curVal = attrs.value ? attrs.value[1] : null;
-    if (curVal && attrs.writable && attrs.writable[1]) {
+    if (curVal && canWrite(attrs)) {
       const val = v.slice() as [string | number | boolean, string];
       if (!val[1]) val[1] = curVal[1];
       device.sanitizeParameterValue(val);
@@ -2609,7 +2624,6 @@ export async function rpcResponse(
         timestamp + 1,
         {
           object: [timestamp + 1, 0],
-          writable: [timestamp + 1, 1],
           value: [
             timestamp + 1,
             p.slice(1) as [string | number | boolean, string],

lib/soap.ts

@@ -127,7 +127,8 @@ function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
 
       if (parsed == null) {
         warnings.push({
-          message: "Invalid writable attribute",
+          message: "Missing or invalid XML node",
+          element: "Writable",
           parameter: param,
         });
         parsed = false;
@@ -167,7 +168,8 @@ function parameterValueList(
       let valueType = getValueType(valueElement.attrs);
       if (!valueType) {
         warnings.push({
-          message: "Invalid value type attribute",
+          message: "Missing or invalid XML node",
+          attribute: "type",
           parameter: param,
         });
         valueType = "xsd:string";
@@ -179,7 +181,8 @@ function parameterValueList(
         parsed = parseBool(value);
         if (parsed == null) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
+            element: "Value",
             parameter: param,
           });
           parsed = value;
@@ -188,7 +191,8 @@ function parameterValueList(
         parsed = parseInt(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
+            element: "Value",
             parameter: param,
           });
           parsed = value;
@@ -197,7 +201,8 @@ function parameterValueList(
         parsed = Date.parse(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
+            element: "Value",
             parameter: param,
           });
           parsed = value;
@@ -232,7 +237,8 @@ function parameterAttributeList(xml: Element): [Path, number, string[]][] {
       let notification = parseInt(notificationElement.text);
       if (isNaN(notification)) {
         warnings.push({
-          message: "Invalid notification attribute",
+          message: "Missing or invalid XML node",
+          element: "Notification",
           parameter: param,
         });
         notification = 0;
@@ -331,7 +337,10 @@ function SetParameterValuesResponse(xml: Element): SetParameterValuesResponse {
   }
 
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid SetParameterValuesResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
@@ -392,8 +401,14 @@ function AddObjectResponse(xml: Element): AddObjectResponse {
     }
   }
 
+  if (!/^[0-9]+$/.test(instanceNumber))
+    throw new Error("Missing or invalid instance number");
+
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid AddObjectResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
@@ -424,7 +439,10 @@ function DeleteObjectResponse(xml: Element): DeleteObjectResponse {
   }
 
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid DeleteObjectResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
@@ -495,10 +513,29 @@ function DownloadResponse(xml: Element): DownloadResponse {
   }
 
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid DownloadResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "DownloadResponse",
     status: status,
@@ -537,6 +574,30 @@ function Inform(xml: Element): InformRequest {
     }
   }
 
+  if (!deviceId || !deviceId.SerialNumber || !deviceId.OUI)
+    throw new Error("Missing or invalid DeviceId element");
+
+  if (!parameterList) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "ParameterList",
+    });
+    parameterList = [];
+  }
+
+  if (!evnt) {
+    warnings.push({ message: "Missing or invalid XML node", element: "Event" });
+    evnt = [];
+  }
+
+  if (retryCount == null || isNaN(retryCount)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "RetryCount",
+    });
+    retryCount = 0;
+  }
+
   return {
     name: "Inform",
     parameterList: parameterList,
@@ -563,7 +624,10 @@ function GetRPCMethodsResponse(methodResponse): string {
 }
 
 function TransferComplete(xml: Element): TransferCompleteRequest {
-  let commandKey, _faultStruct, startTime, completeTime;
+  let commandKey: string,
+    _faultStruct: FaultStruct,
+    startTime: number,
+    completeTime: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "CommandKey":
@@ -581,6 +645,38 @@ function TransferComplete(xml: Element): TransferCompleteRequest {
     }
   }
 
+  if (commandKey == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CommandKey",
+    });
+    commandKey = "";
+  }
+
+  if (!_faultStruct) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultStruct",
+    });
+    _faultStruct = { faultCode: "0", faultString: "" };
+  }
+
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "TransferComplete",
     commandKey: commandKey,
@@ -606,7 +702,12 @@ function RequestDownloadResponse(): string {
 }
 
 function faultStruct(xml: Element): FaultStruct {
-  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
+  let faultCode: string,
+    faultString: string,
+    setParameterValuesFault: SpvFault[],
+    pn: string,
+    fc: string,
+    fs: string;
   for (const c of xml.children) {
     switch (c.localName) {
       case "FaultCode":
@@ -617,6 +718,7 @@ function faultStruct(xml: Element): FaultStruct {
         break;
       case "SetParameterValuesFault":
         setParameterValuesFault = setParameterValuesFault || [];
+        pn = fc = fs = null;
         for (const cc of c.children) {
           switch (cc.localName) {
             case "ParameterName":
@@ -638,11 +740,27 @@ function faultStruct(xml: Element): FaultStruct {
     }
   }
 
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultCode",
+    });
+    faultCode = "";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultString",
+    });
+    faultString = "";
+  }
+
   return { faultCode, faultString, setParameterValuesFault };
 }
 
 function fault(xml: Element): CpeFault {
-  let faultCode, faultString, detail;
+  let faultCode: string, faultString: string, detail: FaultStruct;
   for (const c of xml.children) {
     switch (c.localName) {
       case "faultcode":
@@ -657,7 +775,25 @@ function fault(xml: Element): CpeFault {
     }
   }
 
-  return { faultCode, faultString, detail };
+  if (!detail) throw new Error("Missing detail element");
+
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultcode",
+    });
+    faultCode = "Client";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultstring",
+    });
+    faultString = "CWMP fault";
+  }
+
+  return { faultCode, faultString, detail } as CpeFault;
 }
 
 export function request(

lib/types.ts

@@ -163,6 +163,7 @@ export interface SessionContext {
   cacheUntil?: number;
   syncState?: SyncState;
   lastActivity?: number;
+  extendLock?: number;
   rpcRequest?: AcsRequest;
   operationsTouched?: { [commandKey: string]: 1 | 0 };
   provisionsRet?: any[];
@@ -289,8 +290,8 @@ export interface FaultStruct {
 }
 
 export interface CpeFault {
-  faultCode: string;
-  faultString: string;
+  faultCode: "Client" | "Server";
+  faultString: "CWMP fault";
   detail?: FaultStruct;
 }
 

lib/ui.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Z_SYNC_FLUSH } from "zlib";
+import { constants } from "zlib";
 import Koa from "koa";
 import Router from "koa-router";
 import * as jwt from "jsonwebtoken";
@@ -60,7 +60,9 @@ const getAuthorizer = memoize(
 );
 
 koa.on("error", async (err) => {
+  setTimeout(() => {
     throw err;
+  });
 });
 
 koa.use(async (ctx, next) => {
@@ -252,10 +254,10 @@ router.get("/", async (ctx) => {
 koa.use(
   koaCompress({
     gzip: {
-      flush: Z_SYNC_FLUSH,
+      flush: constants.Z_SYNC_FLUSH,
     },
     deflate: {
-      flush: Z_SYNC_FLUSH,
+      flush: constants.Z_SYNC_FLUSH,
     },
   })
 );

lib/ui/api-functions.ts

@@ -24,7 +24,8 @@ import { getUsers } from "../local-cache";
 import { hashPassword } from "../auth";
 import {
   insertTasks,
-  watchTask,
+  awaitSessionEnd,
+  awaitSessionStart,
   connectionRequest,
   deleteDevice,
 } from "../api-functions";
@@ -104,23 +105,45 @@ export async function postTasks(
   const statuses = tasks.map((t) => {
     return { _id: t._id, status: "pending" };
   });
-
-  await del(`${deviceId}_tasks_faults_operations`);
-
-  try {
-    await connectionRequest(deviceId, device);
-  } catch (err) {
-    for (const t of statuses) db.deleteTask(new ObjectID(t._id));
+  const lastInform = Date.now();
+  const notInSession = await awaitSessionEnd(deviceId, 30000);
+  if (!notInSession) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
     return {
-      connectionRequest: err.message,
+      connectionRequest: "Session took too long to complete",
       tasks: statuses,
     };
   }
 
-  const sample = tasks[tasks.length - 1];
+  await del(`${deviceId}_tasks_faults_operations`);
 
-  // Waiting for session to finish or timeout
-  await watchTask(deviceId, sample._id, timeout);
+  const status = await connectionRequest(deviceId, device);
+
+  if (status) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: status,
+      tasks: statuses,
+    };
+  }
+
+  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
+  if (!sessionStarted) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: "No contact from CPE",
+      tasks: statuses,
+    };
+  }
+
+  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
+  if (!sessionEnded) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: "Session took too long to complete",
+      tasks: statuses,
+    };
+  }
 
   const promises = [];
   for (const s of statuses) {
@@ -138,20 +161,11 @@ export async function postTasks(
       r.status = "fault";
       r["fault"] = res[i * 2 + 1][0];
     }
-    db.deleteTask(new ObjectID(r._id));
   }
 
-  return { connectionRequest: "OK", tasks: statuses };
-}
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
 
-interface PingResponse {
-  packetsTransmitted: number;
-  packetsReceived: number;
-  packetLoss: string;
-  min: number;
-  avg: number;
-  max: number;
-  mdev: number;
+  return { connectionRequest: "OK", tasks: statuses };
 }
 
 export async function putResource(

lib/ui/api.ts

@@ -31,6 +31,7 @@ import { del } from "../cache";
 import Authorizer from "../common/authorizer";
 import { ping } from "../ping";
 import * as url from "url";
+import { decodeTag } from "../common";
 
 const router = new Router();
 export default router;
@@ -85,7 +86,7 @@ router.get(`/devices/:id.csv`, async (ctx) => {
 
   if (!authorizer.hasAccess("devices", 2)) {
     logUnauthorizedWarning(log);
-    return void (ctx.status = 404);
+    return void (ctx.status = 403);
   }
 
   const res = await db.query("devices", filter);
@@ -126,7 +127,7 @@ router.get(`/devices/:id.csv`, async (ctx) => {
 });
 
 for (const [resource, flags] of Object.entries(resources)) {
-  router.head(`/${resource}`, async (ctx, next) => {
+  router.head(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     let filter: Expression = authorizer.getFilter(resource, 1);
     if (ctx.request.query.filter)
@@ -141,14 +142,15 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     if (!authorizer.hasAccess(resource, 1)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -161,7 +163,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.get(`/${resource}`, async (ctx, next) => {
+  router.get(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = {};
     let filter: Expression = authorizer.getFilter(resource, 2);
@@ -189,14 +191,15 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -214,7 +217,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   });
 
   // CSV download
-  router.get(`/${resource}.csv`, async (ctx, next) => {
+  router.get(`/${resource}.csv`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = { projection: {} };
     let filter: Expression = authorizer.getFilter(resource, 2);
@@ -236,7 +239,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const columns: Record<string, Expression> = JSON.parse(
@@ -254,8 +257,9 @@ for (const [resource, flags] of Object.entries(resources)) {
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -277,7 +281,7 @@ for (const [resource, flags] of Object.entries(resources)) {
                 if (e[1] === "Tags") {
                   const tags = [];
                   for (const p in obj)
-                    if (p.startsWith("Tags.")) tags.push(p.slice(5));
+                    if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
 
                   return tags.join(", ");
                 }
@@ -304,7 +308,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.head(`/${resource}/:id`, async (ctx, next) => {
+  router.head(`/${resource}/:id`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Count ${resource}`,
@@ -319,18 +323,18 @@ for (const [resource, flags] of Object.entries(resources)) {
     ]);
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void next();
+    if (!res.length) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = "";
   });
 
-  router.get(`/${resource}/:id`, async (ctx, next) => {
+  router.get(`/${resource}/:id`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Query ${resource}`,
@@ -345,19 +349,19 @@ for (const [resource, flags] of Object.entries(resources)) {
     ]);
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void next();
+    if (!res.length) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = res[0];
   });
 
   if (flags & RESOURCE_DELETE) {
-    router.delete(`/${resource}/:id`, async (ctx, next) => {
+    router.delete(`/${resource}/:id`, async (ctx) => {
       const authorizer: Authorizer = ctx.state.authorizer;
       const log = {
         message: `Delete ${resource}`,
@@ -372,10 +376,10 @@ for (const [resource, flags] of Object.entries(resources)) {
       ]);
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
       const res = await db.query(resource, filter);
-      if (!res.length) return void next();
+      if (!res.length) return void (ctx.status = 404);
 
       const validate = authorizer.getValidator(resource, res[0]);
       if (!validate("delete")) {
@@ -392,7 +396,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 
   if (flags & RESOURCE_PUT) {
-    router.put(`/${resource}/:id`, async (ctx, next) => {
+    router.put(`/${resource}/:id`, async (ctx) => {
       const authorizer: Authorizer = ctx.state.authorizer;
       const id = ctx.params.id;
 
@@ -404,7 +408,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
 
       const obj = ctx.request.body;
@@ -431,7 +435,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 }
 
-router.put("/files/:id", async (ctx, next) => {
+router.put("/files/:id", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const resource = "files";
   const id = ctx.params.id;
@@ -445,7 +449,7 @@ router.put("/files/:id", async (ctx, next) => {
 
   if (!authorizer.hasAccess(resource, 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
   const metadata = {
@@ -474,7 +478,7 @@ router.put("/files/:id", async (ctx, next) => {
   ctx.body = "";
 });
 
-router.post("/devices/:id/tasks", async (ctx, next) => {
+router.post("/devices/:id/tasks", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Commit tasks",
@@ -490,10 +494,10 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
   ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
   const devices = await db.query("devices", filter);
-  if (!devices.length) return void next();
+  if (!devices.length) return void (ctx.status = 404);
   const device = devices[0];
 
   const validate = authorizer.getValidator("devices", device);
@@ -527,6 +531,11 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
     }
   ) as number;
 
+  const socketTimeout: number = ctx.socket["timeout"];
+
+  // Disable socket timeout while waiting for postTasks()
+  if (socketTimeout) ctx.socket.setTimeout(0);
+
   const res = await apiFunctions.postTasks(
     ctx.params.id,
     ctx.request.body,
@@ -534,6 +543,9 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
     device
   );
 
+  // Restore socket timeout
+  if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
+
   log.tasks = res.tasks.map((t) => t._id).join(",");
 
   logger.accessInfo(log);
@@ -542,7 +554,7 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
   ctx.body = res.tasks;
 });
 
-router.post("/devices/:id/tags", async (ctx, next) => {
+router.post("/devices/:id/tags", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Update tags",
@@ -558,10 +570,10 @@ router.post("/devices/:id/tags", async (ctx, next) => {
   ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
   const res = await db.query("devices", filter);
-  if (!res.length) return void next();
+  if (!res.length) return void (ctx.status = 404);
 
   const validate = authorizer.getValidator("devices", res[0]);
   if (!validate("tags", ctx.request.body)) {
@@ -597,7 +609,7 @@ router.get("/ping/:host", async (ctx) => {
   });
 });
 
-router.put("/users/:id/password", async (ctx, next) => {
+router.put("/users/:id/password", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const username = ctx.params.id;
   const log = {
@@ -622,7 +634,7 @@ router.put("/users/:id/password", async (ctx, next) => {
     }
   } else if (!authorizer.hasAccess("users", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
   const filter = and(authorizer.getFilter("users", 3), [
@@ -631,7 +643,7 @@ router.put("/users/:id/password", async (ctx, next) => {
     username,
   ]);
   const res = await db.query("users", filter);
-  if (!res.length) return void next();
+  if (!res.length) return void (ctx.status = 404);
 
   const newPassword = ctx.request.body.newPassword;
   if (ctx.state.user) {

lib/ui/db.ts

@@ -17,85 +17,24 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, GridFSBucket, ObjectID } from "mongodb";
+import { Db, GridFSBucket, ObjectID } from "mongodb";
 import { Script } from "vm";
-import * as config from "../config";
+import { onConnect } from "../db";
 import * as mongodbFunctions from "../mongodb-functions";
 import * as expression from "../common/expression";
 import { QueryOptions, Expression } from "../types";
 import { Readable } from "stream";
 import { minimize } from "../common/boolean-expression";
 
-const CACHE_TTL = 300000;
-
-let clientPromise: Promise<MongoClient>;
-
 const RESOURCE_COLLECTION = {
   files: "fs.files",
 };
 
-function ensureIndexes(client): void {
-  client
-    .db()
-    .collection("cache")
-    .createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
-}
+let db: Db;
 
-function getClient(): Promise<MongoClient> {
-  if (!clientPromise) {
-    clientPromise = new Promise((resolve, reject) => {
-      const CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
-      MongoClient.connect(
-        CONNECTION_URL,
-        { useNewUrlParser: true, useUnifiedTopology: true },
-        (err, client) => {
-          if (err) return void reject(err);
-          ensureIndexes(client);
-          resolve(client);
-        }
-      );
-    });
-  }
-
-  return clientPromise;
-}
-
-export function cache<T>(
-  key: string,
-  valueGetter: () => Promise<T>,
-  ttl: number
-): Promise<T> {
-  return new Promise((resolve, reject) => {
-    getClient()
-      .then((client) => {
-        const collection = client.db().collection("cache");
-        collection.findOne({ _id: key }, (err, doc) => {
-          if (err) return void reject(err);
-          if (doc != null) return void resolve(JSON.parse(doc.value) as T);
-          valueGetter()
-            .then((res) => {
-              const expire = Date.now() + (ttl || CACHE_TTL);
-              const cacheDoc = {
-                _id: key,
-                value: JSON.stringify(res),
-                expire: new Date(expire),
-              };
-              collection.updateOne(
-                { _id: key },
-                { $set: cacheDoc },
-                { upsert: true },
-                (err) => {
-                  if (err) reject(err);
-                  else resolve(res);
-                }
-              );
-            })
-            .catch(reject);
-        });
-      })
-      .catch(reject);
-  });
-}
+onConnect(async (_db) => {
+  db = _db;
+});
 
 export function query(
   resource: string,
@@ -123,15 +62,12 @@ export function query(
     }
 
     q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (filter != null && !filter) {
+  } else if (!filter) {
     return Promise.resolve([]);
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then((client) => {
-      const collection = client
-        .db()
-        .collection(RESOURCE_COLLECTION[resource] || resource);
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
     const cursor = collection.find(q);
     if (options.projection) {
       let projection = options.projection;
@@ -181,8 +117,7 @@ export function query(
     } else {
       cursor.forEach(
         (doc) => {
-            if (resource === "devices")
-              doc = mongodbFunctions.flattenDevice(doc);
+          if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc);
           else if (resource === "faults")
             doc = mongodbFunctions.flattenFault(doc);
           else if (resource === "tasks")
@@ -200,7 +135,6 @@ export function query(
       );
     }
   });
-  });
 }
 
 export function count(resource: string, filter: Expression): Promise<number> {
@@ -216,71 +150,48 @@ export function count(resource: string, filter: Expression): Promise<number> {
     else if (resource === "faults")
       filter = mongodbFunctions.processFaultsFilter(filter);
     q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (filter != null && !filter) {
+  } else if (!filter) {
     return Promise.resolve(0);
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then((client) => {
-      const collection = client
-        .db()
-        .collection(RESOURCE_COLLECTION[resource] || resource);
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
     collection.find(q).count((err, c) => {
       if (err) reject(err);
       else resolve(c);
     });
   });
-  });
 }
 
-export function updateDeviceTags(
+export async function updateDeviceTags(
   deviceId: string,
   tags: Record<string, boolean>
 ): Promise<void> {
-  return new Promise((resolve, reject) => {
   const add = [];
   const pull = [];
 
-    const regex = /^[0-9a-zA-Z_]+$/;
   for (let [tag, onOff] of Object.entries(tags)) {
     tag = tag.trim();
-      if (onOff) {
-        if (!tag.match(regex))
-          return void reject(new Error(`Invalid tag '${tag}'`));
-        add.push(tag);
-      } else {
-        pull.push(tag);
+    if (onOff) add.push(tag);
+    else pull.push(tag);
   }
-    }
-    getClient()
-      .then((client) => {
-        const collection = client.db().collection("devices");
 
+  const collection = db.collection("devices");
   const object = {};
+
   if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
   if (pull && pull.length) object["$pullAll"] = { _tags: pull };
-        collection.updateOne({ _id: deviceId }, object, (err) => {
-          if (err) return void reject(err);
-          resolve();
-        });
-      })
-      .catch(reject);
-  });
+
+  await collection.updateOne({ _id: deviceId }, object);
 }
 
 function putResource(resource, id, object): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
-      .then((client) => {
-        const collection = client
-          .db()
-          .collection(RESOURCE_COLLECTION[resource] || resource);
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
     collection.replaceOne({ _id: id }, object, { upsert: true }, (err) => {
       if (err) return void reject(err);
       resolve();
     });
-      })
-      .catch(reject);
   });
 }
 
@@ -289,17 +200,11 @@ function deleteResource(
   id: string | ObjectID
 ): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
-      .then((client) => {
-        const collection = client
-          .db()
-          .collection(RESOURCE_COLLECTION[resource] || resource);
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
     collection.deleteOne({ _id: id }, (err) => {
       if (err) return void reject(err);
       resolve();
     });
-      })
-      .catch(reject);
   });
 }
 
@@ -376,9 +281,7 @@ export function putUser(
   object: Record<string, unknown>
 ): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
-      .then((client) => {
-        const collection = client.db().collection("users");
+    const collection = db.collection("users");
     // update instead of replace to keep the password if not set by user
     collection.updateOne(
       { _id: id },
@@ -389,8 +292,6 @@ export function putUser(
         resolve();
       }
     );
-      })
-      .catch(reject);
   });
 }
 
@@ -404,9 +305,7 @@ export function putFile(
   contentStream: Readable
 ): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
-      .then((client) => {
-        const bucket = new GridFSBucket(client.db());
+    const bucket = new GridFSBucket(db);
     const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
       metadata: metadata,
     });
@@ -414,22 +313,16 @@ export function putFile(
     contentStream.on("error", reject);
     uploadStream.on("finish", resolve);
     contentStream.pipe(uploadStream);
-      })
-      .catch(reject);
   });
 }
 
 export function deleteFile(filename: string): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
-      .then((client) => {
-        const bucket = new GridFSBucket(client.db());
+    const bucket = new GridFSBucket(db);
     bucket.delete(filename as any, (err) => {
       if (err) return void reject(err);
       resolve();
     });
-      })
-      .catch(reject);
   });
 }
 
@@ -440,7 +333,3 @@ export function deleteFault(id: string): Promise<void> {
 export function deleteTask(id: ObjectID): Promise<void> {
   return deleteResource("tasks", id);
 }
-
-export async function disconnect(): Promise<void> {
-  if (clientPromise) await (await clientPromise).close();
-}

lib/xml-parser.ts

@@ -53,12 +53,13 @@ export interface Element {
 }
 
 export function parseXmlDeclaration(buffer: Buffer): Attribute[] {
-  for (const enc of ["utf16le", "utf8", "latin1", "ascii"]) {
+  const encodings: BufferEncoding[] = ["utf16le", "utf8", "latin1", "ascii"];
+  for (const enc of encodings) {
     let str = buffer.toString(enc, 0, 150);
     if (str.startsWith("<?xml")) {
-      str = str.split("\n")[0].trim();
+      str = str.slice(0, str.indexOf("?>"));
       try {
-        return parseAttrs(str.slice(5, -2));
+        return parseAttrs(str.slice(5));
       } catch (err) {
         // Ignore
       }
@@ -103,10 +104,12 @@ export function parseAttrs(string: string): Attribute[] {
         continue;
 
       case CHAR_COLON:
+        if (state) continue;
         if (idx >= colonIdx) colonIdx = i;
         continue;
 
       case CHAR_EQUAL:
+        if (state) continue;
         if (name) throw new Error(`Unexpected character at ${i}`);
         name = string.slice(idx, i).trim();
         // TODO validate name

lib/xmpp-client.ts

@@ -0,0 +1,481 @@
+/**
+ * Copyright 2013-2020  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as net from "net";
+import * as tls from "tls";
+import { EventEmitter } from "events";
+import { createHash, createHmac, randomBytes } from "crypto";
+import { parseXml, Element, parseAttrs } from "./xml-parser";
+
+function encodeBase64(str: string): string {
+  return Buffer.from(str).toString("base64");
+}
+
+function decodeBase64(str: string): string {
+  return Buffer.from(str, "base64").toString();
+}
+
+function detectStreamTag(data: string): number {
+  const i1 = data.indexOf("<stream:stream");
+  if (i1 < 0) throw new Error("Cannot detect opening stream tag");
+  const i2 = data.indexOf(">", i1);
+  if (i2 < 0) throw new Error("Cannot detect opening stream tag");
+  return i2 + 1;
+}
+
+function xmppStream<T>(
+  socket: net.Socket,
+  callback: Generator<void, T, Element>
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const onError = (err: Error): void => {
+      socket.removeListener("error", onError);
+      reject(err);
+    };
+    socket.on("error", onError);
+
+    const onData = (chunk: Buffer): void => {
+      try {
+        const str = chunk.toString("utf8");
+        const xml = parseXml(str);
+        const { value, done } = callback.next(xml.children[0]);
+        if (done) {
+          socket.removeListener("error", onError);
+          socket.removeListener("data", onData);
+          resolve(value as T);
+        }
+      } catch (err) {
+        socket.removeListener("error", onError);
+        socket.removeListener("data", onData);
+        reject(err);
+      }
+    };
+
+    socket.once("data", (chunk: Buffer) => {
+      try {
+        const str = chunk.toString("utf8");
+        const i = detectStreamTag(str);
+        const streamTagStr = str.slice(0, i);
+        const xml = parseXml(streamTagStr + "</stream:stream>");
+        callback.next(xml.children[0]);
+        chunk = chunk.slice(Buffer.byteLength(streamTagStr));
+        if (chunk.length) onData(chunk);
+        socket.on("data", onData);
+      } catch (err) {
+        socket.removeListener("error", onError);
+        socket.removeListener("data", onData);
+        reject(err);
+      }
+    });
+
+    try {
+      callback.next();
+    } catch (err) {
+      socket.removeListener("error", onError);
+      socket.removeListener("data", onData);
+      reject(err);
+    }
+  });
+}
+
+const INT_1 = Buffer.from([0, 0, 0, 1]);
+const saltedPasswordCache = {
+  password: "",
+  iterationCount: 0,
+  saltBase64: "",
+  salted: Buffer.allocUnsafe(0),
+};
+
+function saltPassword(
+  password: string,
+  saltBase64: string,
+  iteractionCount: number
+): Buffer {
+  if (
+    password === saltedPasswordCache.password &&
+    saltBase64 === saltedPasswordCache.saltBase64 &&
+    iteractionCount === saltedPasswordCache.iterationCount
+  )
+    return saltedPasswordCache.salted;
+
+  const hi = createHmac("sha1", password)
+    .update(Buffer.concat([Buffer.from(saltBase64, "base64"), INT_1]))
+    .digest();
+  let hi2: Buffer = hi;
+  for (let i = 1; i < iteractionCount; ++i) {
+    hi2 = createHmac("sha1", password).update(hi2).digest();
+    for (const [j, b] of hi2.entries()) hi[j] ^= b;
+  }
+
+  saltedPasswordCache.saltBase64 = saltBase64;
+  saltedPasswordCache.password = password;
+  saltedPasswordCache.iterationCount = iteractionCount;
+  saltedPasswordCache.salted = hi;
+
+  return hi;
+}
+
+function* loginPlain(
+  socket: net.Socket,
+  username: string,
+  password: string
+): Generator<void, void, Element> {
+  socket.write(
+    `<auth xmlns="urn:ietf:params:xml:ns:xmpp-sasl" mechanism="PLAIN">${encodeBase64(
+      `\x00${username}\x00${password}`
+    )}</auth>`
+  );
+  const res1 = yield;
+
+  if (
+    res1.name === "failure" &&
+    res1.children.some((c) => c.name === "not-authorized")
+  )
+    throw new Error("Not authorized");
+
+  if (res1.name !== "success")
+    throw new Error(`Unexpected response ${res1.name}`);
+}
+
+function* loginScram(
+  socket: net.Socket,
+  username: string,
+  password: string
+): Generator<void, void, Element> {
+  const cnonce = randomBytes(8).toString("base64");
+  const gs2Header = "n,,";
+  const clientFirstMessageBare = `n=${username},r=${cnonce}`;
+  const clientFirstMessage = gs2Header + clientFirstMessageBare;
+  socket.write(
+    `<auth xmlns="urn:ietf:params:xml:ns:xmpp-sasl" mechanism="SCRAM-SHA-1">${encodeBase64(
+      clientFirstMessage
+    )}</auth>`
+  );
+  const res1 = yield;
+  if (res1.name !== "challenge")
+    throw new Error(`Unexpected element ${res1.name}`);
+  const serverFirstMessage = decodeBase64(res1.text);
+
+  let iterationCount: number;
+  let saltBase64: string;
+  let nonce: string;
+  for (const s of serverFirstMessage.split(",")) {
+    if (s.startsWith("i=")) iterationCount = parseInt(s.slice(2));
+    else if (s.startsWith("s=")) saltBase64 = s.slice(2);
+    else if (s.startsWith("r=")) nonce = s.slice(2);
+  }
+  if (iterationCount == null || isNaN(iterationCount))
+    throw new Error("Invalid iteration count");
+  if (saltBase64 == null) throw new Error("Missing salt");
+  if (nonce == null) throw new Error("Missing nonce");
+
+  const saltedPassword = saltPassword(password, saltBase64, iterationCount);
+  const clientKey = createHmac("sha1", saltedPassword)
+    .update("Client Key")
+    .digest();
+  const storedKey = createHash("sha1").update(clientKey).digest();
+
+  const clientFinalMessageWithoutProof = `c=${encodeBase64(
+    gs2Header
+  )},r=${nonce}`;
+  const authMessage = `${clientFirstMessageBare},${serverFirstMessage},${clientFinalMessageWithoutProof}`;
+  const clientSignature = createHmac("sha1", storedKey)
+    .update(authMessage)
+    .digest();
+
+  const clientProof = Buffer.from(clientKey);
+  for (const [i, b] of clientSignature.entries()) clientProof[i] ^= b;
+
+  const clientFinalMessage = `${clientFinalMessageWithoutProof},p=${clientProof.toString(
+    "base64"
+  )}`;
+  socket.write(
+    `<response xmlns="urn:ietf:params:xml:ns:xmpp-sasl">${encodeBase64(
+      clientFinalMessage
+    )}</response>`
+  );
+  const res2 = yield;
+
+  if (
+    res2.name === "failure" &&
+    res2.children.some((c) => c.name === "not-authorized")
+  )
+    throw new Error("Not authorized");
+
+  if (res2.name !== "success")
+    throw new Error(`Unexpected response ${res2.name}`);
+
+  const serverKey = createHmac("sha1", saltedPassword)
+    .update("Server Key")
+    .digest();
+  const serverSignature = createHmac("sha1", serverKey)
+    .update(authMessage)
+    .digest("base64");
+
+  if (!decodeBase64(res2.text).endsWith(serverSignature))
+    throw new Error("Invalid server signature");
+}
+
+function* starttls(socket: net.Socket): Generator<void, void, Element> {
+  socket.write("<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>");
+  const res1 = yield;
+  if (res1.name !== "proceed") throw new Error("Failed to initiate STARTTLS");
+}
+
+function* bind(
+  socket: net.Socket,
+  resource: string
+): Generator<void, void, Element> {
+  const id = randomBytes(8).toString("base64");
+  socket.write(
+    `<iq id='${id}' type='set'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'><resource>${resource}</resource></bind></iq>`
+  );
+  const res1 = yield;
+  if (res1.name !== "iq") throw new Error(`Unexpected element ${res1.name}`);
+  const attrs1 = parseAttrs(res1.attrs);
+  const idAttr = attrs1.find((a) => a.name === "id");
+  if (!idAttr || idAttr.value !== id) throw new Error("Invalid ID");
+  const typeAttr = attrs1.find((a) => a.name === "type");
+  if (!typeAttr) throw new Error("Missing type attribute");
+  if (typeAttr.value !== "result") throw new Error("Cannot bind to resource");
+}
+
+const STATUS_RESTART_STREAM = 1;
+const STATUS_STARTTLS = 2;
+
+function* init(
+  socket: net.Socket,
+  host: string,
+  username: string,
+  password: string,
+  resource: string
+): Generator<void, number, Element> {
+  socket.write(
+    `<?xml version='1.0'?><stream:stream from='${username}@${host}' to='${host}' version='1.0' xml:lang='en' xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams'>`
+  );
+  const open = yield;
+  if (open.name !== "stream:stream")
+    throw new Error(`Unexpected element ${open.name}`);
+  const features = yield;
+  if (features.name !== "stream:features")
+    throw new Error(`Unexpected element ${features.name}`);
+  for (const feature of features.children) {
+    if (feature.name === "starttls") {
+      if (feature.children.some((c) => c.name === "required")) {
+        yield* starttls(socket);
+        return STATUS_STARTTLS;
+      }
+    } else if (feature.name === "mechanisms") {
+      const mechanisms: Set<string> = new Set(
+        feature.children.map((c) => c.text)
+      );
+      if (mechanisms.has("PLAIN")) {
+        yield* loginPlain(socket, username, password);
+        return STATUS_RESTART_STREAM;
+      } else if (mechanisms.has("SCRAM-SHA-1")) {
+        yield* loginScram(socket, username, password);
+        return STATUS_RESTART_STREAM;
+      } else {
+        throw new Error("No supported SASL method");
+      }
+    } else if (feature.name === "bind") {
+      yield* bind(socket, resource);
+      return 0;
+    }
+  }
+  return 0;
+}
+
+function upgradeTls(socket: net.Socket, host: string): Promise<tls.TLSSocket> {
+  return new Promise((resolve, reject) => {
+    socket.on("error", reject);
+    const newSocket = tls.connect({ socket, host }, () => {
+      socket.removeListener("error", reject);
+      resolve(newSocket);
+    });
+  });
+}
+
+interface XmppClientOptions {
+  host: string;
+  port?: number;
+  username?: string;
+  password?: string;
+  resource?: string;
+  timeout?: number;
+}
+
+export default class XmppClient extends EventEmitter {
+  private _socket: net.Socket;
+  private _host: string;
+  private _username: string;
+  private _resource: string;
+  private _iqStanzaCallbacks: Map<
+    string,
+    (err: Error, r?: { rawRes: string; res: Element }) => void
+  >;
+
+  private constructor() {
+    super();
+    this._socket = null;
+    this._host = null;
+    this._username = null;
+    this._resource = null;
+    this._iqStanzaCallbacks = new Map();
+  }
+
+  static connect(opts: XmppClientOptions): Promise<XmppClient> {
+    return new Promise((resolve, reject) => {
+      let socket = new net.Socket();
+      socket.on("error", reject);
+
+      socket.connect(opts.port || 5222, opts.host, async () => {
+        socket.removeListener("error", reject);
+        try {
+          let status = 1;
+          while (status) {
+            if (status === STATUS_STARTTLS)
+              socket = await upgradeTls(socket, opts.host);
+            status = await xmppStream(
+              socket,
+              init(
+                socket,
+                opts.host,
+                opts.username,
+                opts.password,
+                opts.resource
+              )
+            );
+          }
+
+          const client = new XmppClient();
+          client._socket = socket;
+          client._host = opts.host;
+          client._username = opts.username;
+          client._resource = opts.resource;
+          socket.on("data", client._onData.bind(client));
+          socket.on("error", client._onError.bind(client));
+          if (opts.timeout)
+            socket.setTimeout(opts.timeout, client.close.bind(client));
+          resolve(client);
+        } catch (err) {
+          socket.destroy();
+          reject(err);
+        }
+      });
+    });
+  }
+
+  close(): void {
+    this._socket.end("</stream:stream>");
+    this._socket.removeAllListeners("data");
+    this._socket.removeAllListeners("error");
+    this.emit("close");
+  }
+
+  ref(): void {
+    this._socket.ref();
+  }
+
+  unref(): void {
+    this._socket.unref();
+  }
+
+  get host(): string {
+    return this._host;
+  }
+
+  get username(): string {
+    return this._username;
+  }
+
+  get resource(): string {
+    return this._resource;
+  }
+
+  private _onData(chunk: Buffer): void {
+    try {
+      let close = false;
+      let str = chunk.toString("utf8");
+      if (str.endsWith("</stream:stream>")) {
+        str = str.slice(0, -16);
+        close = true;
+      }
+      const xml = parseXml(str);
+      const idx = xml.children.map((c) => c.bodyIndex);
+      for (const [i, c] of xml.children.entries()) {
+        const s = str.slice(idx[i], idx[i + 1]);
+        if (c.name === "iq") {
+          const attrs = parseAttrs(c.attrs);
+          const id = attrs.find((a) => a.name === "id");
+          if (id) {
+            const cb = this._iqStanzaCallbacks.get(id.value);
+            if (cb) cb(null, { rawRes: s, res: c });
+          }
+        }
+        this.emit("stanza", c, s);
+      }
+      if (close) {
+        this._socket.removeAllListeners("data");
+        this._socket.removeAllListeners("error");
+        this.emit("close");
+      }
+    } catch (err) {
+      this._socket.removeAllListeners("data");
+      this._socket.removeAllListeners("error");
+      this.emit("error", err);
+    }
+  }
+
+  private _onError(err: Error): void {
+    this._socket.end();
+    this.emit("error", err);
+    for (const cb of this._iqStanzaCallbacks.values()) cb(err);
+  }
+
+  send(msg: string): void {
+    this._socket.write(msg);
+  }
+
+  sendIqStanza(
+    from: string,
+    to: string,
+    type: string,
+    body: string,
+    timeout = 3000
+  ): Promise<{ rawReq: string; rawRes: string; res: Element }> {
+    return new Promise((resolve, reject) => {
+      const id = randomBytes(8).toString("base64");
+      const rawReq = `<iq from="${from}" to="${to}" id="${id}" type="${type}">${body}</iq>`;
+      this.send(rawReq);
+      const t = setTimeout(() => {
+        this._iqStanzaCallbacks.delete(id);
+        reject(
+          new Error("Did not receive IQ stanza response in a timely manner")
+        );
+      }, timeout);
+      this._iqStanzaCallbacks.set(id, (err, r) => {
+        this._iqStanzaCallbacks.delete(id);
+        clearTimeout(t);
+        if (err) reject(err);
+        else resolve(Object.assign(r, { rawReq }));
+      });
+    });
+  }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.0",
+  "version": "1.2.3",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -20,6 +20,7 @@
     "tools": "tools"
   },
   "dependencies": {
+    "espresso-iisojs": "^1.0.3",
     "iconv-lite": "^0.6.2",
     "ipaddr.js": "^2.0.0",
     "jsonwebtoken": "^8.5.1",
@@ -30,48 +31,48 @@
     "koa-router": "^9.4.0",
     "koa-static": "^5.0.0",
     "later": "^1.2.0",
-    "mongodb": "^3.6.0",
+    "mongodb": "^3.6.2",
     "parsimmon": "^1.16.0",
     "seedrandom": "^3.0.5",
-    "yaml": "^1.10.0",
-    "yebool": "^1.0.2"
+    "yaml": "^1.10.0"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^15.0.0",
+    "@rollup/plugin-commonjs": "^16.0.0",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^9.0.0",
-    "@rollup/plugin-typescript": "^5.0.2",
+    "@rollup/plugin-node-resolve": "^10.0.0",
+    "@rollup/plugin-typescript": "^6.0.0",
     "@types/jsonwebtoken": "^8.5.0",
-    "@types/koa": "^2.11.4",
-    "@types/koa-compress": "^4.0.0",
+    "@types/koa": "^2.11.6",
+    "@types/koa-compress": "^4.0.1",
     "@types/koa-router": "^7.4.1",
     "@types/koa-static": "^4.0.1",
-    "@types/mithril": "^2.0.3",
-    "@types/mongodb": "^3.5.26",
-    "@types/parsimmon": "^1.10.2",
+    "@types/mithril": "^2.0.5",
+    "@types/mongodb": "^3.5.31",
+    "@types/node": "^14.14.5",
+    "@types/parsimmon": "^1.10.3",
     "@types/seedrandom": "^2.4.28",
     "@types/svgo": "^1.3.3",
-    "@typescript-eslint/eslint-plugin": "^3.10.1",
-    "@typescript-eslint/parser": "^3.10.1",
-    "ava": "^3.12.1",
-    "codemirror": "^5.57.0",
+    "@typescript-eslint/eslint-plugin": "^4.6.0",
+    "@typescript-eslint/parser": "^4.6.0",
+    "ava": "^3.13.0",
+    "codemirror": "^5.58.2",
     "cssnano": "^4.1.10",
-    "eslint": "^7.8.0",
-    "eslint-config-prettier": "^6.11.0",
+    "eslint": "^7.12.1",
+    "eslint-config-prettier": "^6.14.0",
     "esm": "^3.2.25",
-    "jsbi": "^3.1.3",
+    "jsbi": "^3.1.4",
     "mithril": "^2.0.4",
-    "postcss": "^7.0.32",
-    "postcss-import": "^12.0.1",
+    "postcss": "^8.1.4",
+    "postcss-import": "^13.0.0",
     "postcss-preset-env": "^6.7.0",
-    "prettier": "^2.1.1",
-    "rollup": "^2.26.9",
-    "rollup-plugin-terser": "^7.0.1",
-    "source-map-loader": "^1.1.0",
+    "prettier": "^2.1.2",
+    "rollup": "^2.32.1",
+    "rollup-plugin-terser": "^7.0.2",
+    "source-map-loader": "^1.1.1",
     "svgo": "^1.3.2",
     "ts-node": "^9.0.0",
-    "typescript": "^4.0.2",
-    "webpack": "^4.44.1"
+    "typescript": "^4.0.5",
+    "webpack": "^5.2.0"
   },
   "engines": {
     "node": ">=12.13.0"

test/mongodb-functions.ts

@@ -1,13 +1,13 @@
 import ava from "ava";
 import * as mongodbFunctions from "../lib/mongodb-functions";
-import { stringify } from "../lib/common/expression-parser";
+import { stringify, parse } from "../lib/common/expression-parser";
 
-ava("mongoQueryToFilter", (t) => {
+ava("convertOldPrecondition", (t) => {
   const tests = [
     [{}, "true"],
     [{ test: "test" }, 'test = "test"'],
     [{ test: { $eq: "test" } }, 'test = "test"'],
-    [{ test: { $ne: "test" } }, 'test <> "test"'],
+    [{ test: { $ne: "test" } }, 'test <> "test" OR test IS NULL'],
     [{ test: { $gte: "test" } }, 'test >= "test"'],
     [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
     [{ "Tags.test": false }, "Tags.test IS NULL"],
@@ -22,20 +22,20 @@ ava("mongoQueryToFilter", (t) => {
     [{ _tags: { $eq: "test" } }, "Tags.test IS NOT NULL"],
     [
       { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" AND test <> "test"',
+      'test = "test" AND (test <> "test" OR test IS NULL)',
     ],
     [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
     [
       { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" OR test <> "test"',
+      'test = "test" OR (test <> "test" OR test IS NULL)',
     ],
     [
       { test: { $gte: "test1", $ne: "test2" } },
-      'test >= "test1" AND test <> "test2"',
+      'test >= "test1" AND (test <> "test2" OR test IS NULL)',
     ],
     [
       { test: "test", test2: { $ne: "test2" } },
-      'test = "test" AND test2 <> "test2"',
+      'test = "test" AND (test2 <> "test2" OR test2 IS NULL)',
     ],
   ];
 
@@ -50,13 +50,83 @@ ava("mongoQueryToFilter", (t) => {
 
   t.plan(tests.length + 2 * shouldFailTests.length);
   for (const test of tests)
-    t.is(stringify(mongodbFunctions.mongoQueryToFilter(test[0])), test[1]);
+    t.is(stringify(mongodbFunctions.convertOldPrecondition(test[0])), test[1]);
 
   for (const test of shouldFailTests) {
     const func = (): void => {
-      mongodbFunctions.mongoQueryToFilter(test[0] as Record<string, unknown>);
+      mongodbFunctions.convertOldPrecondition(
+        test[0] as Record<string, unknown>
+      );
     };
     const error = t.throws(func, { instanceOf: Error });
     t.is(error.message, test[1]);
   }
 });
+
+ava("filterToMongoQuery", async (t) => {
+  const queries: [string, Record<string, unknown>][] = [
+    ["true", {}],
+    ["Tags.tag1 = true", { _tags: "tag1" }],
+    ["Tags.tag1 <> false", { _tags: "tag1" }],
+    ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }],
+    ["Tags.tag1 = 123", { "Tags.tag1": 123 }],
+    ["Param1 = 'value1'", { "Param1._value": "value1" }],
+    ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }],
+    ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
+    ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
+    ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }],
+    ["LOWER(Param1) LIKE 'value'", { "Param1._value": /^value$/i }],
+    [
+      "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          {
+            $and: [
+              { "Param2._value": { $nin: ["value1", null] } },
+              { "Param1._value": { $gte: "value2" } },
+            ],
+          },
+        ],
+      },
+    ],
+    [
+      "Param1 <> 'value2' OR Param1 IS NULL",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          { "Param1._value": null },
+        ],
+      },
+    ],
+  ];
+
+  for (const [expStr, expect] of queries) {
+    const exp = parse(expStr);
+    const query = mongodbFunctions.filterToMongoQuery(
+      mongodbFunctions.processDeviceFilter(exp)
+    );
+    t.deepEqual(query, expect);
+  }
+
+  const failQueries: [any, string][] = [
+    ["Param1 = Param2", "Invalid RHS operand of = clause"],
+    ["Param1 LIKE Param2", "Invalid RHS operand of LIKE clause"],
+    ["1 LIKE '1'", "Invalid LHS operand of LIKE clause"],
+    ["1 = 2", "Invalid LHS operand of = clause"],
+    ["1 IS NULL", "Invalid LHS operand of IS NULL clause"],
+    ["false", "Primitives are not valid queries"],
+    ["UPPER(Param1) LIKE 'value'", "Invalid RHS operand of LIKE clause"],
+  ];
+
+  for (const [expStr, err] of failQueries) {
+    const exp = parse(expStr);
+    t.throws(
+      () =>
+        mongodbFunctions.filterToMongoQuery(
+          mongodbFunctions.processDeviceFilter(exp)
+        ),
+      { message: err }
+    );
+  }
+});

ui/app.ts

@@ -71,7 +71,7 @@ let fulfillTimeout = null;
 function fulfill(): void {
   clearTimeout(fulfillTimeout);
   fulfillTimeout = setTimeout(() => {
-    store.fulfill(lastRenderTimestamp, pageVisitTimestamp).then((updated) => {
+    store.fulfill(lastRenderTimestamp, pageVisitTimestamp, (updated) => {
       if (updated) m.redraw();
     });
   }, 100);

ui/components/device-actions.ts

@@ -70,7 +70,7 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageDownload({
                 name: "download",
-                device: device["DeviceID.ID"].value[0],
+                devices: [device["DeviceID.ID"].value[0]],
               });
             },
           },

ui/components/parameter.ts

@@ -90,7 +90,7 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageSpv({
                 name: "setParameterValues",
-                device: device["DeviceID.ID"].value[0],
+                devices: [device["DeviceID.ID"].value[0]],
                 parameterValues: [
                   [
                     parameter,

ui/components/ping.ts

@@ -27,23 +27,28 @@ const component: ClosureComponent = (vn): Component => {
   let interval: ReturnType<typeof setInterval>;
   let host: string;
 
-  const refresh = async (): Promise<void> => {
+  const refresh = (): void => {
+    if (!host) {
+      const dom = (vn as VnodeDOM).dom;
+      if (dom) dom.innerHTML = "";
+      return;
+    }
+
     let status = "";
-    if (host) {
-      try {
-        const res = await store.ping(host);
+    store
+      .ping(host)
+      .then((res) => {
         if (res["avg"] != null) status = `${Math.trunc(res["avg"])} ms`;
         else status = "Unreachable";
-      } catch (err) {
-        setTimeout(() => {
-          throw err;
-        }, 0);
-      }
-    }
-
+      })
+      .catch(() => {
+        status = "Error!";
+        clearInterval(interval);
+      })
+      .finally(() => {
         const dom = (vn as VnodeDOM).dom;
-
         if (dom) dom.innerHTML = `Pinging ${host}: ${status}`;
+      });
   };
 
   return {

ui/components/summon-button.ts

@@ -79,6 +79,10 @@ const component: ClosureComponent = (): Component => {
               .then(() => {
                 e.target.disabled = false;
                 store.fulfill(0, Date.now());
+              })
+              .catch((err) => {
+                e.target.disabled = false;
+                notifications.push("error", err.message);
               });
           },
         },

ui/components/tags.ts

@@ -22,6 +22,7 @@ import { m } from "../components";
 import * as notifications from "../notifications";
 import * as store from "../store";
 import { getIcon } from "../icons";
+import { decodeTag } from "../../lib/common";
 
 const component: ClosureComponent = (): Component => {
   return {
@@ -32,7 +33,7 @@ const component: ClosureComponent = (): Component => {
 
       const tags = [];
       for (const p of Object.keys(device))
-        if (p.startsWith("Tags.")) tags.push(p.slice(5));
+        if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
 
       tags.sort();
 

ui/config-functions.ts

@@ -42,6 +42,35 @@ export function flattenConfig(config: Record<string, unknown>): any {
   return flatten;
 }
 
+// Order keys such that nested objects come last
+function orderKeys(config: any): number {
+  let res = 1;
+  if (typeof config !== "object") return res;
+  if (Array.isArray(config)) {
+    for (const c of config) res += orderKeys(c);
+    return res;
+  }
+
+  const weights: [string, number][] = Object.entries(config).map(([k, v]) => [
+    k,
+    orderKeys(v),
+  ]);
+
+  weights.sort((a, b) => {
+    if (a[1] !== b[1]) return a[1] - b[1];
+    if (b[0] > a[0]) return -1;
+    else return 1;
+  });
+
+  for (const [k, w] of weights) {
+    res += w;
+    const v = config[k];
+    delete config[k];
+    config[k] = v;
+  }
+  return res;
+}
+
 export function structureConfig(config: Config[]): any {
   config.sort((a, b) => (a._id > b._id ? 1 : a._id < b._id ? -1 : 0));
   const _config = {};
@@ -93,7 +122,9 @@ export function structureConfig(config: Config[]): any {
     return object;
   };
 
-  return toArray(_config);
+  const res = toArray(_config);
+  orderKeys(res);
+  return res;
 }
 
 export function diffConfig(

ui/config-page.ts

@@ -147,8 +147,8 @@ function renderTable(confsResponse, searchString): Children {
       {
         title: "Edit config value",
         onclick: () => {
-          const cb = (): Children => {
-            return m(
+          let cb: () => Children = null;
+          const comp = m(
             putFormComponent,
             Object.assign(
               {
@@ -176,8 +176,13 @@ function renderTable(confsResponse, searchString): Children {
               formData
             )
           );
-          };
-          overlay.open(cb);
+          cb = () => comp;
+          overlay.open(
+            cb,
+            () =>
+              !comp.state["current"]["modified"] ||
+              confirm("You have unsaved changes. Close anyway?")
+          );
         },
       },
       getIcon("edit")
@@ -244,8 +249,8 @@ export const component: ClosureComponent = (): Component => {
           {
             title: "Create new config",
             onclick: () => {
-              const cb = (): Children => {
-                return m(
+              let cb: () => Children = null;
+              const comp = m(
                 putFormComponent,
                 Object.assign(
                   {
@@ -274,8 +279,13 @@ export const component: ClosureComponent = (): Component => {
                   formData
                 )
               );
-              };
-              overlay.open(cb);
+              cb = () => comp;
+              overlay.open(
+                cb,
+                () =>
+                  !comp.state["current"]["modified"] ||
+                  confirm("You have unsaved changes. Close anyway?")
+              );
             },
           },
           "New config"
@@ -307,8 +317,8 @@ export const component: ClosureComponent = (): Component => {
               "button",
               {
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     uiConfigComponent,
                     Object.assign(
                       {
@@ -338,8 +348,13 @@ export const component: ClosureComponent = (): Component => {
                       attrs
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               `Edit ${sub.name}`

ui/css/app.css

@@ -178,6 +178,8 @@ body {
       }
 
       & > .queue {
+        max-height: 400px;
+        overflow-y: auto;
         & > .pending {
           animation: 0.5s linear 0s infinite alternate pending-blink;
           border-radius: 0.4em;

ui/devices-page.ts

@@ -23,7 +23,7 @@ import config from "./config";
 import indexTableComponent from "./index-table-component";
 import filterComponent from "./filter-component";
 import * as store from "./store";
-import * as taskQueue from "./task-queue";
+import { queueTask, stageDownload } from "./task-queue";
 import * as notifications from "./notifications";
 import { parse, stringify } from "../lib/common/expression-parser";
 import { evaluate, extractParams } from "../lib/common/expression";
@@ -91,12 +91,11 @@ function renderActions(selected: Set<string>): Children {
         title: "Reboot selected devices",
         disabled: !selected.size,
         onclick: () => {
-          for (const d of selected) {
-            taskQueue.queueTask({
+          const tasks = [...selected].map((s) => ({
             name: "reboot",
-              device: d,
-            });
-          }
+            device: s,
+          }));
+          queueTask(...tasks);
         },
       },
       "Reboot"
@@ -110,18 +109,34 @@ function renderActions(selected: Set<string>): Children {
         title: "Factory reset selected devices",
         disabled: !selected.size,
         onclick: () => {
-          for (const d of selected) {
-            taskQueue.queueTask({
+          const tasks = [...selected].map((s) => ({
             name: "factoryReset",
-              device: d,
-            });
-          }
+            device: s,
+          }));
+          queueTask(...tasks);
         },
       },
       "Reset"
     )
   );
 
+  buttons.push(
+    m(
+      "button.critical",
+      {
+        title: "Push a firmware or a config file",
+        disabled: !selected.size,
+        onclick: () => {
+          stageDownload({
+            name: "download",
+            devices: [...selected],
+          });
+        },
+      },
+      "Push file"
+    )
+  );
+
   buttons.push(
     m(
       "button.primary",

ui/drawer-component.ts

@@ -24,10 +24,21 @@ import m, {
   Component,
   VnodeDOM,
 } from "mithril";
-import * as taskQueue from "./task-queue";
 import * as store from "./store";
 import * as notifications from "./notifications";
 import { getIcon } from "./icons";
+import {
+  clear,
+  commit,
+  deleteTask,
+  getQueue,
+  getStaging,
+  QueueTask,
+  queueTask,
+  StageTask,
+} from "./task-queue";
+
+const invalid: WeakSet<StageTask> = new WeakSet();
 
 function mparam(param): Children {
   return m("span.parameter", { title: param }, `${param}`);
@@ -37,11 +48,11 @@ function mval(val): Children {
   return m("span.value", { title: val }, `${val}`);
 }
 
-function renderStagingSpv(task, queueFunc, cancelFunc): Children {
-  function keydown(e): void {
-    if (e.keyCode === 13) queueFunc();
-    else if (e.keyCode === 27) cancelFunc();
-    else e.redraw = false;
+function renderStagingSpv(task: StageTask, queueFunc, cancelFunc): Children {
+  function keydown(e: KeyboardEvent): void {
+    if (e.key === "Enter") queueFunc();
+    else if (e.key === "Escape") cancelFunc();
+    else e["redraw"] = false;
   }
 
   let input;
@@ -90,13 +101,24 @@ function renderStagingSpv(task, queueFunc, cancelFunc): Children {
   return [m("span", "Editing ", mparam(task.parameterValues[0][0])), input];
 }
 
-function renderStagingDownload(task): Children {
-  task.invalid = !task.fileName || !task.fileType;
+function renderStagingDownload(task: StageTask): Children {
+  if (!task.fileName || !task.fileType) invalid.add(task);
+  else invalid.delete(task);
   const files = store.fetch("files", true);
-  const idParts = task.device.split("-");
-  const oui = decodeURIComponent(idParts[0]);
-  const productClass =
-    idParts.length === 3 ? decodeURIComponent(idParts[1]) : "";
+  let oui = "";
+  let productClass = "";
+  for (const d of task.devices) {
+    const parts = d.split("-");
+    if (oui === "") oui = parts[0];
+    else if (oui !== parts[0]) oui = null;
+    if (parts.length === 3) {
+      if (productClass === "") productClass = parts[1];
+      else if (productClass !== parts[1]) productClass = null;
+    }
+  }
+
+  if (oui) oui = decodeURIComponent(oui);
+  if (productClass) productClass = decodeURIComponent(productClass);
 
   const typesList = [
     "",
@@ -162,13 +184,17 @@ function renderStagingDownload(task): Children {
   ];
 }
 
-function renderStaging(staging): Child[] {
+function renderStaging(staging: Set<StageTask>): Child[] {
   const elements: Child[] = [];
 
   for (const s of staging) {
     const queueFunc = (): void => {
       staging.delete(s);
-      taskQueue.queueTask(s);
+      for (const d of s.devices) {
+        const t = Object.assign({ device: d }, s);
+        delete t.devices;
+        queueTask(t);
+      }
     };
     const cancelFunc = (): void => {
       staging.delete(s);
@@ -181,7 +207,7 @@ function renderStaging(staging): Child[] {
 
     const queue = m(
       "button.primary",
-      { title: "Queue task", onclick: queueFunc, disabled: s.invalid },
+      { title: "Queue task", onclick: queueFunc, disabled: invalid.has(s) },
       "Queue"
     );
     const cancel = m(
@@ -195,7 +221,7 @@ function renderStaging(staging): Child[] {
   return elements;
 }
 
-function renderQueue(queue): Child[] {
+function renderQueue(queue: Set<QueueTask>): Child[] {
   const details: Child[] = [];
   const devices: { [deviceId: string]: any[] } = {};
   for (const t of queue) {
@@ -215,7 +241,7 @@ function renderQueue(queue): Child[] {
             {
               title: "Retry this task",
               onclick: () => {
-                taskQueue.queueTask(t);
+                queueTask(t);
               },
             },
             getIcon("retry")
@@ -229,7 +255,7 @@ function renderQueue(queue): Child[] {
           {
             title: "Remove this task",
             onclick: () => {
-              taskQueue.deleteTask(t);
+              deleteTask(t);
             },
           },
           getIcon("remove")
@@ -347,8 +373,8 @@ function renderNotifications(notifs): Child[] {
 const component: ClosureComponent = (): Component => {
   return {
     view: (vnode) => {
-      const queue = taskQueue.getQueue();
-      const staging = taskQueue.getStaging();
+      const queue = getQueue();
+      const staging = getStaging();
       const notifs = notifications.getNotifications();
 
       let drawerElement, statusElement;
@@ -394,11 +420,10 @@ const component: ClosureComponent = (): Component => {
               title: "Commit queued tasks",
               disabled: !statusCount.queued,
               onclick: () => {
-                const tasks = Array.from(taskQueue.getQueue()).filter(
+                const tasks = Array.from(getQueue()).filter(
                   (t) => t["status"] === "queued"
                 );
-                taskQueue
-                  .commit(
+                commit(
                   tasks,
                   (deviceId, err, connectionRequestStatus, tasks2) => {
                     if (err) {
@@ -441,6 +466,9 @@ const component: ClosureComponent = (): Component => {
                 )
                   .then(() => {
                     store.fulfill(0, Date.now());
+                  })
+                  .catch((err) => {
+                    notifications.push("error", err.message);
                   });
               },
             },
@@ -450,7 +478,7 @@ const component: ClosureComponent = (): Component => {
             "button",
             {
               title: "Clear tasks",
-              onclick: taskQueue.clear,
+              onclick: clear,
               disabled: !queueElements.length,
             },
             "Clear"

ui/dynamic-loader.ts

@@ -51,7 +51,7 @@ export function loadCodeMirror(): Promise<void> {
     ];
     Promise.all(promises)
       .then((modules) => {
-        codeMirror = modules[0];
+        codeMirror = modules[0].default;
         resolve();
       })
       .catch((err) => {

ui/files-page.ts

@@ -221,8 +221,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new file",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -251,8 +251,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/filter-component.ts

@@ -44,8 +44,12 @@ const getAutocomplete = memoize((resource) => {
 
 function validateQuery(q: Expression): void {
   q = normalize(q);
-  if (Array.isArray(q) && q[0] === "CASE") q = ["AND", ...q.slice(1)];
-  filterToMongoQuery(q);
+  if (Array.isArray(q) && q[0] === "CASE") {
+    q = q.slice(1).filter((a) => Array.isArray(a));
+    if (q.length > 1) q = ["AND", ...q];
+    else q = q[0];
+  }
+  if (Array.isArray(q)) filterToMongoQuery(q);
 }
 
 function parseFilter(resource, f): Expression {

ui/index-table-component.ts

@@ -27,6 +27,8 @@ interface Attribute {
   type?: string;
 }
 
+const MAX_PAGE_SIZE = 200;
+
 function renderTable(
   attributes: Attribute[],
   data: Record<string, any>[],
@@ -37,7 +39,7 @@ function renderTable(
   onSortChange: (obj: Record<string, number>) => void,
   downloadUrl?: string,
   valueCallback?: (attr: Attribute, record: Record<string, any>) => Children,
-  actionsCallback?: Children | ((selected: Set<string>) => Children),
+  actionsCallback?: Children | ((sel: Set<string>) => Children),
   recordActionsCallback?: Children | ((record: Record<string, any>) => Children)
 ): Children {
   const records = data || [];
@@ -180,7 +182,8 @@ function renderTable(
       {
         title: "Show more records",
         onclick: showMoreCallback,
-        disabled: !data.length || records.length >= total,
+        disabled:
+          !data.length || records.length >= Math.min(MAX_PAGE_SIZE, total),
       },
       "More"
     )

ui/login-page.ts

@@ -81,7 +81,7 @@ export const component: ClosureComponent = (): Component => {
                       location.reload();
                     })
                     .catch((err) => {
-                      notifications.push("error", err.message);
+                      notifications.push("error", err.response || err.message);
                       e.target.disabled = false;
                     });
                   return false;

ui/overlay.ts

@@ -20,16 +20,24 @@
 import m, { Children } from "mithril";
 
 type OverlayCallback = () => Children;
+type CloseCallback = () => boolean;
 
 let overlayCallback: OverlayCallback = null;
+let closeCallback: CloseCallback = null;
 
-export function open(callback: OverlayCallback): void {
+export function open(
+  callback: OverlayCallback,
+  closeCb: CloseCallback = null
+): void {
   overlayCallback = callback;
+  closeCallback = closeCb;
 }
 
-export function close(callback: OverlayCallback): boolean {
+export function close(callback: OverlayCallback, force = true): boolean {
   if (callback === overlayCallback) {
+    if (!force && closeCallback && !closeCallback()) return false;
     overlayCallback = null;
+    closeCallback = null;
     return true;
   }
 
@@ -43,7 +51,7 @@ export function render(): Children {
       {
         tabindex: 0,
         onclick: () => {
-          close(overlayCallback);
+          close(overlayCallback, false);
         },
         style: "opacity: 0",
         oncreate: (vnode) => {
@@ -75,9 +83,10 @@ export function render(): Children {
 }
 
 document.addEventListener("keydown", (e) => {
-  if (overlayCallback && e.keyCode === 27 && close(overlayCallback)) m.redraw();
+  if (overlayCallback && e.key === "Escape" && close(overlayCallback, false))
+    m.redraw();
 });
 
 window.addEventListener("popstate", () => {
-  if (close(overlayCallback)) m.redraw();
+  if (close(overlayCallback, false)) m.redraw();
 });

ui/permissions-page.ts

@@ -285,8 +285,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new permission",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -315,8 +315,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/presets-page.ts

@@ -287,16 +287,8 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
-                  if (!preset.provision) {
-                    return m(
-                      "div",
-                      { style: "margin:20px" },
-                      "This UI only supports presets with a single 'provision' configuraiton. If this preset was originally created from the old UI (genieacs-gui), you must edit it there."
-                    );
-                  }
-
-                  return m(
+                let cb: () => Children = null;
+                const comp = m(
                   putFormComponent,
                   Object.assign(
                     {
@@ -326,8 +318,22 @@ export const component: ClosureComponent = (): Component => {
                     formData
                   )
                 );
+                cb = (): Children => {
+                  if (!preset.provision) {
+                    return m(
+                      "div",
+                      { style: "margin:20px" },
+                      "This UI only supports presets with a single 'provision' configuraiton. If this preset was originally created from the old UI (genieacs-gui), you must edit it there."
+                    );
+                  }
+                  return comp;
                 };
-                overlay.open(cb);
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
               },
             },
             "Show"
@@ -343,8 +349,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new preset",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -373,8 +379,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/provisions-page.ts

@@ -207,8 +207,8 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
-                  return m(
+                let cb: () => Children = null;
+                const comp = m(
                   putFormComponent,
                   Object.assign(
                     {
@@ -238,8 +238,13 @@ export const component: ClosureComponent = (): Component => {
                     formData
                   )
                 );
-                };
-                overlay.open(cb);
+                cb = () => comp;
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
               },
             },
             "Show"
@@ -255,8 +260,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new provision",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -285,8 +290,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/put-form-component.ts

@@ -56,6 +56,7 @@ function createField(current, attr, focus): Children {
           : null,
         onchange: (e) => {
           current.object[attr.id] = e.target.value;
+          current.modified = true;
           e.redraw = false;
         },
       },
@@ -80,6 +81,7 @@ function createField(current, attr, focus): Children {
           if (e.target.checked) currentSelected.add(op);
           else currentSelected.delete(op);
           current.object[attr.id] = Array.from(currentSelected);
+          current.modified = true;
           e.redraw = false;
         },
       };
@@ -98,6 +100,7 @@ function createField(current, attr, focus): Children {
       },
       onChange: (value) => {
         current.object[attr.id] = value;
+        current.modified = true;
       },
     };
     return m(codeEditorComponent, attrs);
@@ -112,6 +115,7 @@ function createField(current, attr, focus): Children {
         : null,
       onchange: (e) => {
         current.object[attr.id] = e.target.files;
+        current.modified = true;
         e.redraw = false;
       },
     });
@@ -132,6 +136,7 @@ function createField(current, attr, focus): Children {
         : null,
       oninput: (e) => {
         current.object[attr.id] = e.target.value;
+        current.modified = true;
         e.redraw = false;
       },
       onkeypress: (e) => {
@@ -158,6 +163,7 @@ function createField(current, attr, focus): Children {
       : null,
     oninput: (e) => {
       current.object[attr.id] = e.target.value;
+      current.modified = true;
       e.redraw = false;
     },
   });
@@ -174,6 +180,7 @@ const component: ClosureComponent = (): Component => {
         vnode.state["current"] = {
           isNew: !base["_id"],
           object: Object.assign({}, base),
+          modified: false,
         };
       }
 

ui/smart-query.ts

@@ -19,6 +19,7 @@
 
 import config from "./config";
 import { Expression } from "../lib/types";
+import { encodeTag } from "../lib/common";
 
 const resources = {
   devices: {},
@@ -138,7 +139,7 @@ function queryMac(param, value): Expression {
 }
 
 function queryTag(tag: string): Expression {
-  const t = tag.replace(/[^a-zA-Z0-9-]+/g, "_");
+  const t = encodeTag(tag);
   return ["IS NOT NULL", ["PARAM", `Tags.${t}`]];
 }
 

ui/store.ts

@@ -152,6 +152,8 @@ function checkConnection(): void {
         }
       }
     },
+  }).catch((err) => {
+    notifications.push("error", err.message);
   });
 }
 
@@ -169,11 +171,26 @@ export async function xhrRequest(
   ): any => {
     if (typeof extract === "function") return extract(xhr, _options);
 
+    // https://mithril.js.org/request.html#error-handling
+    if (xhr.status !== 304 && Math.floor(xhr.status / 100) !== 2) {
+      if (xhr.status === 403) throw new Error("Not authorized");
+      const err = new Error();
+      err["message"] =
+        xhr.status === 0
+          ? "Server is unreachable"
+          : `Unexpected response status code ${xhr.status}`;
+      err["code"] = xhr.status;
+      err["response"] = xhr.responseText;
+      throw err;
+    }
+
     let response: any;
     if (typeof deserialize === "function") {
       response = deserialize(xhr.responseText);
     } else if (
-      xhr.getResponseHeader("content-type").startsWith("application/json")
+      (xhr.getResponseHeader("content-type") || "").startsWith(
+        "application/json"
+      )
     ) {
       try {
         response = xhr.responseText ? JSON.parse(xhr.responseText) : null;
@@ -184,15 +201,6 @@ export async function xhrRequest(
       response = xhr.responseText;
     }
 
-    // https://mithril.js.org/request.html#error-handling
-    if (xhr.status !== 304 && Math.floor(xhr.status / 100) !== 2) {
-      const err = new Error();
-      err["message"] = xhr.responseText;
-      err["code"] = xhr.status;
-      err["response"] = response;
-      throw err;
-    }
-
     return response;
   };
 
@@ -340,7 +348,7 @@ export function fetch(
   return queryResponse;
 }
 
-export function fulfill(
+function _fulfill(
   accessTimestamp: number,
   _fulfillTimestamp: number
 ): Promise<boolean> {
@@ -377,7 +385,17 @@ export function fulfill(
                 m.buildQueryString({
                   filter: memoizedStringify(filter),
                 }),
-              extract: (xhr) => +xhr.getResponseHeader("x-total-count"),
+              extract: (xhr) => {
+                if (xhr.status === 403) throw new Error("Not authorized");
+                if (!xhr.status) {
+                  throw new Error("Server is unreachable");
+                } else if (xhr.status !== 200) {
+                  throw new Error(
+                    `Unexpected response status code ${xhr.status}`
+                  );
+                }
+                return +xhr.getResponseHeader("x-total-count");
+              },
               background: true,
             })
               .then((c) => {
@@ -497,14 +515,14 @@ export function fulfill(
             continue;
           }
 
-          const combinedFilterDiff = diff;
-          resources[resourceType].combinedFilter = union;
-
           updated = true;
           let deleted = new Set<string>();
           if (!resources[resourceType].combinedFilter)
             deleted = new Set(resources[resourceType].objects.keys());
 
+          const combinedFilterDiff = diff;
+          resources[resourceType].combinedFilter = union;
+
           allPromises2.push(
             new Promise((resolve2, reject2) => {
               xhrRequest({
@@ -560,6 +578,17 @@ export function fulfill(
   });
 }
 
+export function fulfill(
+  accessTimestamp: number,
+  _fulfillTimestamp: number,
+  callback?: (updated: boolean) => void
+): void {
+  _fulfill(accessTimestamp, _fulfillTimestamp).then(callback, (err) => {
+    notifications.push("error", err.message);
+    if (callback) callback(false);
+  });
+}
+
 export function getTimestamp(): number {
   return fulfillTimestamp;
 }
@@ -578,6 +607,8 @@ export function postTasks(
     url: `api/devices/${encodeURIComponent(deviceId)}/tasks`,
     body: tasks,
     extract: (xhr) => {
+      if (xhr.status === 403) throw new Error("Not authorized");
+      if (!xhr.status) throw new Error("Server is unreachable");
       if (xhr.status !== 200) throw new Error(xhr.response);
       const connectionRequestStatus = xhr.getResponseHeader(
         "Connection-Request"
@@ -647,7 +678,13 @@ export function resourceExists(resource: string, id: string): Promise<number> {
       m.buildQueryString({
         filter: memoizedStringify(filter),
       }),
-    extract: (xhr) => +xhr.getResponseHeader("x-total-count"),
+    extract: (xhr) => {
+      if (xhr.status === 403) throw new Error("Not authorized");
+      if (!xhr.status) throw new Error("Server is unreachable");
+      else if (xhr.status !== 200)
+        throw new Error(`Unexpected response status code ${xhr.status}`);
+      return +xhr.getResponseHeader("x-total-count");
+    },
     background: true,
   });
 }

ui/task-queue.ts

@@ -19,23 +19,41 @@
 
 import * as store from "./store";
 import { Task } from "../lib/types";
+import * as notifications from "./notifications";
 
 export interface QueueTask extends Task {
   status?: string;
-  device?: string;
+  device: string;
 }
 
-const queue: Set<QueueTask> = new Set();
-const staging: Set<QueueTask> = new Set();
+export interface StageTask extends Task {
+  devices: string[];
+}
 
-export function queueTask(task: QueueTask): void {
-  staging.delete(task);
+const MAX_QUEUE = 100;
+
+const queue: Set<QueueTask> = new Set();
+const staging: Set<StageTask> = new Set();
+
+function canQueue(tasks: QueueTask[]): boolean {
+  let count = queue.size;
+  for (const task of tasks) if (!queue.has(task)) ++count;
+  return count <= MAX_QUEUE;
+}
+
+export function queueTask(...tasks: QueueTask[]): void {
+  if (!canQueue(tasks)) {
+    notifications.push("error", "Too many tasks in queue");
+    return;
+  }
+
+  for (const task of tasks) {
     task.status = "queued";
     queue.add(task);
+  }
 }
 
 export function deleteTask(task: QueueTask): void {
-  staging.delete(task);
   queue.delete(task);
 }
 
@@ -47,7 +65,7 @@ export function clear(): void {
   queue.clear();
 }
 
-export function getStaging(): Set<QueueTask> {
+export function getStaging(): Set<StageTask> {
   return staging;
 }
 
@@ -55,11 +73,19 @@ export function clearStaging(): void {
   staging.clear();
 }
 
-export function stageSpv(task: QueueTask): void {
+export function stageSpv(task: StageTask): void {
+  if (queue.size + task.devices.length > MAX_QUEUE) {
+    notifications.push("error", "Too many tasks in queue");
+    return;
+  }
   staging.add(task);
 }
 
-export function stageDownload(task: QueueTask): void {
+export function stageDownload(task: StageTask): void {
+  if (queue.size + task.devices.length > MAX_QUEUE) {
+    notifications.push("error", "Too many tasks in queue");
+    return;
+  }
   staging.add(task);
 }
 
@@ -69,14 +95,19 @@ export function commit(
     deviceId: string,
     err: Error,
     conReqStatus: string,
-    tasks: QueueTask[]
+    _tasks: QueueTask[]
   ) => void
 ): Promise<void> {
   const devices: { [deviceId: string]: any[] } = {};
+
+  if (!canQueue(tasks))
+    return Promise.reject(new Error("Too many tasks in queue"));
+
   for (const t of tasks) {
     devices[t.device] = devices[t.device] || [];
     devices[t.device].push(t);
-    queueTask(t);
+    t.status = "queued";
+    queue.add(t);
   }
 
   return new Promise((resolve) => {

ui/ui-config-component.ts

@@ -102,12 +102,7 @@ const component: ClosureComponent = (): Component => {
 
       const yamlString =
         config && Object.values(config).length
-          ? yaml.stringify(config, {
-              schema: "failsafe",
-              tags: (args) => {
-                return args;
-              },
-            })
+          ? yaml.stringify(config, { schema: "failsafe" })
           : "";
 
       const attrs = {
@@ -120,6 +115,7 @@ const component: ClosureComponent = (): Component => {
         },
         onChange: (value) => {
           vnode.state["updatedYaml"] = value;
+          vnode.state["modified"] = true;
         },
       };
 

ui/users-page.ts

@@ -237,9 +237,8 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
-                  const children = [
-                    m(
+                let cb: () => Children = null;
+                const comp = m(
                   putFormComponent,
                   Object.assign(
                     {
@@ -274,9 +273,10 @@ export const component: ClosureComponent = (): Component => {
                       attributes: attributes,
                     }
                   )
-                    ),
-                  ];
+                );
 
+                cb = () => {
+                  const children: Children = [comp];
                   if (canWrite) {
                     children.push(m("hr"));
                     const _attrs = {
@@ -292,7 +292,13 @@ export const component: ClosureComponent = (): Component => {
 
                   return children;
                 };
-                overlay.open(cb);
+
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
               },
             },
             "Show"
@@ -317,8 +323,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new user",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -347,8 +353,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/virtual-parameters-page.ts

@@ -207,8 +207,8 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
-                  return m(
+                let cb: () => Children = null;
+                const comp = m(
                   putFormComponent,
                   Object.assign(
                     {
@@ -238,8 +238,13 @@ export const component: ClosureComponent = (): Component => {
                     formData
                   )
                 );
-                };
-                overlay.open(cb);
+                cb = () => comp;
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
               },
             },
             "Show"
@@ -255,8 +260,8 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new virtual parameter",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
@@ -285,8 +290,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                  };
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"

ui/wizard-page.ts

@@ -87,15 +87,23 @@ export const component: ClosureComponent = (vnode): Component => {
               })
                 .then(() => {
                   setTimeout(() => {
-                    m.request({ url: "init" }).then((o) => {
+                    m.request({ url: "init" })
+                      .then((o) => {
                         e.target.disabled = false;
                         options = o;
-                      notifications.push("success", "Initialization complete", {
+                        notifications.push(
+                          "success",
+                          "Initialization complete",
+                          {
                             "Open Sesame!": () => {
                               m.route.set("/login");
                               window.location.reload();
                             },
-                      });
+                          }
+                        );
+                      })
+                      .catch((err) => {
+                        notifications.push("error", err.message);
                       });
                   }, 3000);
                   if (opts["users"]) {