Add more upload file types

Because the hostInfo MongoDB command may not be allowed in some
environments. Also, in sharded deployments the clock skew may differ
between different shards so using hostInfo is not enough. This new
implementation simply assumes the clocks are in sync within a certain
range and throws an error if clocks are found to be out of sync.

.eslintrc.json

@@ -21,6 +21,7 @@
     "@typescript-eslint/no-explicit-any": 0,
     "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
     "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
+    "@typescript-eslint/no-floating-promises": "error",
     "require-atomic-updates": 0,
     "no-prototype-builtins": 0
   }

CHANGELOG.md

@@ -1,5 +1,61 @@
 # Change Log
 
+## 1.2.3 (2020-10-26)
+
+- New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
+report writable parameters as non-writable. When set to true, the scripts will
+no longer respect the 'writable' attribute of the CPE parameters and will send
+a SetParamteerValues, AddObject, or DeleteObject request anyway.
+- Tags no longer restrict what characters are allowed. Any character other than
+alphanumeric characters, hyphen, or underscore is now encoded in the data
+model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
+- Ask for a confirmation before closing a pop-up dialog with unsaved changes.
+- Better XML validation to avoid crashes caused by invalid CPE requests.
+- Fix confusing 404 error message when the user attempts to modify a resource
+when they don't have the necessary permissions.
+- Fix a rare issue where genieacs-cwmp stops accepting new connections after
+running for a few weeks.
+- Fix exception when IS NULL operator is used in certain situations.
+
+## 1.2.2 (2020-10-03)
+
+- Added button to push files to selected devices from device listing page.
+- A few minor UI improvements.
+- Fix exception that can happen and persist after a Download request.
+- Fix validation bug preventing running refreshObject task on data model root.
+- Fix invalid arguments fault in refresh preset configuration when upgrading
+from v1.1.
+
+## 1.2.1 (2020-09-08)
+
+- Fix bug causing faults to not be displayed in the UI.
+- Fix bug where deleting objects does not get reflected immediately in the UI.
+- Improve conversion between filters written in the expression format and
+MongoDB queries. There should now be fewer edge cases where the two are not
+equisatisfiable.
+
+## 1.2.0 (2020-09-01)
+
+- Support GetParameterAttributes and SetParameterAttributes TR-069 methods.
+- Support CASE statement and COALESCE function in expressions.
+- Provision arguments can now be a list of expressions that are dynamically
+evaluated.
+- Support Forwarded HTTP header to display in the logs the correct IP of CPEs
+behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
+- Config expressions can now access all available device parameters, not only
+serial number, product class, and OUI.
+- Use relative URLs throughout the UI to allow serving from a subdirectory
+using a reverse proxy.
+- Make Date.parse() and Date.UTC() available to provision scripts.
+- libxmljs has been entirely removed in favor of our bespoke XML parser.
+- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now
+used to determine the TCP connection timeout.
+- The all-parameters component now limits the number of parameters displayed
+for better performance.
+- The process genieacs-cwmp is now much less likely to throw exceptions as a
+result of invalid requests from CPE.
+- A large number of bug fixes and stability improvements.
+
 ## 1.2.0-beta.0 (2019-07-30)
 
 - A brand new UI superseding genieacs-gui.

CONTRIBUTING.md

@@ -1,22 +1,21 @@
 # Have a question?
 
-Please ask in the [Users mailing list][mailing-lists] instead of opening a
-Github Issue. The Github issue tracker is strictly for bug reports.
+Please ask in the [forum] instead of opening a Github Issue. The Github issue
+tracker is strictly for bug reports.
 
 # Issues
 
 We use GitHub issues to track bugs. Please ensure your bug description is
 clear and has sufficient instructions to be able to reproduce the issue.
 
-If you face interoperability issues with your device, more often than not it's
-a device related issue. Please consult the [Users mailing list][mailing-lists]
-before submitting an Issue.
+If you face interoperability issues with your CPE, more often than not it's a
+device related issue. Please consult the [forum] before submitting an Issue.
 
 # Pull Requests
 
 Pull requests are welcome. If you have a fix for a bug, go ahead and create a
-pull request. For other kinds of enhancements, please discuss in the [Users
-mailing list][mailing-lists] first.
+pull request. For other kinds of enhancements, please discuss in the [forum]
+first.
 
 Please follow the guidelines [here][commit-guidelines] when committing your
 changes.
@@ -36,4 +35,4 @@ source code contained therein, and that your content does not infringe upon
 the intellectual property rights of a third party.
 
 [commit-guidelines]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
-[mailing-lists]: http://lists.genieacs.com
+[forum]: https://forum.genieacs.com

README.md

@@ -8,36 +8,31 @@ millions of concurrent devices.
 
 ## Quick Start
 
-*Important: This is a pre-release branch. Use
-[v1.1](https://github.com/genieacs/genieacs/tree/v1.1) for production
-deployments.*
-
 Install [Node.js](http://nodejs.org/) and [MongoDB](http://www.mongodb.org/).
 Refer to their corresponding documentation for installation instructions. The
 supported versions are:
 
-- Node.js: 10.x through 12.x
-- MongoDB: 2.6 through 4.1
+- Node.js: 12.3+
+- MongoDB: 3.6+
 
-Clone this repo or download the source archive then *cd* into the source
-directory and install the required dependencies:
+Install GenieACS from NPM:
+
+    sudo npm install -g --unsafe-perm genieacs
+
+To build from source instead, clone this repo or download the source archive
+then *cd* into the source directory then run:
 
     npm install
-
-Now build:
-
     npm run build
 
-Finally, run the following services from the build that's generated under
-'dist' directory:
+Finally, run the following services (found under `./dist/bin/` if building from
+source):
 
 ### genieacs-cwmp
 
 This is the service that the CPEs will communicate with. It listens on port
 7547 by default. Configure the ACS URL in your devices accordingly.
 
-    ./dist/bin/genieacs-cwmp
-
 You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
 as a dummy TR-069 simulator if you don't have a CPE at hand.
 
@@ -47,40 +42,36 @@ This is the northbound interface module. It exposes a REST API on port 7557 by
 default. This one is only required if you have an external system integrating
 with GenieACS using this API.
 
-    ./dist/bin/genieacs-nbi
-
 ### genieacs-fs
 
 This is the file server from which the CPEs will download firmware images and
-such.
-
-    ./dist/bin/genieacs-fs
+such. It listens on port 7567 by default.
 
 ### genieacs-ui
 
 This serves the web based user interface. It listens on port 3000 by default.
+You must pass *--ui-jwt-secret* argument to supply the secret key used for
+signing browser cookies:
 
-    ./dist/bin/genieacs-ui --ui-jwt-secret secret
-
-The argument *--ui-jwt-secret* supplies the key used for signing browser
-cookies.
+    genieacs-ui --ui-jwt-secret secret
 
 The UI has plenty of configuration options. When you open GenieACS's UI in a
 browser you'll be greeted with a database initialization wizard to help you
 populate some initial configuration.
 
+Visit [docs.genieacs.com](https://docs.genieacs.com) for more documentation and
+a complete installation guide for production deployments.
+
 ## Support
 
 The [forum](https://forum.genieacs.com) is a good place to get guidance and
-help from the community. Head on over and join the conversation! In addition,
-the [wiki](https://github.com/genieacs/genieacs/wiki) provides useful
-documentation and tips from GenieACS users.
+help from the community. Head on over and join the conversation!
 
-For commercial support options and professional services, please visit
+For commercial support options, please visit
 [genieacs.com](https://genieacs.com/support/).
 
 ## License
 
-Copyright 2013-2019 GenieACS Inc. GenieACS is released under the [AGPLv3
+Copyright 2013-2020 GenieACS Inc. GenieACS is released under the [AGPLv3
 license
 terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

ava.config.js

@@ -1,5 +1,4 @@
 export default {
-  "compileEnhancements": false,
   "extensions": ["ts"],
   "require": ["esm", "ts-node/register/transpile-only"]
 };

bin/genieacs-cwmp.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -26,37 +24,35 @@ import * as server from "../lib/server";
 import * as cwmp from "../lib/cwmp";
 import * as db from "../lib/db";
 import * as extensions from "../lib/extensions";
-import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("cwmp", VERSION);
 
-const SERVICE_ADDRESS = config.get("CWMP_INTERFACE");
-const SERVICE_PORT = config.get("CWMP_PORT");
+const SERVICE_ADDRESS = config.get("CWMP_INTERFACE") as string;
+const SERVICE_PORT = config.get("CWMP_PORT") as number;
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
-    cache.disconnect(),
     extensions.killAll(),
-    cluster.worker.disconnect()
+    cluster.worker.disconnect(),
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES");
+  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES") as number;
 
   logger.info({
     message: `genieacs-cwmp starting`,
     pid: process.pid,
-    version: VERSION
+    version: VERSION,
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -64,7 +60,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
@@ -73,42 +69,40 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("CWMP_SSL_KEY"),
-    cert: config.get("CWMP_SSL_CERT")
+  const key = config.get("CWMP_SSL_KEY") as string;
+  const cert = config.get("CWMP_SSL_CERT") as string;
+
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    onConnection: cwmp.onConnection,
+    timeout: 30000,
+    keepAliveTimeout: 0,
   };
 
-  process.on("uncaughtException", err => {
+  process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid
+      pid: process.pid,
     });
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
-  const initPromise = Promise.all([db.connect(), cache.connect()])
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(
-        SERVICE_PORT,
-        SERVICE_ADDRESS,
-        ssl,
-        cwmp.listener,
-        cwmp.onConnection,
-        0
-      );
+      server.start(options, cwmp.listener);
     })
-    .catch(err => {
+    .catch((err) => {
       setTimeout(() => {
         throw err;
       });
@@ -116,19 +110,13 @@ if (!cluster.worker) {
 
   process.on("SIGINT", () => {
     initPromise.finally(() => {
-      server
-        .stop()
-        .then(exitWorkerGracefully)
-        .catch(exitWorkerUngracefully);
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
     initPromise.finally(() => {
-      server
-        .stop()
-        .then(exitWorkerGracefully)
-        .catch(exitWorkerUngracefully);
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 }

bin/genieacs-ext.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -35,8 +33,8 @@ function errorToFault(err: Error): Fault {
     message: err.message,
     detail: {
       name: err.name,
-      message: err.message
-    }
+      message: err.message,
+    },
   };
 
   if (err.stack) {
@@ -56,16 +54,16 @@ function errorToFault(err: Error): Fault {
   return fault;
 }
 
-process.on("uncaughtException", err => {
+process.on("uncaughtException", (err) => {
   const fault = errorToFault(err);
-  jobs.forEach(jobId => {
+  jobs.forEach((jobId) => {
     process.send([jobId, fault, null]);
   });
   jobs.clear();
   process.disconnect();
 });
 
-process.on("message", message => {
+process.on("message", (message) => {
   jobs.add(message[0]);
 
   if (!script) {
@@ -79,7 +77,7 @@ process.on("message", message => {
   if (!script[funcName]) {
     const fault = {
       code: "ext",
-      message: `No such function '${funcName}' in extension '${fileName}'`
+      message: `No such function '${funcName}' in extension '${fileName}'`,
     };
     process.send([message[0], fault, null]);
     return;
@@ -93,4 +91,6 @@ process.on("message", message => {
 });
 
 // Ignore SIGINT
-process.on("SIGINT", () => {});
+process.on("SIGINT", () => {
+  // Ignore
+});

bin/genieacs-fs.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -25,21 +23,18 @@ import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import { listener } from "../lib/fs";
 import * as db from "../lib/db";
-import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("fs", VERSION);
 
-const SERVICE_ADDRESS = config.get("FS_INTERFACE");
-const SERVICE_PORT = config.get("FS_PORT");
+const SERVICE_ADDRESS = config.get("FS_INTERFACE") as string;
+const SERVICE_PORT = config.get("FS_PORT") as number;
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
-  Promise.all([
-    db.disconnect(),
-    cache.disconnect(),
-    cluster.worker.disconnect()
-  ]).catch(exitWorkerUngracefully);
+  Promise.all([db.disconnect(), cluster.worker.disconnect()]).catch(
+    exitWorkerUngracefully
+  );
 }
 
 function exitWorkerUngracefully(): void {
@@ -47,12 +42,12 @@ function exitWorkerUngracefully(): void {
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES");
+  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES") as number;
 
   logger.info({
     message: `genieacs-fs starting`,
     pid: process.pid,
-    version: VERSION
+    version: VERSION,
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -60,7 +55,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
@@ -69,55 +64,63 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("FS_SSL_KEY"),
-    cert: config.get("FS_SSL_CERT")
+  const key = config.get("FS_SSL_KEY") as string;
+  const cert = config.get("FS_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
-  process.on("uncaughtException", err => {
+  let stopping = false;
+
+  process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid
+      pid: process.pid,
     });
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
+    stopping = true;
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
-  const initPromise = Promise.all([db.connect(), cache.connect()])
+  const _listener = (req, res): void => {
+    if (stopping) res.setHeader("Connection", "close");
+    listener(req, res).catch((err) => {
+      throw err;
+    });
+  };
+
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
+      server.start(options, _listener);
     })
-    .catch(err => {
+    .catch((err) => {
       setTimeout(() => {
         throw err;
       });
     });
 
   process.on("SIGINT", () => {
+    stopping = true;
     initPromise.finally(() => {
-      server
-        .stop()
-        .then(exitWorkerGracefully)
-        .catch(exitWorkerUngracefully);
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
+    stopping = true;
     initPromise.finally(() => {
-      server
-        .stop()
-        .then(exitWorkerGracefully)
-        .catch(exitWorkerUngracefully);
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 }

bin/genieacs-nbi.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -26,37 +24,35 @@ import * as server from "../lib/server";
 import { listener } from "../lib/nbi";
 import * as db from "../lib/db";
 import * as extensions from "../lib/extensions";
-import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("nbi", VERSION);
 
-const SERVICE_ADDRESS = config.get("NBI_INTERFACE");
-const SERVICE_PORT = config.get("NBI_PORT");
+const SERVICE_ADDRESS = config.get("NBI_INTERFACE") as string;
+const SERVICE_PORT = config.get("NBI_PORT") as number;
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
-    cache.disconnect(),
     extensions.killAll(),
-    cluster.worker.disconnect()
+    cluster.worker.disconnect(),
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES");
+  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES") as number;
 
   logger.info({
     message: `genieacs-nbi starting`,
     pid: process.pid,
-    version: VERSION
+    version: VERSION,
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -64,7 +60,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
@@ -73,55 +69,61 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("NBI_SSL_KEY"),
-    cert: config.get("NBI_SSL_CERT")
+  const key = config.get("NBI_SSL_KEY") as string;
+  const cert = config.get("NBI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
-  process.on("uncaughtException", err => {
+  let stopping = false;
+
+  process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid
+      pid: process.pid,
     });
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
+    stopping = true;
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
-  const initPromise = Promise.all([db.connect(), cache.connect()])
+  const _listener = (req, res): void => {
+    if (stopping) res.setHeader("Connection", "close");
+    listener(req, res);
+  };
+
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
+      server.start(options, _listener);
     })
-    .catch(err => {
+    .catch((err) => {
       setTimeout(() => {
         throw err;
       });
     });
 
   process.on("SIGINT", () => {
+    stopping = true;
     initPromise.finally(() => {
-      server
-        .stop()
-        .then(exitWorkerGracefully)
-        .catch(exitWorkerUngracefully);
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
+    stopping = true;
     initPromise.finally(() => {
-      server
-        .stop()
-        .then(exitWorkerGracefully)
-        .catch(exitWorkerUngracefully);
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 }

bin/genieacs-ui.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -25,40 +23,36 @@ import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import { listener } from "../lib/ui";
 import * as extensions from "../lib/extensions";
-import * as db from "../lib/ui/db";
-import * as db2 from "../lib/db";
-import * as cache from "../lib/cache";
+import * as db from "../lib/db";
 import { version as VERSION } from "../package.json";
 
 logger.init("ui", VERSION);
 
-const SERVICE_ADDRESS = config.get("UI_INTERFACE");
-const SERVICE_PORT = config.get("UI_PORT");
+const SERVICE_ADDRESS = config.get("UI_INTERFACE") as string;
+const SERVICE_PORT = config.get("UI_PORT") as number;
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
-    db2.disconnect(),
-    cache.disconnect(),
     extensions.killAll(),
-    cluster.worker.disconnect()
+    cluster.worker.disconnect(),
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES");
+  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES") as number;
 
   logger.info({
     message: `genieacs-ui starting`,
     pid: process.pid,
-    version: VERSION
+    version: VERSION,
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -66,7 +60,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
@@ -75,31 +69,32 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
   });
 } else {
-  const ssl = {
-    key: config.get("UI_SSL_KEY"),
-    cert: config.get("UI_SSL_CERT")
+  const key = config.get("UI_SSL_KEY") as string;
+  const cert = config.get("UI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
   let stopping = false;
 
-  process.on("uncaughtException", err => {
+  process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid
+      pid: process.pid,
     });
     stopping = true;
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
   const _listener = (req, res): void => {
@@ -107,11 +102,12 @@ if (!cluster.worker) {
     listener(req, res);
   };
 
-  const initPromise = Promise.all([db2.connect(), cache.connect()])
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+      server.start(options, _listener);
     })
-    .catch(err => {
+    .catch((err) => {
       setTimeout(() => {
         throw err;
       });
@@ -120,20 +116,14 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     stopping = true;
     initPromise.finally(() => {
-      server
-        .stop()
-        .then(exitWorkerGracefully)
-        .catch(exitWorkerUngracefully);
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
     stopping = true;
     initPromise.finally(() => {
-      server
-        .stop()
-        .then(exitWorkerGracefully)
-        .catch(exitWorkerUngracefully);
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 }

build/build.ts

@@ -19,13 +19,13 @@
 
 import * as path from "path";
 import * as fs from "fs";
-import { promisify } from "util";
 import { rollup, WarningHandler } from "rollup";
-import rollupReplace from "rollup-plugin-replace";
-import rollupJson from "rollup-plugin-json";
-import typescript from "rollup-plugin-typescript";
+import rollupJson from "@rollup/plugin-json";
+import typescript from "@rollup/plugin-typescript";
 import { terser } from "rollup-plugin-terser";
-import webpack from "webpack";
+import nodeResolve from "@rollup/plugin-node-resolve";
+import commonjs from "@rollup/plugin-commonjs";
+import webpack, { Configuration } from "webpack";
 import postcss from "postcss";
 import postcssImport from "postcss-import";
 import postcssPresetEnv from "postcss-preset-env";
@@ -55,7 +55,6 @@ const externals = [
   "crypto",
   "util",
   "mongodb",
-  "libxmljs",
   "vm",
   "later",
   "parsimmon",
@@ -78,7 +77,10 @@ const externals = [
   "yaml",
   "codemirror",
   "codemirror/mode/javascript/javascript",
-  "codemirror/mode/yaml/yaml"
+  "codemirror/mode/yaml/yaml",
+  "ipaddr.js",
+  "jsbi",
+  "espresso-iisojs",
 ];
 
 function rmDirSync(dirPath): void {
@@ -123,9 +125,7 @@ function generateSymbol(id: string, svgStr: string): string {
     }
   }
   const symbolBody = xml.children[0].children
-    .map(c => {
-      return xmlTostring(c);
-    })
+    .map((c) => xmlTostring(c))
     .join("");
   return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
 }
@@ -147,7 +147,7 @@ async function init(): Promise<void> {
   delete packageJson["devDependencies"];
   packageJson["scripts"] = {
     install: packageJson["scripts"].install,
-    configure: packageJson["scripts"].configure
+    configure: packageJson["scripts"].configure,
   };
   packageJson["version"] = `${packageJson["version"]}+${BUILD_METADATA}`;
   fs.writeFileSync(
@@ -173,7 +173,7 @@ async function copyStatic(): Promise<void> {
     "README.md",
     "CHANGELOG.md",
     "public/logo.svg",
-    "public/favicon.png"
+    "public/favicon.png",
   ];
 
   for (const file of files) {
@@ -194,35 +194,31 @@ async function generateCss(): Promise<void> {
       stage: 3,
       features: {
         "nesting-rules": true,
-        "color-mod-function": true
-      }
+        "color-mod-function": true,
+      },
     }),
-    cssnano
+    cssnano,
   ]).process(cssIn, { from: cssInPath, to: cssOutPath });
   fs.writeFileSync(cssOutPath, cssOut.css);
 }
 
 async function generateToolsJs(): Promise<void> {
   for (const bin of ["dump-data-model"]) {
-    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}`);
+    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}.ts`);
     const outputFile = path.resolve(OUTPUT_DIR, `tools/${bin}`);
     const bundle = await rollup({
       input: inputFile,
       external: externals,
       acorn: {
-        allowHashBang: true
+        allowHashBang: true,
       },
       plugins: [
-        rollupReplace({
-          delimiters: ["", ""],
-          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
-        }),
         typescript({
           tsconfig: "./tsconfig.json",
-          include: [`tools/${bin}`, "lib/**/*.ts"]
+          include: [`tools/${bin}.ts`, "lib/**/*.ts"],
         }),
-        MODE === "production" ? terser() : null
-      ]
+        MODE === "production" ? terser() : null,
+      ],
     });
 
     await bundle.write({
@@ -231,7 +227,7 @@ async function generateToolsJs(): Promise<void> {
       sourcemap: "inline",
       sourcemapExcludeSources: true,
       banner: "#!/usr/bin/env node",
-      file: outputFile
+      file: outputFile,
     });
 
     // Mark as executable
@@ -246,27 +242,24 @@ async function generateBackendJs(): Promise<void> {
     "genieacs-ext",
     "genieacs-nbi",
     "genieacs-fs",
-    "genieacs-ui"
+    "genieacs-ui",
   ]) {
-    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}`);
+    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}.ts`);
     const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
     const bundle = await rollup({
       input: inputFile,
       external: externals,
       acorn: {
-        allowHashBang: true
+        allowHashBang: true,
       },
       treeshake: {
         propertyReadSideEffects: false,
-        pureExternalModules: true
+        moduleSideEffects: false,
       },
       plugins: [
-        rollupReplace({
-          delimiters: ["", ""],
-          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
-        }),
         rollupJson({ preferConst: true }),
         {
+          name: "",
           resolveId: (importee, importer) => {
             if (importee.endsWith("/package.json")) {
               const p = path.resolve(path.dirname(importer), importee);
@@ -274,14 +267,14 @@ async function generateBackendJs(): Promise<void> {
                 return path.resolve(OUTPUT_DIR, "package.json");
             }
             return null;
-          }
+          },
         },
         typescript({
           tsconfig: "./tsconfig.json",
-          include: [`bin/${bin}`, "lib/**/*.ts"]
+          include: [`bin/${bin}.ts`, "lib/**/*.ts"],
         }),
-        MODE === "production" ? terser() : null
-      ]
+        MODE === "production" ? terser() : null,
+      ],
     });
 
     await bundle.write({
@@ -290,7 +283,7 @@ async function generateBackendJs(): Promise<void> {
       sourcemap: "inline",
       sourcemapExcludeSources: true,
       banner: "#!/usr/bin/env node",
-      file: outputFile
+      file: outputFile,
     });
 
     // Mark as executable
@@ -303,37 +296,54 @@ async function generateFrontendJs(): Promise<void> {
   const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
   const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
 
+  const inlineDeps = ["mithril", "parsimmon", "jsbi", "espresso-iisojs"];
   const bundle = await rollup({
     input: inputFile,
-    external: externals,
+    external: externals.filter((e) => !inlineDeps.includes(e)),
     plugins: [
       rollupJson({ preferConst: true }),
-      typescript({ tsconfig: "./tsconfig.json" })
+      {
+        name: "",
+        resolveId: function (importee, importer) {
+          if (importee.endsWith("/bigint")) {
+            return this.resolve(importee + "-jsbi", importer);
+          } else if (importee === "espresso-iisojs") {
+            return this.resolve(
+              "espresso-iisojs/dist/espresso-iisojs-jsbi.mjs",
+              importer
+            );
+          }
+          return null;
+        },
+      },
+      typescript({ tsconfig: "./tsconfig.json" }),
+      nodeResolve(),
+      commonjs(),
     ],
     inlineDynamicImports: true,
     treeshake: {
       propertyReadSideEffects: false,
-      pureExternalModules: true
+      moduleSideEffects: false,
     },
     onwarn: ((warning, warn) => {
       // Ignore circular dependency warnings
       if (warning.code !== "CIRCULAR_DEPENDENCY") warn(warning);
-    }) as WarningHandler
+    }) as WarningHandler,
   });
 
   await bundle.write({
     preferConst: true,
-    format: "esm",
+    format: "es",
     sourcemap: "inline",
     sourcemapExcludeSources: true,
-    file: outputFile
+    file: outputFile,
   });
 
-  const webpackConf = {
-    mode: MODE,
+  const webpackConf: Configuration = {
+    mode: MODE as "development" | "production",
     entry: outputFile,
     resolve: {
-      aliasFields: ["module"]
+      aliasFields: ["module"],
     },
     devtool: "nosources-source-map",
     module: {
@@ -341,18 +351,23 @@ async function generateFrontendJs(): Promise<void> {
         {
           test: /\.js$/,
           use: ["source-map-loader"],
-          enforce: "pre"
-        }
-      ]
+          enforce: "pre",
+        },
+      ],
     },
     output: {
       path: path.resolve(OUTPUT_DIR, "public"),
-      filename: "app.js"
-    }
+      filename: "app.js",
+    },
   };
 
-  const stats = await promisify(webpack)(webpackConf);
-  process.stdout.write(stats.toString({ colors: true }) + "\n");
+  return new Promise((resolve, reject) => {
+    webpack(webpackConf, (err, stats) => {
+      if (err) return reject(err);
+      process.stdout.write(stats.toString({ colors: true }) + "\n");
+      resolve();
+    });
+  });
 }
 
 async function generateIconsSprite(): Promise<void> {
@@ -379,13 +394,15 @@ init()
       generateIconsSprite(),
       generateToolsJs(),
       generateBackendJs(),
-      generateFrontendJs()
+      generateFrontendJs(),
     ])
-      .then(() => {})
-      .catch(err => {
+      .then(() => {
+        // Ignore
+      })
+      .catch((err) => {
         process.stderr.write(err.stack + "\n");
       });
   })
-  .catch(err => {
+  .catch((err) => {
     process.stderr.write(err.stack + "\n");
   });

docs/environment-variables.rst

@@ -164,12 +164,13 @@ FS_ACCESS_LOG_FILE
 
   Default: unset
 
-FS_HOSTNAME
-  The hostname to use when generating the file URL for TR-069 Download
-  requests. Set this if the hostname where genieacs-fs is listening differs
-  from the hostname in the ACS URL.
+FS_URL_PREFIX
+  The URL prefix (e.g. 'https://example.com:7657/') to use when generating the
+  file URL for TR-069 Download requests. Set this if genieacs-fs and
+  genieacs-cwmp are behind a proxy or running on different servers.
 
-  Default: the hostname from the ACS URL
+  Default: auto generated based on the hostname from the ACS URL, FS_PORT
+  config, and whether or not SSL is enabled for genieacs-fs.
 
 UI_WORKER_PROCESSES
   The number of worker processes to spawn for genieacs-ui. A value of 0 means

docs/virtual-parameters.rst

@@ -18,8 +18,8 @@ differences between the scripts of provisions and virtual parameters are:
 
 .. code:: javascript
 
-  // [<current attr timestamps>, <current attr values>, <declared attr timestamps, declared attr values>]
-  [{path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}, {path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}]
+  // [<declared attr timestamps, declared attr values>, <current attr timestamps>, <current attr values>]
+  [{path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}, {path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}]
 
 - Virtual parameter scripts must return an object containing the attributes of
   this parameter.
@@ -69,9 +69,9 @@ Expose an external value as a virtual parameter
 
   // Example: Expose an external value as a virtual parameter
   let serial = declare("DeviceID.SerialNumber", {value: 1});
-  if (values.value) {
-    ext("example-ext", "set", serial.value[0], values.value[0]);
-    return {writable: true, value: [values.value[0], "xsd:string"]};
+  if (args[1].value) {
+    ext("example-ext", "set", serial.value[0], args[1].value[0]);
+    return {writable: true, value: [args[1].value[0], "xsd:string"]};
   }
   else {
     let v = ext("example-ext", "get", serial.value[0]);
@@ -85,8 +85,8 @@ Create an editable virtual parameter for WPA passphrase
 
   // Example: Create an editable virtual parameter for WPA passphrase
   let m = "";
-  if (VALUES && VALUES.value) {
-    m = VALUES.value[0];
+  if (args[1].value) {
+    m = args[1].value[0];
     declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", null, {value: m});
     declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", null, {value: m});
   }

lib/api-functions.ts

@@ -24,64 +24,84 @@ import * as cache from "./cache";
 import {
   getCurrentSnapshot,
   getConfig,
-  getConfigExpression
+  getConfigExpression,
 } from "./local-cache";
 import {
   httpConnectionRequest,
-  udpConnectionRequest
+  udpConnectionRequest,
 } from "./connection-request";
 import { Expression, Task } from "./types";
+import { flattenDevice } from "./mongodb-functions";
+import { evaluate } from "./common/expression";
 
-export async function connectionRequest(deviceId): Promise<void> {
-  const options = {
-    projection: {
-      _deviceId: 1,
-      "Device.ManagementServer.ConnectionRequestURL._value": 1,
-      "Device.ManagementServer.UDPConnectionRequestAddress._value": 1,
-      "Device.ManagementServer.ConnectionRequestUsername._value": 1,
-      "Device.ManagementServer.ConnectionRequestPassword._value": 1,
-      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value": 1,
-      "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress._value": 1,
-      "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername._value": 1,
-      "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword._value": 1
-    }
-  };
-
-  const device = await db.devicesCollection.findOne({ _id: deviceId }, options);
-  if (!device) throw new Error("No such device");
-
-  let managementServer,
-    connectionRequestUrl,
-    udpConnectionRequestAddress,
-    username,
-    password;
-  if (device.Device)
-    // TR-181 data model
-    managementServer = device.Device.ManagementServer;
-  // TR-098 data model
-  else managementServer = device.InternetGatewayDevice.ManagementServer;
-
-  if (managementServer.ConnectionRequestURL)
-    connectionRequestUrl = managementServer.ConnectionRequestURL._value;
-  if (managementServer.UDPConnectionRequestAddress) {
-    udpConnectionRequestAddress =
-      managementServer.UDPConnectionRequestAddress._value;
+export async function connectionRequest(
+  deviceId: string,
+  device?: Record<string, { value?: [boolean | number | string, string] }>
+): Promise<string> {
+  if (!device) {
+    const res = await db.devicesCollection.findOne({ _id: deviceId });
+    if (!res) throw new Error("No such device");
+    device = flattenDevice(res);
   }
-  if (managementServer.ConnectionRequestUsername)
-    username = managementServer.ConnectionRequestUsername._value;
-  if (managementServer.ConnectionRequestPassword)
-    password = managementServer.ConnectionRequestPassword._value;
 
-  const context = {
-    id: device["_id"],
-    serialNumber: device["_deviceId"]["SerialNumber"],
-    productClass: device["_deviceId"]["ProductClass"],
-    oui: device["_deviceId"]["OUI"],
-    remoteAddress: connectionRequestUrl
-      ? parse(connectionRequestUrl).host
-      : null,
-    username: username || "",
-    password: password || ""
+  let connectionRequestUrl, udpConnectionRequestAddress, username, password;
+
+  if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
+    connectionRequestUrl = (device[
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL"
+    ].value || [""])[0];
+    udpConnectionRequestAddress = ((
+      device[
+        "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
+      ] || {}
+    ).value || [""])[0];
+    username = ((
+      device[
+        "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
+      ] || {}
+    ).value || [""])[0];
+    password = ((
+      device[
+        "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword"
+      ] || {}
+    ).value || [""])[0];
+  } else {
+    connectionRequestUrl = (device[
+      "Device.ManagementServer.ConnectionRequestURL"
+    ].value || [""])[0];
+    udpConnectionRequestAddress = ((
+      device["Device.ManagementServer.UDPConnectionRequestAddress"] || {}
+    ).value || [""])[0];
+    username = ((
+      device["Device.ManagementServer.ConnectionRequestUsername"] || {}
+    ).value || [""])[0];
+    password = ((
+      device["Device.ManagementServer.ConnectionRequestPassword"] || {}
+    ).value || [""])[0];
+  }
+
+  const remoteAddress = parse(connectionRequestUrl).host;
+
+  const evalCallback = (exp): Expression => {
+    if (!Array.isArray(exp)) return exp;
+    if (exp[0] === "PARAM" && typeof exp[1] === "string") {
+      let name = exp[1];
+      if (name === "id") name = "DeviceID.ID";
+      else if (name === "serialNumber") name = "DeviceID.SerialNumber";
+      else if (name === "productClass") name = "DeviceID.ProductClass";
+      else if (name === "oui") name = "DeviceID.OUI";
+      else if (name === "remoteAddress") return remoteAddress;
+      else if (name === "username") return username;
+      else if (name === "password") return password;
+
+      const p = device[name];
+      if (p && p.value) return p.value[0];
+    } else if (exp[0] === "FUNC") {
+      if (exp[1] === "REMOTE_ADDRESS") return remoteAddress;
+      else if (exp[1] === "USERNAME") return username;
+      else if (exp[1] === "PASSWORD") return password;
+    }
+    return exp;
   };
 
   const snapshot = await getCurrentSnapshot();
@@ -89,20 +109,23 @@ export async function connectionRequest(deviceId): Promise<void> {
   const UDP_CONNECTION_REQUEST_PORT = +getConfig(
     snapshot,
     "cwmp.udpConnectionRequestPort",
-    context,
-    now
+    {},
+    now,
+    evalCallback
   );
   const CONNECTION_REQUEST_TIMEOUT = +getConfig(
     snapshot,
     "cwmp.connectionRequestTimeout",
-    context,
-    now
+    {},
+    now,
+    evalCallback
   );
   const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = !!getConfig(
     snapshot,
     "cwmp.connectionRequestAllowBasicAuth",
-    context,
-    now
+    {},
+    now,
+    evalCallback
   );
   let authExp: Expression = getConfigExpression(
     snapshot,
@@ -114,64 +137,79 @@ export async function connectionRequest(deviceId): Promise<void> {
       "FUNC",
       "AUTH",
       ["PARAM", "username"],
-      ["PARAM", "password"]
+      ["PARAM", "password"],
     ] as Expression;
   }
 
-  const debug = !!getConfig(snapshot, "cwmp.debug", context, now);
+  authExp = evaluate(authExp, {}, now, evalCallback);
+
+  const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
 
   let udpProm;
   if (udpConnectionRequestAddress) {
     udpProm = udpConnectionRequest(
       udpConnectionRequestAddress,
       authExp,
-      context,
       UDP_CONNECTION_REQUEST_PORT,
       debug,
       deviceId
     );
   }
 
-  try {
-    await httpConnectionRequest(
-      connectionRequestUrl,
-      authExp,
-      context,
-      CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
-      CONNECTION_REQUEST_TIMEOUT,
-      debug,
-      deviceId
-    );
-  } catch (err) {
-    if (!udpProm) throw err;
+  const status = await httpConnectionRequest(
+    connectionRequestUrl,
+    authExp,
+    CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
+    CONNECTION_REQUEST_TIMEOUT,
+    debug,
+    deviceId
+  );
+
+  if (udpProm) {
     await udpProm;
+    return "";
   }
+
+  return status;
 }
 
-export async function watchTask(deviceId, taskId, timeout): Promise<string> {
-  await new Promise(resolve => setTimeout(resolve, 500));
-
-  const task = await db.tasksCollection.findOne(
-    { _id: taskId },
-    { projection: { _id: 1 } }
+export async function awaitSessionStart(
+  deviceId: string,
+  lastInform: number,
+  timeout: number
+): Promise<boolean> {
+  const now = Date.now();
+  const device = await db.devicesCollection.findOne(
+    { _id: deviceId },
+    { projection: { _lastInform: 1 } }
   );
-  if (!task) return "completed";
+  const li = (device["_lastInform"] as Date).getTime();
+  if (li > lastInform) return true;
+  const token = await cache.get(`cwmp_session_${deviceId}`);
+  if (token) return true;
+  if (timeout < 500) return false;
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionStart(deviceId, lastInform, timeout);
+}
 
-  const q = { _id: `${deviceId}:task_${taskId}` };
-  const fault = await db.faultsCollection.findOne(q, {
-    projection: { _id: 1 }
-  });
-  if (fault) return "fault";
-
-  if ((timeout -= 500) <= 0) return "timeout";
-
-  return watchTask(deviceId, taskId, timeout);
+export async function awaitSessionEnd(
+  deviceId: string,
+  timeout: number
+): Promise<boolean> {
+  const now = Date.now();
+  const token = await cache.get(`cwmp_session_${deviceId}`);
+  if (!token) return true;
+  if (timeout < 500) return false;
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionEnd(deviceId, timeout);
 }
 
 function sanitizeTask(task): void {
   task.timestamp = new Date(task.timestamp || Date.now());
   if (task.expiry) {
-    if (common.typeOf(task.expiry) === common.DATE_TYPE || isNaN(task.expiry))
+    if (task.expiry instanceof Date || isNaN(task.expiry))
       task.expiry = new Date(task.expiry);
     else task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000);
   }
@@ -209,6 +247,10 @@ function sanitizeTask(task): void {
       break;
 
     case "refreshObject":
+      if (typeof task.objectName !== "string")
+        throw new Error("Missing 'objectName' property");
+      break;
+
     case "deleteObject":
       if (typeof task.objectName !== "string" || !task.objectName.length)
         throw new Error("Missing 'objectName' property");
@@ -245,9 +287,10 @@ function sanitizeTask(task): void {
     case "provisions":
       if (
         !Array.isArray(task.provisions) ||
-        !task.provisions.every(arr =>
+        !task.provisions.every((arr) =>
           arr.every(
-            s => s == null || ["boolean", "number", "string"].includes(typeof s)
+            (s) =>
+              s == null || ["boolean", "number", "string"].includes(typeof s)
           )
         )
       )
@@ -258,8 +301,8 @@ function sanitizeTask(task): void {
   return task;
 }
 
-export async function insertTasks(tasks): Promise<Task[]> {
-  if (tasks && common.typeOf(tasks) !== common.ARRAY_TYPE) tasks = [tasks];
+export async function insertTasks(tasks: any[]): Promise<Task[]> {
+  if (tasks && !Array.isArray(tasks)) tasks = [tasks];
   else if (!tasks || tasks.length === 0) return tasks || [];
 
   for (const task of tasks) {
@@ -267,7 +310,7 @@ export async function insertTasks(tasks): Promise<Task[]> {
     if (task.uniqueKey) {
       await db.tasksCollection.deleteOne({
         device: task.device,
-        uniqueKey: task.uniqueKey
+        uniqueKey: task.uniqueKey,
       });
     }
   }
@@ -275,20 +318,21 @@ export async function insertTasks(tasks): Promise<Task[]> {
   return tasks;
 }
 
-export async function deleteDevice(deviceId): Promise<void> {
+export async function deleteDevice(deviceId: string): Promise<void> {
   await Promise.all([
     db.tasksCollection.deleteMany({ device: deviceId }),
     db.devicesCollection.deleteOne({ _id: deviceId }),
     db.faultsCollection.deleteMany({
       _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`
-      }
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+      },
     }),
     db.operationsCollection.deleteMany({
       _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`
-      }
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+      },
     }),
-    cache.del(`${deviceId}_tasks_faults_operations`)
+    cache.del(`${deviceId}_tasks_faults_operations`),
+    db.deleteDeviceUploads(deviceId),
   ]);
 }

lib/auth.ts

@@ -19,15 +19,17 @@
 
 import { createHash, randomBytes, pbkdf2 } from "crypto";
 
-function parseHeaderFeilds(str: string): {} {
+function parseHeaderFeilds(str: string): Record<string, string> {
   const res = {};
   const parts = str.split(",");
 
-  let part;
+  let part: string;
   while ((part = parts.shift()) != null) {
     const name = part.split("=", 1)[0];
-    if (name.length === part.length)
+    if (name.length === part.length) {
+      if (!part.trim()) continue;
       throw new Error("Unable to parse auth header");
+    }
 
     let value = part.slice(name.length + 1);
     if (!/^\s*"/.test(value)) {
@@ -50,7 +52,9 @@ function parseHeaderFeilds(str: string): {} {
   return res;
 }
 
-export function parseAuthorizationHeader(authHeader): { method: string } {
+export function parseAuthorizationHeader(
+  authHeader: string
+): { method: string } {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };
@@ -72,7 +76,9 @@ export function parseAuthorizationHeader(authHeader): { method: string } {
   return res;
 }
 
-export function parseWwwAuthenticateHeader(authHeader): {} {
+export function parseWwwAuthenticateHeader(
+  authHeader: string
+): Record<string, string> {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };
@@ -80,7 +86,7 @@ export function parseWwwAuthenticateHeader(authHeader): {} {
   return res;
 }
 
-export function basic(username, password): string {
+export function basic(username: string, password: string): string {
   return "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
 }
 
@@ -97,20 +103,12 @@ export function digest(
   nc?: string | Buffer
 ): string {
   const ha1 = createHash("md5");
-  ha1
-    .update(username)
-    .update(":")
-    .update(realm)
-    .update(":")
-    .update(password);
+  ha1.update(username).update(":").update(realm).update(":").update(password);
   // TODO support "MD5-sess" algorithm directive
   const ha1d = ha1.digest("hex");
 
   const ha2 = createHash("md5");
-  ha2
-    .update(httpMethod)
-    .update(":")
-    .update(uri);
+  ha2.update(httpMethod).update(":").update(uri);
 
   if (qop === "auth-int") {
     const bodyHash = createHash("md5")
@@ -122,10 +120,7 @@ export function digest(
   const ha2d = ha2.digest("hex");
 
   const hash = createHash("md5");
-  hash
-    .update(ha1d)
-    .update(":")
-    .update(nonce);
+  hash.update(ha1d).update(":").update(nonce);
   if (qop) {
     hash
       .update(":")
@@ -146,7 +141,7 @@ export function solveDigest(
   uri: string | Buffer,
   httpMethod: string | Buffer,
   body: string | Buffer,
-  authHeader
+  authHeader: Record<string, string>
 ): string {
   const cnonce = randomBytes(8).toString("hex");
   const nc = "00000001";
@@ -183,7 +178,7 @@ export function solveDigest(
   return authString;
 }
 
-export function generateSalt(length): Promise<string> {
+export function generateSalt(length: number): Promise<string> {
   return new Promise((resolve, reject) => {
     randomBytes(length, (err, rand) => {
       if (err) return void reject(err);
@@ -192,7 +187,7 @@ export function generateSalt(length): Promise<string> {
   });
 }
 
-export function hashPassword(pass, salt): Promise<string> {
+export function hashPassword(pass: string, salt: string): Promise<string> {
   return new Promise((resolve, reject) => {
     pbkdf2(pass, salt, 10000, 128, "sha512", (err, hash) => {
       if (err) return void reject(err);

lib/cache.ts

@@ -17,124 +17,102 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, Collection } from "mongodb";
+import { Collection } from "mongodb";
+import { onConnect } from "./db";
 import * as config from "./config";
 
+const CLOCK_SKEW_TOLERANCE = 30000;
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let clientPromise: Promise<MongoClient>;
-let mongoCollection: Collection;
-let mongoTimeOffset = 0;
+let cacheCollection: Collection;
 
-export async function connect(): Promise<void> {
-  const MONGODB_CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
-  clientPromise = MongoClient.connect(MONGODB_CONNECTION_URL, {
-    useNewUrlParser: true
-  });
-  const db = (await clientPromise).db();
-  mongoCollection = db.collection("cache");
-  await mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
-  const now = Date.now();
-  const res = await db.command({ hostInfo: 1 });
-  mongoTimeOffset = res.system.currentTime.getTime() - now;
+onConnect(async (db) => {
+  cacheCollection = db.collection("cache");
+  await cacheCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+});
+
+export async function get(key: string): Promise<string> {
+  const res = await cacheCollection.findOne({ _id: key });
+  if (res) return res["value"];
+  return null;
 }
 
-export async function disconnect(): Promise<void> {
-  if (clientPromise) await (await clientPromise).close();
-}
-
-export async function get(key): Promise<any> {
-  const expire = new Date(Date.now() - mongoTimeOffset);
-  if (Array.isArray(key)) {
-    const res = await mongoCollection.find({ _id: { $in: key } }).toArray();
-
-    const indices = {};
-    key.forEach((v, i) => {
-      indices[v] = i;
-    });
-
-    const values = [];
-    res.forEach(r => {
-      if (r["expire"] > expire) values[indices[r["_id"]]] = r["value"];
-    });
-
-    return values;
-  } else {
-    const res = await mongoCollection.findOne({ _id: { $in: [key] } });
-    if (res && res["expire"] > expire) return res["value"];
-    return null;
-  }
-}
-
-export async function del(key): Promise<void> {
-  if (Array.isArray(key))
-    await mongoCollection.deleteMany({ _id: { $in: key } });
-  else await mongoCollection.deleteOne({ _id: key });
+export async function del(key: string): Promise<void> {
+  await cacheCollection.deleteOne({ _id: key });
 }
 
 export async function set(
   key: string,
-  value: string | number,
+  value: string,
   ttl: number = MAX_CACHE_TTL
 ): Promise<void> {
-  const expire = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
-  await mongoCollection.replaceOne(
+  const timestamp = new Date();
+  const expire = new Date(
+    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000
+  );
+  await cacheCollection.replaceOne(
     { _id: key },
-    { _id: key, value: value, expire: expire },
+    { _id: key, value, expire, timestamp },
     { upsert: true }
   );
 }
 
-export async function pop(key): Promise<any> {
-  const res = await mongoCollection.findOneAndDelete({ _id: key });
-
-  if (
-    res &&
-    res["value"] &&
-    +res["value"]["expire"] - (Date.now() - mongoTimeOffset)
-  )
-    return res["value"]["value"];
-
+export async function pop(key: string): Promise<any> {
+  const res = await cacheCollection.findOneAndDelete({ _id: key });
+  if (res && res["value"]) return res["value"]["value"];
   return null;
 }
 
-export async function lock(lockName, ttl): Promise<Function> {
-  const token = Math.random()
-    .toString(36)
-    .slice(2);
-
-  async function unlockOrExtend(extendTtl): Promise<void> {
-    if (!extendTtl) {
-      const res = await mongoCollection.deleteOne({
-        _id: lockName,
-        value: token
-      });
-      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
-    } else {
-      const expire = new Date(Date.now() - mongoTimeOffset + extendTtl * 1000);
-      const res = await mongoCollection.updateOne(
-        { _id: lockName, value: token },
-        { expire: expire }
-      );
-      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
-    }
-  }
-
-  const expireTest = new Date(Date.now() - mongoTimeOffset);
-  const expireSet = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
-
+export async function acquireLock(
+  lockName: string,
+  ttl: number,
+  timeout = 0,
+  token = Math.random().toString(36).slice(2)
+): Promise<string> {
   try {
-    await mongoCollection.updateOne(
-      { _id: lockName, expire: { $lte: expireTest } },
-      { $set: { value: token, expire: expireSet } },
-      { upsert: true }
+    const now = Date.now();
+    const r = await cacheCollection.findOneAndUpdate(
+      { _id: lockName, token },
+      {
+        $set: {
+          value: token,
+          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
+        },
+        $currentDate: { timestamp: true },
+      },
+      { upsert: true, returnOriginal: false }
     );
+    const v = r.value;
+    if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
+      throw new Error("Database clock skew too great");
   } catch (err) {
-    if (err && err.code === 11000) {
-      await new Promise(resolve => setTimeout(resolve, 200));
-      return lock(lockName, ttl);
+    if (err.code === 11000) {
+      if (timeout > 0) {
+        return new Promise((resolve, reject) => {
+          const w = 50 + Math.random() * 50;
+          setTimeout(() => {
+            acquireLock(lockName, ttl, timeout - w, token).then(
+              resolve,
+              reject
+            );
+          }, w);
+        });
+      }
+      throw new Error("Failed to acquire lock");
     }
+    throw err;
   }
 
-  return unlockOrExtend;
+  return token;
+}
+
+export async function releaseLock(
+  lockName: string,
+  token: string
+): Promise<void> {
+  const res = await cacheCollection.deleteOne({
+    _id: lockName,
+    value: token,
+  });
+  if (res["result"]["n"] !== 1) throw new Error("Lock expired");
 }

lib/cluster.ts

@@ -41,7 +41,7 @@ function restartWorker(worker, code, signal): void {
     message: "Worker died",
     pid: worker.process.pid,
     exitCode: null,
-    signal: null
+    signal: null,
   };
 
   if (code != null) msg.exitCode = code;
@@ -57,7 +57,7 @@ function restartWorker(worker, code, signal): void {
     min2 = 0,
     min3 = 0;
 
-  crashes = crashes.filter(n => {
+  crashes = crashes.filter((n) => {
     if (n > now - 60000) ++min1;
     else if (n > now - 120000) ++min2;
     else if (n > now - 180000) ++min3;
@@ -72,7 +72,7 @@ function restartWorker(worker, code, signal): void {
 
     logger.error({
       message: "Too many crashes, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
     return;
   }
@@ -89,7 +89,11 @@ function restartWorker(worker, code, signal): void {
   }, respawnTimestamp - now);
 }
 
-export function start(workerCount, servicePort, serviceAddress): void {
+export function start(
+  workerCount: number,
+  servicePort: number,
+  serviceAddress: string
+): void {
   cluster.on("listening", (worker, address) => {
     if (
       (address.addressType === 4 || address.addressType === 6) &&
@@ -100,7 +104,7 @@ export function start(workerCount, servicePort, serviceAddress): void {
         message: "Worker listening",
         pid: worker.process.pid,
         address: address.address,
-        port: address.port
+        port: address.port,
       });
     }
   });

lib/common.ts

@@ -17,22 +17,12 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-export const UNDEFINED_TYPE = "[object Undefined]";
-export const NULL_TYPE = "[object Null]";
-export const BOOLEAN_TYPE = "[object Boolean]";
-export const NUMBER_TYPE = "[object Number]";
-export const STRING_TYPE = "[object String]";
-export const ARRAY_TYPE = "[object Array]";
-export const OBJECT_TYPE = "[object Object]";
-export const REGEXP_TYPE = "[object RegExp]";
-export const DATE_TYPE = "[object Date]";
-
-export const typeOf = (obj): string => Object.prototype.toString.call(obj);
-
-export function generateDeviceId(deviceIdStruct): string {
+export function generateDeviceId(
+  deviceIdStruct: Record<string, string>
+): string {
   // Escapes everything except alphanumerics and underscore
   function esc(str): string {
-    return str.replace(/[^A-Za-z0-9_]/g, chr => {
+    return str.replace(/[^A-Za-z0-9_]/g, (chr) => {
       const buf = Buffer.from(chr, "utf8");
       let rep = "";
       for (const b of buf) rep += "%" + b.toString(16).toUpperCase();
@@ -54,6 +44,20 @@ export function generateDeviceId(deviceIdStruct): string {
 }
 
 // Source: http://stackoverflow.com/a/6969486
-export function escapeRegExp(str): string {
+export function escapeRegExp(str: string): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
+
+export function encodeTag(tag: string): string {
+  return encodeURIComponent(tag)
+    .replace(
+      /[!~*'()]/g,
+      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
+    )
+    .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")
+    .replace(/%/g, "0x");
+}
+
+export function decodeTag(tag: string): string {
+  return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%"));
+}

lib/common/authorizer.ts

@@ -23,13 +23,13 @@ import { evaluate, or } from "./expression";
 export default class Authorizer {
   private permissionSets: PermissionSet[];
   private validatorCache: WeakMap<
-    object,
+    any,
     (mutationType, mutation, any) => boolean
   >;
   private hasAccessCache: Map<string, boolean>;
   private getFilterCache: Map<string, Expression>;
 
-  public constructor(permissionSets) {
+  public constructor(permissionSets: PermissionSet[]) {
     this.permissionSets = permissionSets;
     this.validatorCache = new WeakMap();
     this.hasAccessCache = new Map();
@@ -77,8 +77,8 @@ export default class Authorizer {
   }
 
   public getValidator(
-    resourceType,
-    resource
+    resourceType: string,
+    resource: unknown
   ): (mutationType: string, mutation?: any, args?: any) => boolean {
     if (this.validatorCache.has(resource))
       return this.validatorCache.get(resource);
@@ -108,7 +108,7 @@ export default class Authorizer {
         mutation,
         resourceType,
         object: resource,
-        options: any
+        options: any,
       };
 
       const valueFunction = (paramName): any => {
@@ -124,7 +124,8 @@ export default class Authorizer {
             if (value == null) break;
           }
         } else if (object[entry]) {
-          value = object[entry][paramName];
+          if (paramName) value = object[entry][paramName];
+          else value = object[entry];
         }
 
         return value;

lib/common/bigint-jsbi.ts

@@ -0,0 +1,108 @@
+/**
+ * Copyright 2013-2020  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import JSBI from "jsbi";
+
+export function and(a: JSBI, b: JSBI): JSBI {
+  return JSBI.bitwiseAnd(a, b);
+}
+
+export function or(a: JSBI, b: JSBI): JSBI {
+  return JSBI.bitwiseOr(a, b);
+}
+
+export function xor(a: JSBI, b: JSBI): JSBI {
+  return JSBI.bitwiseXor(a, b);
+}
+
+export function not(a: JSBI): JSBI {
+  return JSBI.bitwiseNot(a);
+}
+
+export function lshift(a: JSBI, b: JSBI): JSBI {
+  return JSBI.leftShift(a, b);
+}
+
+export function rshift(a: JSBI, b: JSBI): JSBI {
+  return JSBI.signedRightShift(a, b);
+}
+
+export function add(a: JSBI, b: JSBI): JSBI {
+  return JSBI.add(a, b);
+}
+
+export function sub(a: JSBI, b: JSBI): JSBI {
+  return JSBI.subtract(a, b);
+}
+
+export function mul(a: JSBI, b: JSBI): JSBI {
+  return JSBI.multiply(a, b);
+}
+
+export function div(a: JSBI, b: JSBI): JSBI {
+  return JSBI.divide(a, b);
+}
+
+export function exp(a: JSBI, b: JSBI): JSBI {
+  return JSBI.exponentiate(a, b);
+}
+
+export function rem(a: JSBI, b: JSBI): JSBI {
+  return JSBI.remainder(a, b);
+}
+
+export function toNumber(a: JSBI): number {
+  return JSBI.toNumber(a);
+}
+
+export function eq(a: JSBI, b: JSBI): boolean {
+  return JSBI.equal(a, b);
+}
+
+export function ne(a: JSBI, b: JSBI): boolean {
+  return JSBI.notEqual(a, b);
+}
+
+export function lt(a: JSBI, b: JSBI): boolean {
+  return JSBI.lessThan(a, b);
+}
+
+export function lte(a: JSBI, b: JSBI): boolean {
+  return JSBI.lessThanOrEqual(a, b);
+}
+
+export function gt(a: JSBI, b: JSBI): boolean {
+  return JSBI.greaterThan(a, b);
+}
+
+export function gte(a: JSBI, b: JSBI): boolean {
+  return JSBI.greaterThanOrEqual(a, b);
+}
+
+export function asUintN(a: number, b: JSBI): JSBI {
+  return JSBI.asUintN(a, b);
+}
+
+export function asIntN(a: number, b: JSBI): JSBI {
+  return JSBI.asIntN(a, b);
+}
+
+const _BigInt = JSBI.BigInt;
+type _bigint = JSBI;
+export { _BigInt as BigInt, _bigint as bigint };

lib/common/bigint.ts

@@ -0,0 +1,106 @@
+/**
+ * Copyright 2013-2020  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+export function and(a: bigint, b: bigint): bigint {
+  return a & b;
+}
+
+export function or(a: bigint, b: bigint): bigint {
+  return a | b;
+}
+
+export function xor(a: bigint, b: bigint): bigint {
+  return a ^ b;
+}
+
+export function not(a: bigint): bigint {
+  return ~a;
+}
+
+export function lshift(a: bigint, b: bigint): bigint {
+  return a << b;
+}
+
+export function rshift(a: bigint, b: bigint): bigint {
+  return a >> b;
+}
+
+export function add(a: bigint, b: bigint): bigint {
+  return a + b;
+}
+
+export function sub(a: bigint, b: bigint): bigint {
+  return a - b;
+}
+
+export function mul(a: bigint, b: bigint): bigint {
+  return a * b;
+}
+
+export function div(a: bigint, b: bigint): bigint {
+  return a / b;
+}
+
+export function exp(a: bigint, b: bigint): bigint {
+  return a ** b;
+}
+
+export function rem(a: bigint, b: bigint): bigint {
+  return a % b;
+}
+
+export function toNumber(a: bigint): number {
+  return Number(a);
+}
+
+export function eq(a: bigint, b: bigint): boolean {
+  return a === b;
+}
+
+export function ne(a: bigint, b: bigint): boolean {
+  return a !== b;
+}
+
+export function lt(a: bigint, b: bigint): boolean {
+  return a < b;
+}
+
+export function lte(a: bigint, b: bigint): boolean {
+  return a <= b;
+}
+
+export function gt(a: bigint, b: bigint): boolean {
+  return a > b;
+}
+
+export function gte(a: bigint, b: bigint): boolean {
+  return a >= b;
+}
+
+export function asUintN(a: number, b: bigint): bigint {
+  return BigInt.asUintN(a, b);
+}
+
+export function asIntN(a: number, b: bigint): bigint {
+  return BigInt.asIntN(a, b);
+}
+
+const _BigInt = BigInt;
+type _bigint = bigint;
+export { _BigInt as BigInt, _bigint as bigint };

lib/common/expression-cnf.ts

@@ -1,285 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { map, parseLikePattern } from "./expression-parser";
-import { Expression } from "../types";
-
-type CNF = number[][];
-
-function or(cnf1: CNF, cnf2: CNF): CNF {
-  if (cnf1.length === 0) return cnf2;
-  else if (cnf2.length === 0) return cnf1;
-
-  if (cnf1.length > cnf2.length) {
-    const c = cnf1;
-    cnf1 = cnf2;
-    cnf2 = c;
-  }
-
-  const res = [];
-  for (let i = 0; i < cnf1.length; ++i)
-    for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j]));
-
-  return res;
-}
-
-function likePatternIncludes(pat1, pat2, idx1 = 0, idx2 = 0): boolean {
-  while (idx1 < pat1.length && idx2 < pat2.length) {
-    if (pat1[idx1] === "\\%") {
-      for (let i = idx2; i <= pat2.length; ++i)
-        if (likePatternIncludes(pat1, pat2, idx1 + 1, i)) return true;
-      return false;
-    } else if (pat2[idx2] === "\\%") {
-      if (pat1[idx1] !== "\\%") return false;
-    } else if (pat1[idx1] === "\\_") {
-      // Ignore
-    } else if (pat2[idx2] === "\\_") {
-      return false;
-    } else if (pat2[idx2] !== pat1[idx1]) {
-      return false;
-    }
-    ++idx1;
-    ++idx2;
-  }
-
-  if (idx1 === pat1.length && idx2 === pat2.length) return true;
-  return false;
-}
-
-export function booleanCnf(exp: Expression): { vars: number; clauses: CNF } {
-  const ranges = new Map();
-  const likePatterns = new Map();
-  map(exp, e => {
-    if (!Array.isArray(e)) return e;
-    if (e[0] === "LIKE" || e[0] === "NOT LIKE") {
-      if (
-        typeof e[2] === "string" &&
-        (e.length >= 4 || (e[3] != null && !Array.isArray(e[3])))
-      ) {
-        const key = JSON.stringify(e[1]);
-        let s = likePatterns.get(key);
-        if (!s) likePatterns.set(key, (s = new Map()));
-        s.set(`${e[3] || ""}:${e[2]}`, parseLikePattern(e[2], e[3]));
-      }
-    } else if ([">", ">=", "<", "<=", "=", "<>"].includes(e[0])) {
-      if (!Array.isArray(e[2])) {
-        const key = `${JSON.stringify(e[1])}:${typeof e[2]}`;
-        let s = ranges.get(key);
-        if (!s) ranges.set(key, (s = new Set()));
-        s.add(e[2]);
-      } else if (!Array.isArray(e[1])) {
-        const key = `${JSON.stringify(e[2])}:${typeof e[1]}`;
-        let s = ranges.get(key);
-        if (!s) ranges.set(key, (s = new Set()));
-        s.add(e[2]);
-      }
-    }
-    return e;
-  });
-
-  for (const [k, v] of ranges) {
-    if (k.endsWith(":number")) {
-      ranges.set(
-        k,
-        Array.from(v as number[]).sort((a: number, b: number) => a - b)
-      );
-    } else {
-      ranges.set(k, Array.from(v).sort());
-    }
-  }
-
-  const expressions: Map<string, number> = new Map();
-  const mutuallyExclusive: Map<string, Set<number>> = new Map();
-
-  function getVariable(lhs, op, rhs = ""): number {
-    const key = `${lhs}:${op}:${rhs}`;
-    let f = expressions.get(key);
-    if (!f) {
-      expressions.set(key, (f = expressions.size + 1));
-      let m = mutuallyExclusive.get(lhs);
-      if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
-      m.add(f);
-      if (rhs || op !== "null") m.add(getVariable(lhs, "null"));
-    }
-    return f;
-  }
-
-  function getComparisonVariables(
-    lhs,
-    from,
-    fromInclusive,
-    to,
-    toInclusive
-  ): number[] {
-    const type = from != null ? typeof from : typeof to;
-    const range = ranges.get(`${lhs}:${type}`);
-    const vars = [];
-
-    if (from == null) vars.push(`${lhs}:--${range[0]}`);
-
-    for (let i = 0; i < range.length; ++i) {
-      if (
-        i > 0 &&
-        (from == null || range[i] > from) &&
-        (to == null || range[i] <= to)
-      )
-        vars.push(`${lhs}:${range[i - 1]}--${range[i]}`);
-
-      if (
-        (range[i] > from && range[i] < to) ||
-        (range[i] === from && fromInclusive) ||
-        (range[i] === to && toInclusive)
-      )
-        vars.push(`${lhs}:${range[i]}`);
-    }
-
-    if (to == null) vars.push(`${lhs}:${range[range.length - 1]}--`);
-
-    return vars.map(key => {
-      let f = expressions.get(key);
-      if (!f) {
-        expressions.set(key, (f = expressions.size + 1));
-        let m = mutuallyExclusive.get(lhs);
-        if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
-        m.add(f);
-      }
-      return f;
-    });
-  }
-
-  function getLikeVariables(lhs, pat, esc): number[] {
-    const vars = [];
-    const pats = likePatterns.get(lhs);
-    pat = pats.get(`${esc || ""}:${pat}`);
-    for (const p of pats.values()) {
-      if (pat === p || likePatternIncludes(pat, p))
-        vars.push(`${lhs}:like:${JSON.stringify(p)}`);
-    }
-
-    return vars.map(key => {
-      let f = expressions.get(key);
-      if (!f) expressions.set(key, (f = expressions.size + 1));
-      return f;
-    });
-  }
-
-  interface ComparisonOperators {
-    [name: string]: (v: string) => [string, boolean, string, boolean];
-  }
-
-  const comparisonOperators: ComparisonOperators = {
-    "=": v => [v, true, v, true],
-    ">": v => [v, false, null, null],
-    ">=": v => [v, true, null, null],
-    "<": v => [null, null, v, false],
-    "<=": v => [null, null, v, true]
-  };
-
-  function recursive(clause, negate): CNF {
-    let op = clause[0];
-
-    if ((op === "AND" && !negate) || (op === "OR" && negate)) {
-      let res = [];
-      for (let i = 1; i < clause.length; ++i)
-        res = res.concat(recursive(clause[i], negate));
-      return res;
-    } else if ((op === "OR" && !negate) || (op === "AND" && negate)) {
-      let res = [];
-      for (let i = 1; i < clause.length; ++i)
-        res = or(res, recursive(clause[i], negate));
-      return res;
-    } else if (op === "NOT") {
-      return recursive(clause[1], !negate);
-    } else if (op === "<>") {
-      const vars = getComparisonVariables(
-        JSON.stringify(clause[1]),
-        ...comparisonOperators["="](clause[2])
-      );
-      if (!negate) return vars.map(x => [0 - x]);
-      else return [vars];
-    } else if (op === "IS NULL") {
-      const v = getVariable(JSON.stringify(clause[1]), "null");
-      if (negate) return [[0 - v]];
-      else return [[v]];
-    } else if (op === "IS NOT NULL") {
-      const v = getVariable(JSON.stringify(clause[1]), "null");
-      if (!negate) return [[0 - v]];
-      else return [[v]];
-    } else if (op in comparisonOperators) {
-      if (!Array.isArray(clause[2])) {
-        const vars = getComparisonVariables(
-          JSON.stringify(clause[1]),
-          ...comparisonOperators[op](clause[2])
-        );
-        if (negate) return vars.map(x => [0 - x]);
-        else return [vars];
-      } else if (!Array.isArray(clause[1])) {
-        op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
-        const vars = getComparisonVariables(
-          JSON.stringify(clause[2]),
-          ...comparisonOperators[op](clause[1])
-        );
-        if (negate) return vars.map(x => [0 - x]);
-        else return [vars];
-      } else {
-        let lhs = JSON.stringify(clause[1]);
-        let rhs = JSON.stringify(clause[2]);
-        // For consistency
-        if (rhs > lhs) {
-          const t = rhs;
-          rhs = lhs;
-          lhs = t;
-          op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
-        }
-
-        const v = getVariable(lhs, op, rhs);
-        if (negate) return [[0 - v]];
-        else return [[v]];
-      }
-    } else if (
-      (op === "LIKE" || op === "NOT LIKE") &&
-      !Array.isArray(clause[2]) &&
-      clause[2] != null &&
-      (clause.length >= 4 || clause[3] != null)
-    ) {
-      if (op === "NOT LIKE") negate = !negate;
-      const vars = getLikeVariables(
-        JSON.stringify(clause[1]),
-        clause[2],
-        clause[3]
-      );
-      if (negate) return vars.map(x => [0 - x]);
-      else return [vars];
-    } else {
-      const v = getVariable(JSON.stringify(clause), "");
-      if (negate) return [[0 - v]];
-      else return [[v]];
-    }
-  }
-
-  const cnf = recursive(exp, false);
-
-  for (const m of mutuallyExclusive.values()) {
-    const ar: number[] = Array.from(m);
-    for (let i = 0; i < ar.length; ++i)
-      for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]);
-  }
-
-  return { vars: expressions.size, clauses: cnf };
-}

lib/common/expression-parser.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import * as parsimmon from "parsimmon";
+import parsimmon from "parsimmon";
 import { Expression } from "../types";
 
 // Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
@@ -27,7 +27,7 @@ function interpretEscapes(str): string {
     f: "\f",
     n: "\n",
     r: "\r",
-    t: "\t"
+    t: "\t",
   };
   return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
     const type = escape.charAt(0);
@@ -40,7 +40,10 @@ function interpretEscapes(str): string {
   });
 }
 
-export function map(exp, callback): Expression {
+export function map(
+  exp: Expression,
+  callback: (e: Expression) => Expression
+): Expression {
   if (!Array.isArray(exp)) return callback(exp);
 
   let clone;
@@ -55,7 +58,10 @@ export function map(exp, callback): Expression {
   return callback(clone || exp);
 }
 
-export async function mapAsync(exp, callback): Promise<Expression> {
+export async function mapAsync(
+  exp: Expression,
+  callback: (e: Expression) => Promise<Expression>
+): Promise<Expression> {
   if (!Array.isArray(exp)) return callback(exp);
 
   let clone;
@@ -70,7 +76,10 @@ export async function mapAsync(exp, callback): Promise<Expression> {
   return callback(clone || exp);
 }
 
-function binaryLeft(operatorsParser, nextParser): parsimmon.Parser<{}> {
+function binaryLeft(
+  operatorsParser: parsimmon.Parser<string>,
+  nextParser: parsimmon.Parser<Expression>
+): parsimmon.Parser<Expression> {
   return parsimmon.seqMap(
     nextParser,
     parsimmon.seq(operatorsParser, nextParser).many(),
@@ -86,7 +95,7 @@ function binaryLeft(operatorsParser, nextParser): parsimmon.Parser<{}> {
 }
 
 const lang = parsimmon.createLanguage({
-  ComparisonOperator: function() {
+  ComparisonOperator: function () {
     return parsimmon
       .alt(
         parsimmon.string(">="),
@@ -98,13 +107,10 @@ const lang = parsimmon.createLanguage({
       )
       .skip(parsimmon.optWhitespace);
   },
-  LikeOperator: function() {
+  LikeOperator: function () {
     return parsimmon
       .alt(
-        parsimmon
-          .regexp(/like/i)
-          .result("LIKE")
-          .desc("LIKE"),
+        parsimmon.regexp(/like/i).result("LIKE").desc("LIKE"),
         parsimmon
           .regexp(/not\s+like/i)
           .result("NOT LIKE")
@@ -113,7 +119,7 @@ const lang = parsimmon.createLanguage({
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
   },
-  IsNullOperator: function() {
+  IsNullOperator: function () {
     return parsimmon
       .alt(
         parsimmon
@@ -128,7 +134,7 @@ const lang = parsimmon.createLanguage({
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
   },
-  NotOperator: function() {
+  NotOperator: function () {
     return parsimmon
       .regexp(/not/i)
       .result("NOT")
@@ -136,7 +142,7 @@ const lang = parsimmon.createLanguage({
       .skip(parsimmon.optWhitespace)
       .desc("NOT");
   },
-  AndOperator: function() {
+  AndOperator: function () {
     return parsimmon
       .regexp(/and/i)
       .result("AND")
@@ -144,7 +150,7 @@ const lang = parsimmon.createLanguage({
       .skip(parsimmon.optWhitespace)
       .desc("AND");
   },
-  OrOperator: function() {
+  OrOperator: function () {
     return parsimmon
       .regexp(/or/i)
       .result("OR")
@@ -152,7 +158,7 @@ const lang = parsimmon.createLanguage({
       .skip(parsimmon.optWhitespace)
       .desc("OR");
   },
-  Parameter: function(r) {
+  Parameter: function (r) {
     return parsimmon
       .alt(
         parsimmon.regexp(/[a-zA-Z0-9_.*-]+/),
@@ -162,48 +168,43 @@ const lang = parsimmon.createLanguage({
         )
       )
       .atLeast(1)
-      .map(x => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
+      .map((x) => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
       .skip(parsimmon.optWhitespace)
       .desc("parameter");
   },
-  StringValueSql: function() {
+  StringValueSql: function () {
     return parsimmon
       .regexp(/'([^']*)'/, 1)
       .atLeast(1)
       .skip(parsimmon.optWhitespace)
-      .map(s => s.join("'"))
+      .map((s) => s.join("'"))
       .desc("string");
   },
-  StringValueJs: function() {
+  StringValueJs: function () {
     return parsimmon
       .regexp(/"((?:\\.|.)*?)"/, 1)
       .skip(parsimmon.optWhitespace)
       .map(interpretEscapes)
       .desc("string");
   },
-  NumberValue: function() {
+  NumberValue: function () {
     return parsimmon
       .regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/)
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace)
       .map(Number)
       .desc("number");
   },
-  BooleanValue: function() {
+  BooleanValue: function () {
     return parsimmon
       .alt(
-        parsimmon
-          .regexp(/true/i)
-          .result(true)
-          .desc("TRUE"),
-        parsimmon
-          .regexp(/false/i)
-          .result(false)
-          .desc("FALSE")
+        parsimmon.regexp(/true/i).result(true).desc("TRUE"),
+        parsimmon.regexp(/false/i).result(false).desc("FALSE")
       )
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
   },
-  NullValue: function() {
+  NullValue: function () {
     return parsimmon
       .regexp(/null/i)
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
@@ -211,32 +212,73 @@ const lang = parsimmon.createLanguage({
       .result(null)
       .desc("NULL");
   },
-  FuncValue: function(r) {
+  FuncValue: function (r) {
     return parsimmon.seqMap(
       parsimmon
         .regexp(/([a-zA-Z0-9_]+)/, 1)
         .skip(parsimmon.optWhitespace)
         .desc("function"),
-      r.ValueExpression.sepBy(
-        parsimmon.string(",").skip(parsimmon.optWhitespace)
-      ).wrap(
+      r.ExpressionList.wrap(
         parsimmon.string("(").skip(parsimmon.optWhitespace),
         parsimmon.string(")").skip(parsimmon.optWhitespace)
       ),
       (f, args) => ["FUNC", f.toUpperCase()].concat(args)
     );
   },
-  Value: function(r) {
+  WhenPair: function (r) {
+    return parsimmon.seq(
+      parsimmon
+        .regexp(/when/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("WHEN")
+        .then(r.Expression),
+      parsimmon
+        .regexp(/then/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("THEN")
+        .then(r.Expression)
+    );
+  },
+  CaseStatement: function (r) {
+    return parsimmon.seqMap(
+      parsimmon
+        .regexp(/case/i)
+        .result("CASE")
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("CASE"),
+      r.WhenPair.many(),
+      parsimmon
+        .regexp(/else/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("ELSE")
+        .then(r.Expression)
+        .map((e) => [[true, e]])
+        .fallback(null)
+        .skip(
+          parsimmon
+            .regex(/end/i)
+            .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        )
+        .skip(parsimmon.optWhitespace),
+      (...arr) => arr.flat(2)
+    );
+  },
+  Value: function (r) {
     return parsimmon.alt(
       r.NullValue,
       r.BooleanValue,
       r.NumberValue,
       r.StringValueSql,
       r.StringValueJs,
-      r.FuncValue
+      r.FuncValue,
+      r.CaseStatement
     );
   },
-  ValueExpression: function(r) {
+  ValueExpression: function (r) {
     return binaryLeft(
       parsimmon.string("||").skip(parsimmon.optWhitespace),
       binaryLeft(
@@ -259,7 +301,7 @@ const lang = parsimmon.createLanguage({
       )
     );
   },
-  Comparison: function(r) {
+  Comparison: function (r) {
     return parsimmon.alt(
       parsimmon.seqMap(r.ValueExpression, r.IsNullOperator, (p, o) => [o, p]),
       parsimmon.seqMap(
@@ -289,8 +331,16 @@ const lang = parsimmon.createLanguage({
       )
     );
   },
-  Expression: function(r) {
-    function unary(operatorsParser, nextParser): parsimmon.Parser<{}> {
+  ExpressionList: function (r) {
+    return r.Expression.sepBy(
+      parsimmon.string(",").skip(parsimmon.optWhitespace)
+    );
+  },
+  Expression: function (r) {
+    function unary(
+      operatorsParser: parsimmon.Parser<string>,
+      nextParser: parsimmon.Parser<Expression>
+    ): parsimmon.Parser<Expression> {
       return parsimmon.seq(operatorsParser, nextParser).or(nextParser);
     }
 
@@ -301,7 +351,7 @@ const lang = parsimmon.createLanguage({
         unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
       )
     ).trim(parsimmon.optWhitespace);
-  }
+  },
 });
 
 export function parse(str: string): Expression {
@@ -309,7 +359,12 @@ export function parse(str: string): Expression {
   return lang.Expression.tryParse(str);
 }
 
-export function stringify(exp, level = 0): string {
+export function parseList(str: string): Expression[] {
+  if (!str) return [];
+  return lang.ExpressionList.tryParse(str);
+}
+
+export function stringify(exp: Expression, level = 0): string {
   if (!Array.isArray(exp)) return JSON.stringify(exp);
 
   const opLevels = {
@@ -330,7 +385,7 @@ export function stringify(exp, level = 0): string {
     "+": 31,
     "-": 31,
     "*": 32,
-    "/": 32
+    "/": 32,
   };
 
   const op = exp[0].toUpperCase();
@@ -344,7 +399,7 @@ export function stringify(exp, level = 0): string {
     return wrap(
       `${exp[1]}(${exp
         .slice(2)
-        .map(e => stringify(e))
+        .map((e) => stringify(e))
         .join(", ")})`
     );
   } else if (op === "PARAM") {
@@ -354,7 +409,7 @@ export function stringify(exp, level = 0): string {
       return wrap(
         exp[1]
           .slice(1)
-          .map(p => {
+          .map((p) => {
             if (typeof p === "string") return p;
             else return `{${stringify(p)}}`;
           })
@@ -381,6 +436,17 @@ export function stringify(exp, level = 0): string {
         )}`
       );
     }
+  } else if (op === "CASE") {
+    const parts: string[] = ["CASE"];
+    for (let i = 1; i < exp.length - 1; i += 2) {
+      if (!Array.isArray(exp[i]) && exp[i]) {
+        if (exp[i + 1] != null) parts.push("ELSE", stringify(exp[i + 1]));
+        break;
+      }
+      parts.push("WHEN", stringify(exp[i]), "THEN", stringify(exp[i + 1]));
+    }
+    parts.push("END");
+    return parts.join(" ");
   } else if (op in opLevels) {
     const parts = exp.slice(1).map((e, i) => {
       return stringify(e, opLevels[exp[0]] + Math.min(i - 1, 0));
@@ -393,7 +459,7 @@ export function stringify(exp, level = 0): string {
   }
 }
 
-export function parseLikePattern(pat, esc): string[] {
+export function parseLikePattern(pat: string, esc: string): string[] {
   const chars = pat.split("");
 
   for (let i = 0; i < chars.length; ++i) {
@@ -408,5 +474,5 @@ export function parseLikePattern(pat, esc): string[] {
       while (chars[i + 1] === "%") chars[++i] = "";
     }
   }
-  return chars.filter(c => c);
+  return chars.filter((c) => c);
 }

lib/common/expression.ts

@@ -18,13 +18,11 @@
  */
 
 import { map, mapAsync, parseLikePattern } from "./expression-parser";
-import { booleanCnf } from "./expression-cnf";
-import { naiveDpll } from "./sat-solver";
 import { Expression } from "../types";
 
 const isArray = Array.isArray;
 
-const regExpCache = new WeakMap<object, RegExp>();
+const regExpCache = new WeakMap<any, RegExp>();
 
 const REDUCE_SKIP = {};
 function reduce(exp, callback): Expression {
@@ -44,7 +42,7 @@ function reduce(exp, callback): Expression {
   return exp;
 }
 
-export function likePatternToRegExp(pat, esc = "", flags = ""): RegExp {
+export function likePatternToRegExp(pat: string, esc = "", flags = ""): RegExp {
   const convChars = {
     "-": "\\-",
     "/": "\\/",
@@ -63,159 +61,231 @@ export function likePatternToRegExp(pat, esc = "", flags = ""): RegExp {
     "{": "\\{",
     "}": "\\}",
     "\\%": ".*",
-    "\\_": "."
+    "\\_": ".",
   };
   let chars = parseLikePattern(pat, esc);
   if (!chars.length) return new RegExp("^$", flags);
-  chars = chars.map(c => convChars[c] || c);
+  chars = chars.map((c) => convChars[c] || c);
   chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
   const l = chars.length - 1;
   chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
   return new RegExp(chars.join(""), flags);
 }
 
-function evalExp(e: Expression): Expression {
-  if (e[0] === "AND") {
-    return reduce(e, (a, b) => {
-      if (!isArray(a)) return a ? b : a;
-      if (!isArray(b)) return b ? a : b;
-      return REDUCE_SKIP;
-    });
-  } else if (e[0] === "OR") {
-    return reduce(e, (a, b) => {
-      if (!isArray(a)) return a ? a : b;
-      if (!isArray(b)) return b ? b : a;
-      return REDUCE_SKIP;
-    });
-  } else if (e[0] === "NOT") {
-    if (!isArray(e[1])) return !e[1];
-    else if (e[1][0] === "NOT") return e[1][1];
-  } else if (e[0] === "IS NULL") {
-    if (isArray(e[1])) return e;
-    else if (e[1] == null) return true;
+function compare(
+  a: boolean | number | string,
+  b: boolean | number | string
+): number {
+  if (typeof a === "boolean") a = +a;
+  if (typeof b === "boolean") b = +b;
+  if (typeof a !== typeof b) return typeof a === "string" ? 1 : -1;
+  return a > b ? 1 : a < b ? -1 : 0;
+}
+
+function toNumber(a: boolean | number | string): number {
+  switch (typeof a) {
+    case "number":
+      return a;
+    case "boolean":
+      return +a;
+    case "string":
+      return parseFloat(a) || 0;
+  }
+}
+
+function toString(a: boolean | number | string): string {
+  switch (typeof a) {
+    case "string":
+      return a;
+    case "number":
+      return a.toString();
+    case "boolean":
+      return (+a).toString();
+  }
+}
+
+export function evaluateCallback(exp: Expression): Expression {
+  if (!Array.isArray(exp)) return exp;
+  if (exp[0] === "CASE") {
+    for (let i = 1; i < exp.length; i += 2) {
+      if (Array.isArray(exp[i])) return exp;
+      if (exp[i]) return exp[i + 1];
+    }
+    return null;
+  } else if (exp[0] === "FUNC") {
+    if (exp[1] === "COALESCE") {
+      const args: Expression[] = [];
+      for (let i = 2; i < exp.length; ++i) {
+        const e = exp[i];
+        if (e == null) continue;
+        args.push(e);
+        if (!Array.isArray(e)) break;
+      }
+      if (!args.length) return null;
+      if (args.length === 1) return args[0];
+      return ["FUNC", "COALESCE", ...args];
+    } else if (exp[1] === "UPPER") {
+      if (exp[2] == null) return null;
+      if (!isArray(exp[2])) return toString(exp[2]).toUpperCase();
+    } else if (exp[1] === "LOWER") {
+      if (exp[2] == null) return null;
+      if (!isArray(exp[2])) return toString(exp[2]).toLowerCase();
+    }
+  } else if (exp[0] === "PARAM") {
+    if (exp[1] == null) return null;
+  } else if (exp[0] === "AND") {
+    for (let i = 1; i < exp.length; ++i)
+      if (!Array.isArray(exp[i]) && exp[i] != null && !exp[i]) return false;
+    const args: Expression[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const ee = exp[i];
+      if (ee == null) return null;
+      if (Array.isArray(ee)) {
+        if (ee[0] === "AND") args.push(...ee.slice(1));
+        else args.push(ee);
+      }
+    }
+    if (!args.length) return true;
+    if (args.length === 1) args.push(true);
+    return ["AND", ...args];
+  } else if (exp[0] === "OR") {
+    const args: Expression[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const ee = exp[i];
+      if (Array.isArray(ee)) {
+        if (ee[0] === "OR") args.push(...ee.slice(1));
+        else args.push(ee);
+      } else if (ee) {
+        return true;
+      }
+    }
+    if (!args.length) return exp.some((ee) => ee == null) ? null : false;
+    if (args.length === 1) args.push(false);
+    return ["OR", ...args];
+  } else if (exp[0] === "NOT") {
+    if (exp[1] == null) return null;
+    if (!isArray(exp[1])) return !exp[1];
+    else if (exp[1][0] === "NOT") return exp[1][1];
+  } else if (exp[0] === "IS NULL") {
+    if (isArray(exp[1])) return exp;
+    else if (exp[1] == null) return true;
     else return null;
-  } else if (e[0] === "IS NOT NULL") {
-    if (isArray(e[1])) return e;
-    else if (e[1] != null) return true;
+  } else if (exp[0] === "IS NOT NULL") {
+    if (isArray(exp[1])) return exp;
+    else if (exp[1] != null) return true;
     else return null;
-  } else if (e[0] === "LIKE") {
-    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
+  } else if (exp[0] === "LIKE") {
+    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
     else if (
-      e[1] == null ||
-      e[2] == null ||
-      ((e as any[]).length >= 4 && e[3] == null)
+      exp[1] == null ||
+      exp[2] == null ||
+      ((exp as any[]).length >= 4 && exp[3] == null)
     )
       return null;
-    let r = regExpCache.get(e as any[]);
+    let r = regExpCache.get(exp as any[]);
     if (!r) {
-      r = likePatternToRegExp(e[2], e[3]);
-      regExpCache.set(e as any[], r);
+      r = likePatternToRegExp(exp[2], exp[3]);
+      regExpCache.set(exp as any[], r);
     }
-    return r.test(e[1]);
-  } else if (e[0] === "NOT LIKE") {
-    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
+    return r.test(exp[1]);
+  } else if (exp[0] === "NOT LIKE") {
+    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
     else if (
-      e[1] == null ||
-      e[2] == null ||
-      ((e as any[]).length >= 4 && e[3] == null)
+      exp[1] == null ||
+      exp[2] == null ||
+      ((exp as any[]).length >= 4 && exp[3] == null)
     )
       return null;
-    let r = regExpCache.get(e as any[]);
+    let r = regExpCache.get(exp as any[]);
     if (!r) {
-      r = likePatternToRegExp(e[2], e[3]);
-      regExpCache.set(e as any[], r);
+      r = likePatternToRegExp(exp[2], exp[3]);
+      regExpCache.set(exp as any[], r);
     }
-    return !r.test(e[1]);
-  } else if (e[0] === "=") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] === e[2];
-  } else if (e[0] === "<>") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] !== e[2];
-  } else if (e[0] === ">") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] > e[2];
-  } else if (e[0] === ">=") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] >= e[2];
-  } else if (e[0] === "<") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] < e[2];
-  } else if (e[0] === "<=") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] <= e[2];
-  } else if (e[0] === "*") {
-    return reduce(e, (a, b) => {
-      if (!isArray(a) && !isArray(b)) {
-        if (a == null || b == null) return null;
-        return a * b;
-      }
+    return !r.test(exp[1]);
+  } else if (exp[0] === "=") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) === 0;
+  } else if (exp[0] === "<>") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) !== 0;
+  } else if (exp[0] === ">") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) > 0;
+  } else if (exp[0] === ">=") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) >= 0;
+  } else if (exp[0] === "<") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) < 0;
+  } else if (exp[0] === "<=") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) <= 0;
+  } else if (exp[0] === "*") {
+    return reduce(exp, (a, b) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b)) return toNumber(a) * toNumber(b);
       return REDUCE_SKIP;
     });
-  } else if (e[0] === "/") {
-    return reduce(e, (a, b, i) => {
-      if (!isArray(a) && !isArray(b)) {
-        if (a == null || b == null) return null;
-        return i === 0 ? a / b : a * b;
-      }
+  } else if (exp[0] === "/") {
+    return reduce(exp, (a, b, i) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b))
+        return i === 0 ? toNumber(a) / toNumber(b) : toNumber(a) * toNumber(b);
       return REDUCE_SKIP;
     });
-  } else if (e[0] === "+") {
-    return reduce(e, (a, b) => {
-      if (!isArray(a) && !isArray(b)) {
-        if (a == null || b == null) return null;
-        return a + b;
-      }
+  } else if (exp[0] === "+") {
+    return reduce(exp, (a, b) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b)) return toNumber(a) + toNumber(b);
       return REDUCE_SKIP;
     });
-  } else if (e[0] === "-") {
-    return reduce(e, (a, b, i) => {
-      if (!isArray(a) && !isArray(b)) {
-        if (a == null || b == null) return null;
-        return i === 0 ? a - b : a + b;
-      }
+  } else if (exp[0] === "-") {
+    return reduce(exp, (a, b, i) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b))
+        return i === 0 ? toNumber(a) - toNumber(b) : toNumber(a) + toNumber(b);
       return REDUCE_SKIP;
     });
-  } else if (e[0] === "||") {
-    return reduce(e, (a, b) => {
-      if (!isArray(a) && !isArray(b)) {
-        if (a == null || b == null) return null;
-        return `${a}${b}`;
-      }
+  } else if (exp[0] === "||") {
+    return reduce(exp, (a, b) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b)) return toString(a) + toString(b);
       return REDUCE_SKIP;
     });
   }
-  return e;
+  return exp;
 }
 
 export function evaluate(
-  exp,
-  obj,
+  exp: Expression,
+  obj: Record<string, unknown> | ((e: string) => Expression),
   now: number,
-  cb?: Function
+  cb?: (e: Expression) => Expression
 ): string | number | boolean | null;
-export function evaluate(exp, obj?, now?: number, cb?: Function): Expression;
-export function evaluate(exp, obj?, now?: number, cb?: Function): Expression {
-  return map(exp, e => {
+export function evaluate(
+  exp: Expression,
+  obj?: Record<string, unknown> | ((e: string) => Expression),
+  now?: number,
+  cb?: (e: Expression) => Expression
+): Expression;
+export function evaluate(
+  exp: Expression,
+  obj?: Record<string, unknown> | ((e: string) => Expression),
+  now?: number,
+  cb?: (e: Expression) => Expression
+): Expression {
+  return map(exp, (e) => {
     if (cb) e = cb(e);
     if (!isArray(e)) return e;
 
-    if (e[0] === "FUNC") {
-      if (e[1] === "NOW") {
-        if (now) return now;
-      } else if (e[1] === "UPPER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
-      } else if (e[1] === "LOWER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
-      }
+    if (e[0] === "FUNC" && e[1] === "NOW") {
+      if (now) return now;
     } else if (e[0] === "PARAM") {
       if (e[1] == null) return null;
       if (obj && !isArray(e[1])) {
@@ -228,29 +298,29 @@ export function evaluate(exp, obj?, now?: number, cb?: Function): Expression {
         return v;
       }
     }
-    return evalExp(e);
+    return evaluateCallback(e);
   });
 }
 
 export async function evaluateAsync(
-  exp,
-  obj,
+  exp: Expression,
+  obj: Record<string, unknown>,
   now: number,
-  cb?: Function
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<string | number | boolean | null>;
 export async function evaluateAsync(
-  exp,
-  obj?,
+  exp: Expression,
+  obj?: Record<string, unknown>,
   now?: number,
-  cb?: Function
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<Expression>;
 export async function evaluateAsync(
-  exp,
-  obj?,
+  exp: Expression,
+  obj?: Record<string, unknown>,
   now?: number,
-  cb?: Function
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<Expression> {
-  return mapAsync(exp, async e => {
+  return mapAsync(exp, async (e) => {
     if (cb) e = await cb(e);
     if (!isArray(e)) return e;
 
@@ -269,59 +339,59 @@ export async function evaluateAsync(
       if (obj && !isArray(e[1])) {
         let v = obj[e[1]];
         if (v == null) return null;
-        if (typeof v === "object") v = v.value ? v.value[0] : null;
-        return v;
+        if (typeof v === "object") v = v["value"] ? v["value"][0] : null;
+        return v as Expression;
       }
     }
-    return evalExp(e);
+    return evaluateCallback(e);
   });
 }
 
 export function and(exp1: Expression, exp2: Expression): Expression {
-  if (!isArray(exp1)) return exp1 ? exp2 : exp1;
-  if (!isArray(exp2)) return exp2 ? exp1 : exp2;
+  if (exp1 != null && !exp1) return false;
+  if (exp2 != null && !exp2) return false;
+  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
+    if (exp1 == null || exp2 == null) return null;
+    return true;
+  }
+  if (!Array.isArray(exp2) && exp1[0] === "AND") return exp1;
+  if (!Array.isArray(exp1) && exp2[0] === "AND") return exp2;
 
-  let res: Expression = ["AND"];
+  const res: Expression = ["AND"];
 
-  if (exp1[0] === "AND") res = res.concat(exp1.slice(1));
+  if (Array.isArray(exp1) && exp1[0] === "AND") res.push(...exp1.slice(1));
   else res.push(exp1);
 
-  if (exp2[0] === "AND") res = res.concat(exp2.slice(1));
+  if (Array.isArray(exp2) && exp2[0] === "AND") res.push(...exp2.slice(1));
   else res.push(exp2);
 
   return res;
 }
 
 export function or(exp1: Expression, exp2: Expression): Expression {
-  if (!isArray(exp1)) return exp1 ? exp1 : exp2;
-  if (!isArray(exp2)) return exp2 ? exp2 : exp1;
+  if (!Array.isArray(exp1) && exp1) return true;
+  if (!Array.isArray(exp2) && exp2) return true;
+  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
+    if (exp1 == null || exp2 == null) return null;
+    return false;
+  }
+  if (!Array.isArray(exp2) && exp1[0] === "OR") return exp1;
+  if (!Array.isArray(exp1) && exp2[0] === "OR") return exp2;
 
-  let res: Expression = ["OR"];
+  const res: Expression = ["OR"];
 
-  if (exp1[0] === "OR") res = res.concat(exp1.slice(1));
+  if (Array.isArray(exp1) && exp1[0] === "OR") res.push(...exp1.slice(1));
   else res.push(exp1);
 
-  if (exp2[0] === "OR") res = res.concat(exp2.slice(1));
+  if (Array.isArray(exp2) && exp2[0] === "OR") res.push(...exp2.slice(1));
   else res.push(exp2);
 
   return res;
 }
 
-export function not(exp): Expression {
-  if (isArray(exp) && exp[0] === "NOT") return exp[1];
-  return ["NOT", exp];
-}
-
-export function subset(exp1, exp2): boolean {
-  const e = evaluate(["NOT", ["OR", ["NOT", exp1], exp2]]);
-  if (!isArray(e)) return !e;
-  const { vars, clauses } = booleanCnf(e);
-  return !naiveDpll(clauses, vars);
-}
-
-export function extractParams(exp): Expression[] {
+export function extractParams(exp: Expression): Expression[] {
   const params = new Set<Expression>();
-  map(exp, e => {
+  map(exp, (e) => {
     if (isArray(e) && e[0] === "PARAM") params.add(e[1]);
     return e;
   });

lib/common/memoize.ts

@@ -39,7 +39,7 @@ function getKey(obj): string {
   return k;
 }
 
-export default function memoize<T extends Function>(func: T): T {
+export default function memoize<T extends (...args: any[]) => any>(func: T): T {
   const funcKey = getKey(func);
   return ((...args) => {
     const key = JSON.stringify(args.map(getKey)) + funcKey;

lib/common/path-set.ts

@@ -73,8 +73,8 @@ export default class PathSet {
 
   public find(
     path: Path,
-    superset: boolean = false,
-    subset: boolean = false,
+    superset = false,
+    subset = false,
     depth: number = path.length
   ): Path[] {
     if (path.alias) throw new Error("PathSet does not support aliased paths");
@@ -106,24 +106,24 @@ export default class PathSet {
       if (!fragmentIndexSet1) {
         if (!fragmentIndexSet2) return [];
         if (!res) res = [...fragmentIndexSet2];
-        else res = res.filter(r => fragmentIndexSet2.has(r));
+        else res = res.filter((r) => fragmentIndexSet2.has(r));
       } else if (!fragmentIndexSet2) {
         if (!res) res = [...fragmentIndexSet1];
-        else res = res.filter(r => fragmentIndexSet1.has(r));
+        else res = res.filter((r) => fragmentIndexSet1.has(r));
       } else {
         if (!res) {
           res = [...fragmentIndexSet1, ...fragmentIndexSet2];
         } else {
           res = res.filter(
-            r => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
+            (r) => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
           );
         }
       }
       if (!res.length) return res;
     }
 
-    if (!res) res = [].concat(...lengthIndex.map(a => [...a]));
-    else res = res.filter(r => lengthIndex.some(a => a.has(r)));
+    if (!res) res = [].concat(...lengthIndex.map((a) => [...a]));
+    else res = res.filter((r) => lengthIndex.some((a) => a.has(r)));
 
     return res;
   }

lib/common/path.ts

@@ -97,6 +97,7 @@ export default class Path {
           )
             ++index;
           const s = pattern.slice(j, index).trim();
+          if (!s) throw new Error("Invalid parameter path");
           segments.push(s);
         }
 
@@ -118,7 +119,7 @@ export default class Path {
       if (Array.isArray(s)) {
         alias |= 1 << i;
         const parts = s.map(
-          al => `${al[0].toString()}:${JSON.stringify(al[1])}`
+          (al) => `${al[0].toString()}:${JSON.stringify(al[1])}`
         );
         return `[${parts.join(",")}]`;
       } else if (s === "*") {
@@ -159,7 +160,7 @@ export default class Path {
     return this._string;
   }
 
-  public slice(start: number = 0, end: number = this.segments.length): Path {
+  public slice(start = 0, end: number = this.segments.length): Path {
     if (start < 0) start = Math.max(0, this.segments.length + start);
     if (end < 0) end = Math.max(0, this.segments.length + end);
 
@@ -211,7 +212,7 @@ export default class Path {
 
   public stripAlias(): Path {
     if (!this.alias) return this;
-    const segments = this.segments.map(s => (Array.isArray(s) ? "*" : s));
+    const segments = this.segments.map((s) => (Array.isArray(s) ? "*" : s));
     const str = segments.join(".");
 
     let path = cache1.get(str);

lib/common/sat-solver.ts

@@ -1,175 +0,0 @@
-/**
- * Naive DPLL solver for propositional logic adapted from
- * https://github.com/tammet/logictools
- *
- * Origianl code by Tanel Tammet released under the MIT license.
- *
- * Check whether a clause set is satisfiable or contradictory, using a naive
- * implementation of the dpll (Davis-Putnam-Logemann-Loveland) procedure
- * combining truth table search with the resolution-like unit propagation.
- *
- * The standard pure-literal rule is not implemented in this naive version,
- * since it is not really crucial for the algorithm.
- *
- * The code is very similar to the pure table solver, except we use a
- * unit-propagating, derived-varlist-returing satisfiableAt.
- */
-
-/* eslint-disable @typescript-eslint/no-use-before-define */
-
-function satisfiableAt(clauses, varVals, varnr, val): boolean {
-  if (varnr !== 0) varVals[varnr] = val;
-
-  const propRes = unitPropagate(clauses, varVals);
-  if (propRes === 1) {
-    varVals[varnr] = 0;
-    return true;
-  }
-
-  // if result false (-1) unitPropagate restores the varVals state itself
-  if (propRes === -1) {
-    varVals[varnr] = 0;
-    return false;
-  }
-
-  // find next unassigned var to split
-  let nextVar = 0;
-  for (let i = 1; i < varVals.length; i++) {
-    if (varVals[i] === 0) {
-      nextVar = i;
-      break;
-    }
-  }
-
-  if (nextVar === 0) throw new Error("Error in satisfiableAt");
-
-  if (
-    satisfiableAt(clauses, varVals, nextVar, 1) ||
-    satisfiableAt(clauses, varVals, nextVar, -1)
-  )
-    // not necessary to restore split var: search is halted anyway
-    return true;
-
-  // restore split var
-  varVals[nextVar] = 0;
-
-  // restore derived vars
-  for (let i = 0; i < propRes.length; i++) varVals[propRes[i]] = 0;
-
-  return false;
-}
-
-/**
- * Check whether clauses are satisfiable under the varVals assignment, without
- * search.
- *
- * Iterate through all clauses and check whether
- *
- *  - all literals in the clause are assigned false: undo derived units (set
- *    values in varVals to 0) and return false immediately
- *
- *  - a clause contains a literal assigned true: skip further actions on this
- *    clause
- *
- *  - a single literal in the clause is unassigned, all the rest are assigned
- *    false: derive this literal as a unit clause.
- *
- * In case at least one literal is derived during the process above, iterate
- * the whole process until no more new literals are derived.
- *
- * Returns:
- *
- *   1: all clauses were assigned true, hence the clause set is satisfiable.
- *   The whole search will terminate after that.
- *
- *   -1: clause set is contradictory. Derived units have been restored.
- *
- *   a list of derived variables:  the status of the clause set is
- *   undetermined. The values of variables on the list will be restored (set to
- *   0) during backtracking.
- */
-function unitPropagate(clauses, varVals): number[] | 1 | -1 {
-  // all vars with a value derived in the forthcoming main loop
-  const derivedVars = [];
-
-  // the main loop is run, deriving new unit clauses, until either:
-  // - the clause set is detected to be unsatisfiable
-  // - the clause set is detected to be satisfiable
-  // - there are no more derived unit clauses
-
-  for (;;) {
-    let allClausesFound = true; // set to false if we find an undetermined clause
-    const derivedQueue = []; // derived units (literals) during one iteration
-    for (let i = 0; i < clauses.length; i++) {
-      const clause = clauses[i];
-      let clauseVal = 0;
-      let unassignedCount = 0; // count unassigned vars in clause
-      let unassignedLit = 0; // 0 means none found for this clause
-
-      for (let j = 0; j < clause.length; j++) {
-        const lit = clause[j];
-        let nr, polarity;
-        if (lit < 0) {
-          nr = 0 - lit;
-          polarity = -1;
-        } else {
-          nr = lit;
-          polarity = 1;
-        }
-
-        if (varVals[nr] === polarity) {
-          clauseVal = 1;
-          break;
-        } else if (varVals[nr] === 0) {
-          unassignedCount++;
-          unassignedLit = lit;
-        }
-      }
-
-      if (clauseVal === 1) continue; // clause is subsumed
-
-      // clause is not subsumed by varVals
-      if (unassignedCount === 0) {
-        for (let j = 0; j < derivedVars.length; j++)
-          varVals[derivedVars[j]] = 0; // restore derived
-        return -1;
-      } else if (unassignedCount === 1) {
-        // unassignedLit is a derived literal
-        derivedQueue.push(unassignedLit);
-      }
-
-      if (unassignedCount !== 0) allClausesFound = false;
-    }
-
-    // if all clauses were subsumed or all the variables in a clause had a value
-    // no need to restore varVals: whole search finished
-    if (allClausesFound) return 1;
-
-    // in case there are any derived units present, store to varVals and
-    // iterate the main loop
-    if (derivedQueue.length === 0) break;
-
-    // some units were derived: iterate the main loop
-    for (let j = 0; j < derivedQueue.length; j++) {
-      const lit = derivedQueue[j];
-      let nr, polarity;
-      if (lit < 0) {
-        nr = 0 - lit;
-        polarity = -1;
-      } else {
-        nr = lit;
-        polarity = 1;
-      }
-      varVals[nr] = polarity;
-      derivedVars.push(nr);
-    }
-  }
-
-  return derivedVars;
-}
-
-export function naiveDpll(clauses, maxvarnr): boolean {
-  // variable values are 0 if not set, 1 if positive, -1 if negative
-  const varVals = new Int32Array(maxvarnr + 1);
-  return satisfiableAt(clauses, varVals, 0, 0);
-}

lib/config.ts

@@ -32,13 +32,13 @@ while (!existsSync(`${ROOT_DIR}/package.json`)) {
 }
 
 // For compatibility with v1.1
-let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl;
+let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl, fsHostname;
 
 const options = {
   EXT_DIR: { type: "path", default: resolve(ROOT_DIR, "config/ext") },
   MONGODB_CONNECTION_URL: {
     type: "string",
-    default: "mongodb://127.0.0.1/genieacs"
+    default: "mongodb://127.0.0.1/genieacs",
   },
 
   CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
@@ -62,7 +62,7 @@ const options = {
   FS_INTERFACE: { type: "string", default: "0.0.0.0" },
   FS_SSL_CERT: { type: "string", default: "" },
   FS_SSL_KEY: { type: "string", default: "" },
-  FS_HOSTNAME: { type: "string", default: "" },
+  FS_URL_PREFIX: { type: "string", default: "" },
   FS_LOG_FILE: { type: "path", default: "" },
   FS_ACCESS_LOG_FILE: { type: "path", default: "" },
 
@@ -76,6 +76,7 @@ const options = {
   UI_JWT_SECRET: { type: "string", default: "" },
 
   UDP_CONNECTION_REQUEST_PORT: { type: "int", default: 0 },
+  FORWARDED_HEADER: { type: "string", default: "" },
 
   DOWNLOAD_TIMEOUT: { type: "int", default: 3600 },
   EXT_TIMEOUT: { type: "int", default: 3000 },
@@ -98,16 +99,8 @@ const options = {
   CONNECTION_REQUEST_ALLOW_BASIC_AUTH: { type: "bool", default: false },
   MAX_COMMIT_ITERATIONS: { type: "int", default: 32 },
 
-  // XML configuration
-  XML_RECOVER: { type: "bool", default: false },
-  XML_IGNORE_ENC: { type: "bool", default: false },
-  XML_FORMAT: { type: "bool", default: false },
-  XML_NO_DECL: { type: "bool", default: false },
-  XML_NO_EMPTY: { type: "bool", default: false },
-  XML_LIBXMLJS: { type: "bool", default: false },
-
   // Should probably never be changed
-  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 }
+  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
 };
 
 const allConfig: { [name: string]: string | number } = {};
@@ -119,37 +112,20 @@ function setConfig(name, value, commandLineArgument = false): boolean {
   if (name === "CONFIG_DIR" || name === "config-dir")
     configDir = configDir || resolve(ROOT_DIR, value);
 
-  if (name === "CWMP_SSL" || name === "cwmp-ssl") {
-    cwmpSsl =
-      cwmpSsl ||
-      String(value)
-        .toLowerCase()
-        .trim();
-  }
+  if (name === "CWMP_SSL" || name === "cwmp-ssl")
+    cwmpSsl = cwmpSsl || String(value).toLowerCase().trim();
 
-  if (name === "NBI_SSL" || name === "nbi-ssl") {
-    nbiSsl =
-      nbiSsl ||
-      String(value)
-        .toLowerCase()
-        .trim();
-  }
+  if (name === "NBI_SSL" || name === "nbi-ssl")
+    nbiSsl = nbiSsl || String(value).toLowerCase().trim();
 
-  if (name === "FS_SSL" || name === "fs-ssl") {
-    fsSsl =
-      fsSsl ||
-      String(value)
-        .toLowerCase()
-        .trim();
-  }
+  if (name === "FS_SSL" || name === "fs-ssl")
+    fsSsl = fsSsl || String(value).toLowerCase().trim();
 
-  if (name === "UI_SSL" || name === "ui-ssl") {
-    uiSsl =
-      uiSsl ||
-      String(value)
-        .toLowerCase()
-        .trim();
-  }
+  if (name === "UI_SSL" || name === "ui-ssl")
+    uiSsl = uiSsl || String(value).toLowerCase().trim();
+
+  if (name === "FS_HOSTNAME" || name === "fs-hostname")
+    fsHostname = fsHostname || String(value).trim();
 
   // For compatibility with v1.0
   if (name === "PRESETS_CACHE_DURATION" || name === "presets-cache-duration")
@@ -167,12 +143,6 @@ function setConfig(name, value, commandLineArgument = false): boolean {
   )
     setConfig("GPV_BATCH_SIZE", value);
 
-  if (name === "XML_PARSE_IGNORE_ENC" || name === "xml-parse-ignore-enc")
-    setConfig("XML_IGNORE_ENC", value);
-
-  if (name === "XML_PARSE_RECOVER" || name === "xml-parse-recover")
-    setConfig("XML_RECOVER", value);
-
   if (name === "FS_IP" || name === "fs-ip") setConfig("FS_HOSTNAME", value);
 
   function cast(val, type): string | number | boolean {
@@ -180,11 +150,7 @@ function setConfig(name, value, commandLineArgument = false): boolean {
       case "int":
         return Number(val);
       case "bool":
-        return ["true", "1"].includes(
-          String(val)
-            .trim()
-            .toLowerCase()
-        );
+        return ["true", "1"].includes(String(val).trim().toLowerCase());
       case "string":
         return String(val);
       case "path":
@@ -274,6 +240,15 @@ if (["true", "1"].includes(uiSsl)) {
   setConfig("UI_SSL_KEY", `${d}/cwmp.key`);
 }
 
+if (fsHostname) {
+  const FS_PORT = allConfig["FS_PORT"] || 7567;
+  const FS_SSL = !!allConfig["FS_SSL_CERT"];
+  setConfig(
+    "FS_URL_PREFIX",
+    (FS_SSL ? "https" : "http") + `://${fsHostname}:${FS_PORT}/`
+  );
+}
+
 // Defaults
 for (const [k, v] of Object.entries(options))
   if (v["default"] != null) setConfig(k, v["default"]);
@@ -309,7 +284,7 @@ export function get(
   return null;
 }
 
-export function getDefault(optionName): string | number | boolean {
+export function getDefault(optionName: string): string | number | boolean {
   const option = options[optionName];
   if (!option) return null;
 

lib/connection-request.ts

@@ -29,15 +29,13 @@ import * as debug from "./debug";
 
 async function extractAuth(
   exp: Expression,
-  context: {},
-  now: number,
   dflt: any
 ): Promise<[string, string, Expression]> {
   let username, password;
   const _exp = await evaluateAsync(
     exp,
-    context,
-    now,
+    {},
+    0,
     async (e: Expression): Promise<Expression> => {
       if (!username && Array.isArray(e) && e[0] === "FUNC") {
         if (e[1] === "EXT") {
@@ -50,8 +48,8 @@ async function extractAuth(
           return fault ? null : value;
         } else if (e[1] === "AUTH") {
           if (!Array.isArray(e[2]) && !Array.isArray(e[3])) {
-            username = e[2];
-            password = e[3];
+            username = e[2] || "";
+            password = e[3] || "";
           }
           return dflt;
         }
@@ -64,13 +62,12 @@ async function extractAuth(
 
 function httpGet(
   options: http.RequestOptions,
-  timeout,
   _debug: boolean,
   deviceId: string
-): Promise<{ statusCode: number; headers: {} }> {
+): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
   return new Promise((resolve, reject) => {
     const req = http
-      .get(options, res => {
+      .get(options, (res) => {
         res.resume();
         resolve({ statusCode: res.statusCode, headers: res.headers });
         if (_debug) {
@@ -78,16 +75,13 @@ function httpGet(
           debug.incomingHttpResponse(res, deviceId, null);
         }
       })
-      .on("error", err => {
-        req.abort();
-        reject(new Error("Device is offline"));
+      .on("error", (err) => {
+        req.destroy();
+        reject(err);
         if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
       })
-      .on("socket", socket => {
-        socket.setTimeout(timeout);
-        socket.on("timeout", () => {
-          req.abort();
-        });
+      .on("timeout", () => {
+        req.destroy();
       });
   });
 }
@@ -95,23 +89,22 @@ function httpGet(
 export async function httpConnectionRequest(
   address: string,
   authExp: Expression,
-  context: {},
   allowBasicAuth: boolean,
   timeout: number,
   _debug: boolean,
   deviceId: string
-): Promise<void> {
-  const now = Date.now();
+): Promise<string> {
   const options: http.RequestOptions = parse(address);
   if (options.protocol !== "http:")
-    throw new Error("Invalid connection request URL or protocol");
+    return "Invalid connection request URL or protocol";
 
   options.agent = new http.Agent({
     maxSockets: 1,
-    keepAlive: true
+    keepAlive: true,
+    timeout: timeout,
   });
 
-  let authHeader: {};
+  let authHeader: Record<string, string>;
   let username: string;
   let password: string;
 
@@ -119,14 +112,13 @@ export async function httpConnectionRequest(
     let opts = options;
     if (authHeader) {
       if (authHeader["method"] === "Basic") {
-        if (!allowBasicAuth)
-          throw new Error("Basic HTTP authentication not allowed");
+        if (!allowBasicAuth) return "Basic HTTP authentication not allowed";
 
         opts = Object.assign(
           {
             headers: {
-              Authorization: auth.basic(username || "", password || "")
-            }
+              Authorization: auth.basic(username || "", password || ""),
+            },
           },
           options
         );
@@ -141,48 +133,58 @@ export async function httpConnectionRequest(
                 "GET",
                 null,
                 authHeader
-              )
-            }
+              ),
+            },
           },
           options
         );
       } else {
-        throw new Error("Unrecognized auth method");
+        return "Unrecognized auth method";
       }
     }
 
-    let res = await httpGet(opts, timeout, _debug, deviceId);
+    let res: { statusCode: number; headers: http.IncomingHttpHeaders };
+    try {
+      res = await httpGet(opts, _debug, deviceId);
+    } catch (err) {
+      // Workaround for some devices unexpectedly closing the connection
+      if (authHeader) {
+        try {
+          res = await httpGet(opts, _debug, deviceId);
+        } catch (err) {
+          return `Connection request error: ${err.message}`;
+        }
+      }
 
-    // Workaround for some devices unexpectedly closing the connection
-    if (res.statusCode === 0 && authHeader)
-      res = await httpGet(opts, timeout, _debug, deviceId);
-    if (res.statusCode === 0) throw new Error("Device is offline");
-    if (res.statusCode === 200 || res.statusCode === 204) return;
+      if (err["code"] === "ECONNRESET" || err["code"] === "ECONNREFUSED")
+        return "Device is offline";
+
+      return `Connection request error: ${err.message}`;
+    }
+
+    if (res.statusCode === 200 || res.statusCode === 204) return "";
+
+    // When a Connection Request is received for the Virtual CWMP Device and the Proxied
+    // Device is offline the CPE Proxier MUST respond with an HTTP 503 failure
+    if (res.statusCode === 503) return "Device is offline";
 
     if (res.statusCode === 401 && res.headers["www-authenticate"]) {
       authHeader = auth.parseWwwAuthenticateHeader(
         res.headers["www-authenticate"]
       );
-      [username, password, authExp] = await extractAuth(
-        authExp,
-        context,
-        now,
-        false
-      );
+      [username, password, authExp] = await extractAuth(authExp, false);
     } else {
-      throw new Error(
-        `Unexpected response code from device: ${res.statusCode}`
-      );
+      return `Connection request error: Unexpected status code ${res.statusCode}`;
     }
   }
-  throw new Error("Incorrect connection request credentials");
+
+  return "Connection request error: Incorrect connection request credentials";
 }
 
 export async function udpConnectionRequest(
   address: string,
   authExp: Expression,
-  context: {},
-  sourcePort: number = 0,
+  sourcePort = 0,
   _debug: boolean,
   deviceId: string
 ): Promise<void> {
@@ -202,12 +204,7 @@ export async function udpConnectionRequest(
   let username: string;
   let password: string;
 
-  [username, password, authExp] = await extractAuth(
-    authExp,
-    context,
-    now,
-    null
-  );
+  [username, password, authExp] = await extractAuth(authExp, null);
 
   if (username == null) username = "";
   if (password == null) password = "";
@@ -233,12 +230,7 @@ export async function udpConnectionRequest(
       });
     }
 
-    [username, password, authExp] = await extractAuth(
-      authExp,
-      context,
-      now,
-      null
-    );
+    [username, password, authExp] = await extractAuth(authExp, null);
   }
   client.close();
 }

lib/db.ts

@@ -17,9 +17,9 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, ObjectID, Collection } from "mongodb";
+import { MongoClient, ObjectID, Collection, Db, GridFSBucket } from "mongodb";
 import { get } from "./config";
-import { escapeRegExp } from "./common";
+import { encodeTag, escapeRegExp } from "./common";
 import { parse } from "./common/expression-parser";
 import {
   DeviceData,
@@ -27,7 +27,7 @@ import {
   SessionFault,
   Task,
   Operation,
-  Expression
+  Expression,
 } from "./types";
 import Path from "./common/path";
 
@@ -39,13 +39,14 @@ export let tasksCollection: Collection,
   virtualParametersCollection: Collection,
   faultsCollection: Collection,
   filesCollection: Collection,
+  uploadsCollection: Collection,
   operationsCollection: Collection,
   permissionsCollection: Collection,
   usersCollection: Collection,
-  configCollection: Collection;
+  configCollection: Collection,
+  uploadsBucket: GridFSBucket;
 
 let clientPromise: Promise<MongoClient>;
-export let client: MongoClient;
 
 function compareAccessLists(list1: string[], list2: string[]): boolean {
   if (list1.length !== list2.length) return false;
@@ -53,14 +54,24 @@ function compareAccessLists(list1: string[], list2: string[]): boolean {
   return true;
 }
 
+const onConnectCallbacks: ((db: Db) => Promise<void>)[] = [];
+
+export function onConnect(callback: (db: Db) => Promise<void>): void {
+  onConnectCallbacks.push(callback);
+}
+
 export async function connect(): Promise<void> {
   clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
-    useNewUrlParser: true
+    useNewUrlParser: true,
+    useUnifiedTopology: true,
   });
 
-  client = await clientPromise;
+  const client = await clientPromise;
   const db = client.db();
+  await Promise.all(onConnectCallbacks.map((c) => c(db)));
+}
 
+onConnect(async (db) => {
   tasksCollection = db.collection("tasks");
   await tasksCollection.createIndex({ device: 1, timestamp: 1 });
 
@@ -68,6 +79,7 @@ export async function connect(): Promise<void> {
   presetsCollection = db.collection("presets");
   objectsCollection = db.collection("objects");
   filesCollection = db.collection("fs.files");
+  uploadsCollection = db.collection("uploads.files");
   provisionsCollection = db.collection("provisions");
   virtualParametersCollection = db.collection("virtualParameters");
   faultsCollection = db.collection("faults");
@@ -75,7 +87,8 @@ export async function connect(): Promise<void> {
   permissionsCollection = db.collection("permissions");
   usersCollection = db.collection("users");
   configCollection = db.collection("config");
-}
+  uploadsBucket = new GridFSBucket(db, { bucketName: "uploads" });
+});
 
 export async function disconnect(): Promise<void> {
   if (clientPromise) await (await clientPromise).close();
@@ -83,7 +96,7 @@ export async function disconnect(): Promise<void> {
 
 // Optimize projection by removing overlaps
 // This can modify the object
-function optimizeProjection(obj: {}): {} {
+function optimizeProjection(obj: { [path: string]: 1 }): { [path: string]: 1 } {
   if (obj[""]) return { "": obj[""] };
 
   const keys = Object.keys(obj).sort();
@@ -110,13 +123,13 @@ export async function fetchDevice(
     [
       Path.parse("Events"),
       timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 0] }
+      { object: [timestamp, 1], writable: [timestamp, 0] },
     ],
     [
       Path.parse("DeviceID"),
       timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 0] }
-    ]
+      { object: [timestamp, 1], writable: [timestamp, 0] },
+    ],
   ];
 
   const device = await devicesCollection.findOne({ _id: id });
@@ -146,7 +159,7 @@ export async function fetchDevice(
     if (obj["_notification"] != null) {
       attrs.notification = [
         obj["_attributesTimestamp"] || 1,
-        obj["_notification"]
+        obj["_notification"],
       ];
     }
 
@@ -175,8 +188,8 @@ export async function fetchDevice(
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]]
-          }
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
         ]);
         break;
       case "_lastBoot":
@@ -186,8 +199,8 @@ export async function fetchDevice(
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]]
-          }
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
         ]);
         break;
       case "_lastBootstrap":
@@ -197,8 +210,8 @@ export async function fetchDevice(
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]]
-          }
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
         ]);
         break;
       case "_registered":
@@ -209,8 +222,8 @@ export async function fetchDevice(
           {
             object: [timestamp, 0],
             writable: [timestamp, 0],
-            value: [timestamp, [+v, "xsd:dateTime"]]
-          }
+            value: [timestamp, [+v, "xsd:dateTime"]],
+          },
         ]);
         break;
       case "_id":
@@ -220,8 +233,8 @@ export async function fetchDevice(
           {
             object: [timestamp, 0],
             writable: [timestamp, 0],
-            value: [timestamp, [v as string, "xsd:string"]]
-          }
+            value: [timestamp, [v as string, "xsd:string"]],
+          },
         ]);
         break;
       case "_tags":
@@ -229,20 +242,19 @@ export async function fetchDevice(
           res.push([
             Path.parse("Tags"),
             timestamp,
-            { object: [timestamp, 1], writable: [timestamp, 0] }
+            { object: [timestamp, 1], writable: [timestamp, 0] },
           ]);
         }
 
-        for (let t of v as string[]) {
-          t = t.replace(/[^a-zA-Z0-9-]+/g, "_");
+        for (const t of v as string[]) {
           res.push([
-            Path.parse("Tags." + t),
+            Path.parse("Tags." + encodeTag(t)),
             timestamp,
             {
               object: [timestamp, 0],
               writable: [timestamp, 1],
-              value: [timestamp, [true, "xsd:boolean"]]
-            }
+              value: [timestamp, [true, "xsd:boolean"]],
+            },
           ]);
         }
         break;
@@ -254,8 +266,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_Manufacturer"], "xsd:string"]]
-            }
+              value: [timestamp, [v["_Manufacturer"], "xsd:string"]],
+            },
           ]);
         }
 
@@ -266,8 +278,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_OUI"], "xsd:string"]]
-            }
+              value: [timestamp, [v["_OUI"], "xsd:string"]],
+            },
           ]);
         }
 
@@ -278,8 +290,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_ProductClass"], "xsd:string"]]
-            }
+              value: [timestamp, [v["_ProductClass"], "xsd:string"]],
+            },
           ]);
         }
 
@@ -290,8 +302,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_SerialNumber"], "xsd:string"]]
-            }
+              value: [timestamp, [v["_SerialNumber"], "xsd:string"]],
+            },
           ]);
         }
     }
@@ -422,7 +434,7 @@ export async function saveDevice(
           } else {
             if (!update["$pull"]["_tags"]) {
               update["$pull"]["_tags"] = {
-                $in: []
+                $in: [],
               };
             }
             update["$pull"]["_tags"]["$in"].push(path.segments[1]);
@@ -431,6 +443,17 @@ export async function saveDevice(
 
         break;
       default:
+        if (
+          diff[0].segments[0] === "Uploads" &&
+          diff[0].segments[2] === "LastFileName" &&
+          value1 &&
+          value1 !== value2
+        ) {
+          uploadsBucket.delete(value1 as any, () => {
+            // Ignore error due to mising files
+          });
+        }
+
         if (!diff[2]) {
           update["$unset"][path.toString()] = 1;
           continue;
@@ -580,7 +603,7 @@ export async function saveDevice(
   }
 
   const result = await devicesCollection.updateOne({ _id: deviceId }, update, {
-    upsert: isNew
+    upsert: isNew,
   });
 
   if (result.result.n !== 1)
@@ -593,7 +616,7 @@ export async function saveDevice(
 }
 
 export async function getFaults(
-  deviceId
+  deviceId: string
 ): Promise<{ [channel: string]: SessionFault }> {
   const res = await faultsCollection
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
@@ -613,26 +636,33 @@ export async function getFaults(
   return faults;
 }
 
-export async function saveFault(deviceId, channel, fault): Promise<void> {
+export async function saveFault(
+  deviceId: string,
+  channel: string,
+  fault: SessionFault
+): Promise<void> {
   const id = `${deviceId}:${channel}`;
-  fault = Object.assign({}, fault);
-  fault._id = id;
-  fault.device = deviceId;
-  fault.channel = channel;
-  fault.timestamp = new Date(fault.timestamp);
-  fault.provisions = JSON.stringify(fault.provisions);
-  await faultsCollection.replaceOne({ _id: id }, fault, { upsert: true });
+  const f = Object.assign({}, fault) as Record<string, any>;
+  f["_id"] = id;
+  f["device"] = deviceId;
+  f["channel"] = channel;
+  f["timestamp"] = new Date(fault.timestamp);
+  f["provisions"] = JSON.stringify(fault.provisions);
+  await faultsCollection.replaceOne({ _id: id }, f, { upsert: true });
 }
 
-export async function deleteFault(deviceId, channel): Promise<void> {
+export async function deleteFault(
+  deviceId: string,
+  channel: string
+): Promise<void> {
   await faultsCollection.deleteOne({ _id: `${deviceId}:${channel}` });
 }
 
 export async function getDueTasks(
-  deviceId,
-  timestamp
+  deviceId: string,
+  timestamp: number
 ): Promise<[Task[], number]> {
-  const cur = tasksCollection.find({ device: deviceId }).sort(["timestamp"]);
+  const cur = tasksCollection.find({ device: deviceId }).sort({ timestamp: 1 });
   const tasks = [] as Task[];
 
   for await (const task of cur) {
@@ -663,14 +693,17 @@ export async function getDueTasks(
   return [tasks, null];
 }
 
-export async function clearTasks(deviceId, taskIds): Promise<void> {
+export async function clearTasks(
+  deviceId: string,
+  taskIds: string[]
+): Promise<void> {
   await tasksCollection.deleteMany({
-    _id: { $in: taskIds.map(id => new ObjectID(id)) }
+    _id: { $in: taskIds.map((id) => new ObjectID(id)) },
   });
 }
 
 export async function getOperations(
-  deviceId
+  deviceId: string
 ): Promise<{ [commandKey: string]: Operation }> {
   const res = await operationsCollection
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
@@ -680,6 +713,11 @@ export async function getOperations(
   for (const r of res) {
     const commandKey = r._id.slice(deviceId.length + 1);
     delete r._id;
+    // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
+    if (typeof r.provisions !== "string") {
+      operations[commandKey] = r;
+      continue;
+    }
     r.timestamp = +r.timestamp;
     if (r.args) r.args = JSON.parse(r.args);
     r.provisions = JSON.parse(r.provisions);
@@ -690,43 +728,46 @@ export async function getOperations(
 }
 
 export async function saveOperation(
-  deviceId,
-  commandKey,
-  operation
+  deviceId: string,
+  commandKey: string,
+  operation: Operation
 ): Promise<void> {
   const id = `${deviceId}:${commandKey}`;
-  operation = Object.assign({}, operation);
-  operation._id = id;
-  operation.timestamp = new Date(operation.timestamp);
-  operation.provisions = JSON.stringify(operation.provisions);
-  operation.retries = JSON.stringify(operation.retries);
-  operation.args = JSON.stringify(operation.args);
-  await operationsCollection.replaceOne({ _id: id }, operation, {
-    upsert: true
+  const o = Object.assign({}, operation) as Record<string, any>;
+  o["_id"] = id;
+  o["timestamp"] = new Date(operation.timestamp);
+  o["provisions"] = JSON.stringify(operation.provisions);
+  o["retries"] = JSON.stringify(operation.retries);
+  o["args"] = JSON.stringify(operation.args);
+  await operationsCollection.replaceOne({ _id: id }, o, {
+    upsert: true,
   });
 }
 
-export async function deleteOperation(deviceId, commandKey): Promise<void> {
+export async function deleteOperation(
+  deviceId: string,
+  commandKey: string
+): Promise<void> {
   await operationsCollection.deleteOne({ _id: `${deviceId}:${commandKey}` });
 }
 
-export async function getPresets(): Promise<{}[]> {
+export async function getPresets(): Promise<Record<string, any>[]> {
   return presetsCollection.find().toArray();
 }
 
-export async function getObjects(): Promise<{}[]> {
+export async function getObjects(): Promise<Record<string, any>[]> {
   return objectsCollection.find().toArray();
 }
 
-export async function getProvisions(): Promise<{}[]> {
+export async function getProvisions(): Promise<Record<string, any>[]> {
   return provisionsCollection.find().toArray();
 }
 
-export async function getVirtualParameters(): Promise<{}[]> {
+export async function getVirtualParameters(): Promise<Record<string, any>[]> {
   return virtualParametersCollection.find().toArray();
 }
 
-export function getFiles(): Promise<{}[]> {
+export function getFiles(): Promise<Record<string, any>[]> {
   return filesCollection.find().toArray();
 }
 
@@ -734,9 +775,9 @@ export async function getConfig(): Promise<
   { id: string; value: Expression }[]
 > {
   const res = await configCollection.find().toArray();
-  return res.map(c => ({
+  return res.map((c) => ({
     id: c["_id"],
-    value: parse(c["value"])
+    value: parse(c["value"]),
   }));
 }
 
@@ -762,3 +803,14 @@ interface User {
 export async function getUsers(): Promise<User[]> {
   return usersCollection.find().toArray();
 }
+
+export async function deleteDeviceUploads(deviceId: string): Promise<void> {
+  const files = await uploadsCollection
+    .find({
+      _id: {
+        $regex: `^${escapeRegExp(deviceId)}\\/`,
+      },
+    })
+    .toArray();
+  await Promise.all(files.map((f) => uploadsBucket.delete(f["_id"])));
+}

lib/debug.ts

@@ -21,7 +21,7 @@ import {
   IncomingMessage,
   ServerResponse,
   ClientRequest,
-  RequestOptions
+  RequestOptions,
 } from "http";
 import { Socket } from "net";
 import { appendFileSync } from "fs";
@@ -49,7 +49,7 @@ export function incomingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
   const msg = {
     event: "incoming HTTP request",
     timestamp: now,
@@ -60,7 +60,7 @@ export function incomingHttpRequest(
     method: httpRequest.method,
     url: httpRequest.url,
     headers: httpRequest.headers,
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -77,7 +77,7 @@ export function outgoingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
   const msg = {
     event: "outgoing HTTP response",
     timestamp: now,
@@ -86,7 +86,7 @@ export function outgoingHttpResponse(
     connection: getConnectionTimestamp(con),
     statusCode: httpResponse.statusCode,
     headers: httpResponse.getHeaders(),
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -104,7 +104,7 @@ export function outgoingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
@@ -115,7 +115,7 @@ export function outgoingHttpRequest(
     method: options.method || "GET",
     url: options.path,
     headers: httpRequest.getHeaders(),
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -143,7 +143,7 @@ export function outgoingHttpRequestError(
     method: options.method,
     url: options.path,
     headers: httpRequest.getHeaders(),
-    error: err.message
+    error: err.message,
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -160,16 +160,16 @@ export function incomingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
   const msg = {
     event: "incoming HTTP response",
     timestamp: now,
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
-    connection: getConnectionTimestamp(httpResponse.connection),
+    connection: getConnectionTimestamp(httpResponse.socket),
     statusCode: httpResponse.statusCode,
     headers: httpResponse.headers,
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -193,7 +193,7 @@ export function outgoingUdpMessage(
     remoteAddress: remoteAddress,
     deviceId: deviceId,
     remotePort: remotePort,
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")

lib/default-provisions.ts

@@ -21,10 +21,15 @@ import Path from "./common/path";
 import * as config from "./config";
 import * as device from "./device";
 import * as scheduling from "./scheduling";
+import { SessionContext, Declaration } from "./types";
 
 const MAX_DEPTH = +config.get("MAX_DEPTH");
 
-export function refresh(sessionContext, provision, declarations): boolean {
+export function refresh(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
   if (
     (provision.length !== 2 || typeof provision[1] !== "string") &&
     (provision.length !== 3 ||
@@ -37,7 +42,7 @@ export function refresh(sessionContext, provision, declarations): boolean {
   )
     throw new Error("Invalid arguments");
 
-  const every = 1000 * (provision[2] || 1);
+  const every = 1000 * ((provision[2] as number) || 1);
   const offset = scheduling.variance(sessionContext.deviceId, every);
   const t = scheduling.interval(sessionContext.timestamp, every, offset);
 
@@ -49,7 +54,7 @@ export function refresh(sessionContext, provision, declarations): boolean {
   } else {
     attrGet = {};
     refreshChildren = !!provision[3];
-    for (const a of provision.slice(4)) attrGet[a] = t;
+    for (const a of provision.slice(4)) attrGet[a as string] = t;
   }
 
   let path = Path.parse(provision[1]);
@@ -69,14 +74,18 @@ export function refresh(sessionContext, provision, declarations): boolean {
       pathSet: null,
       attrGet: attrGet,
       attrSet: null,
-      defer: true
+      defer: true,
     });
   }
 
   return true;
 }
 
-export function value(sessionContext, provision, declarations): boolean {
+export function value(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
   if (
     provision.length < 3 ||
     provision.length > 4 ||
@@ -90,15 +99,15 @@ export function value(sessionContext, provision, declarations): boolean {
     attr = "value";
     val = provision[2];
   } else {
-    attr = provision[2] || "";
+    attr = (provision[2] as string) || "";
     val = provision[3];
   }
 
   if (attr === "accessList") {
     val = (val || "")
       .split(",")
-      .map(s => s.trim())
-      .filter(s => !!s);
+      .map((s) => s.trim())
+      .filter((s) => !!s);
   } else if (attr === "value") {
     val = [val];
   }
@@ -109,13 +118,17 @@ export function value(sessionContext, provision, declarations): boolean {
     pathSet: null,
     attrGet: { [attr]: 1 },
     attrSet: { [attr]: val },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
-export function tag(sessionContext, provision, declarations): boolean {
+export function tag(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
   if (
     provision.length !== 3 ||
     typeof provision[1] !== "string" ||
@@ -129,13 +142,17 @@ export function tag(sessionContext, provision, declarations): boolean {
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [provision[2]] },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
-export function reboot(sessionContext, provision, declarations): boolean {
+export function reboot(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
   if (provision.length !== 1) throw new Error("Invalid arguments");
 
   declarations.push({
@@ -144,13 +161,17 @@ export function reboot(sessionContext, provision, declarations): boolean {
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
-export function reset(sessionContext, provision, declarations): boolean {
+export function reset(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
   if (provision.length !== 1) throw new Error("Invalid arguments");
 
   declarations.push({
@@ -159,13 +180,17 @@ export function reset(sessionContext, provision, declarations): boolean {
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
-export function download(sessionContext, provision, declarations): boolean {
+export function download(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
   if (
     (provision.length !== 3 ||
       typeof provision[1] !== "string" ||
@@ -180,7 +205,7 @@ export function download(sessionContext, provision, declarations): boolean {
   const alias = [
     `FileType:${JSON.stringify(provision[1] || "")}`,
     `FileName:${JSON.stringify(provision[2] || "")}`,
-    `TargetFileName:${JSON.stringify(provision[3] || "")}`
+    `TargetFileName:${JSON.stringify(provision[3] || "")}`,
   ].join(",");
 
   declarations.push({
@@ -189,7 +214,7 @@ export function download(sessionContext, provision, declarations): boolean {
     pathSet: 1,
     attrGet: null,
     attrSet: null,
-    defer: true
+    defer: true,
   });
 
   declarations.push({
@@ -198,18 +223,56 @@ export function download(sessionContext, provision, declarations): boolean {
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true
+    defer: true,
+  });
+
+  return true;
+}
+
+export function upload(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (
+    provision.length !== 3 ||
+    typeof provision[1] !== "string" ||
+    typeof provision[2] !== "string"
+  )
+    throw new Error("Invalid arguments");
+
+  const alias = [
+    `FileType:${JSON.stringify(provision[1] || "")}`,
+    `FileName:${JSON.stringify(provision[2] || "")}`,
+  ].join(",");
+
+  declarations.push({
+    path: Path.parse(`Uploads.[${alias}]`),
+    pathGet: 1,
+    pathSet: 1,
+    attrGet: null,
+    attrSet: null,
+    defer: true,
+  });
+
+  declarations.push({
+    path: Path.parse(`Uploads.[${alias}].Upload`),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true,
   });
 
   return true;
 }
 
 export function instances(
-  sessionContext,
-  provision,
-  declarations,
-  startRevision,
-  endRevision
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+  startRevision: number,
+  endRevision: number
 ): boolean {
   if (provision.length !== 3 || typeof provision[1] !== "string")
     throw new Error("Invalid arguments");
@@ -227,7 +290,7 @@ export function instances(
       pathSet: null,
       attrGet: null,
       attrSet: null,
-      defer: true
+      defer: true,
     });
 
     if (endRevision === startRevision) return false;
@@ -246,7 +309,7 @@ export function instances(
     pathSet: count,
     attrGet: null,
     attrSet: null,
-    defer: true
+    defer: true,
   });
 
   return true;

lib/device.ts

@@ -18,14 +18,20 @@
  */
 
 import Path from "./common/path";
-import { DeviceData, Declaration, Clear } from "./types";
+import {
+  DeviceData,
+  Declaration,
+  Clear,
+  AttributeTimestamps,
+  Attributes,
+} from "./types";
 
 const CHANGE_FLAGS = {
   object: 2,
   writable: 4,
   value: 8,
   notification: 16,
-  accessList: 32
+  accessList: 32,
 };
 
 function parseBool(v): boolean {
@@ -38,7 +44,7 @@ function parseBool(v): boolean {
 
 export function sanitizeParameterValue(
   parameterValue: [string | number | boolean, string]
-): [string | number | boolean, string?] {
+): [string | number | boolean, string] {
   if (parameterValue[0] != null) {
     switch (parameterValue[1]) {
       case "xsd:boolean":
@@ -98,8 +104,8 @@ export function getAliasDeclarations(
       pathSet: null,
       attrGet: attrGet,
       attrSet: null,
-      defer: true
-    }
+      defer: true,
+    },
   ];
 
   if (path.alias) {
@@ -109,7 +115,7 @@ export function getAliasDeclarations(
         for (const [p] of alias as [Path, string][]) {
           decs = decs.concat(
             getAliasDeclarations(parent.concat(p), timestamp, {
-              value: timestamp
+              value: timestamp,
             })
           );
         }
@@ -191,10 +197,10 @@ export function unpack(
 }
 
 export function clear(
-  deviceData,
-  path,
-  timestamp,
-  attributes,
+  deviceData: DeviceData,
+  path: Path,
+  timestamp: number,
+  attributes: AttributeTimestamps,
   changeFlags = 0
 ): void {
   const changeTrackers = {};
@@ -273,8 +279,8 @@ function compareEquality(a, b): boolean {
 export function set(
   deviceData: DeviceData,
   path: Path,
-  timestamp,
-  attributes,
+  timestamp: number,
+  attributes: Attributes,
   toClear?: Clear[]
 ): Clear[] {
   path = deviceData.paths.add(path);
@@ -302,7 +308,7 @@ export function set(
       attributes.object[1] &&
       attributes.object[0] >= (attributes.value ? attributes.value[0] : 0)
     )
-      attributes.value = [attributes.object[0]];
+      attributes.value = [attributes.object[0], null];
 
     const newAttributes = Object.assign({}, currentAttributes, attributes);
 
@@ -378,7 +384,7 @@ export function track(
   deviceData: DeviceData,
   path: Path,
   marker: string,
-  attributes?
+  attributes?: string[]
 ): void {
   path = deviceData.paths.add(path);
   let f = 1;
@@ -395,7 +401,10 @@ export function track(
   cur[marker] |= f;
 }
 
-export function clearTrackers(deviceData: DeviceData, tracker): void {
+export function clearTrackers(
+  deviceData: DeviceData,
+  tracker: string | string[]
+): void {
   if (Array.isArray(tracker)) {
     for (const v of deviceData.trackers.values())
       for (const t of tracker) delete v[t];

lib/extensions.ts

@@ -36,8 +36,8 @@ function messageHandler(message): void {
   }
 }
 
-export function run(args): Promise<{ fault: Fault; value: any }> {
-  return new Promise(resolve => {
+export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
+  return new Promise((resolve) => {
     const scriptName = args[0];
 
     const id = crypto.randomBytes(8).toString("hex");
@@ -45,19 +45,20 @@ export function run(args): Promise<{ fault: Fault; value: any }> {
 
     if (!processes[scriptName]) {
       const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
-        stdio: ["inherit", "inherit", "inherit", "ipc"]
+        stdio: ["inherit", "inherit", "inherit", "ipc"],
       });
       processes[scriptName] = p;
 
-      p.on("error", err => {
+      p.on("error", (err) => {
         if (processes[scriptName] === p) {
           if (jobs.delete(id)) {
             resolve({
               fault: { code: err.name, message: err.message },
-              value: null
+              value: null,
             });
           }
 
+          // eslint-disable-next-line @typescript-eslint/no-floating-promises
           kill(processes[scriptName]);
           delete processes[scriptName];
         }
@@ -74,7 +75,7 @@ export function run(args): Promise<{ fault: Fault; value: any }> {
       if (jobs.delete(id)) {
         resolve({
           fault: { code: "timeout", message: "Extension timed out" },
-          value: null
+          value: null,
         });
       }
     }, TIMEOUT);
@@ -86,7 +87,7 @@ export function run(args): Promise<{ fault: Fault; value: any }> {
 }
 
 function kill(process: ChildProcess): Promise<void> {
-  return new Promise(resolve => {
+  return new Promise((resolve) => {
     const timeToKill = Date.now() + 5000;
 
     process.kill();

lib/forwarded.ts

@@ -0,0 +1,175 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { IncomingMessage } from "http";
+import { TLSSocket } from "tls";
+import { parseCIDR, parse, IPv6, IPv4 } from "ipaddr.js";
+import * as config from "./config";
+
+interface RequestOrigin {
+  localAddress: string;
+  localPort: number;
+  remoteAddress: string;
+  remotePort: number;
+  host: string;
+  encrypted: boolean;
+}
+
+const FORWARDED_HEADER = "" + config.get("FORWARDED_HEADER");
+const cache = new WeakMap<IncomingMessage, RequestOrigin>();
+const cidrs: [IPv4 | IPv6, number][] = [];
+
+for (const str of FORWARDED_HEADER.split(",").map((s) => s.trim())) {
+  try {
+    cidrs.push(parseCIDR(str));
+  } catch (err) {
+    // Not a valid CIDR format, try parsing as IP
+    try {
+      const ip = parse(str);
+      cidrs.push([ip, ip.toByteArray().length * 8]);
+    } catch (err) {
+      // Not a valid IP either, ignore
+    }
+  }
+}
+
+function parseForwardedHeader(str: string): { [name: string]: string } {
+  str = str.toLowerCase();
+  const res: { [name: string]: string } = {};
+  let keyIdx = 0;
+  let valueIdx = -1;
+  let key: string;
+  for (let i = 0; i < str.length; ++i) {
+    const char = str.charCodeAt(i);
+    if (char === 61 /* = */) {
+      if (keyIdx >= 0) {
+        key = str.slice(keyIdx, i).trim();
+        keyIdx = -1;
+        valueIdx = i + 1;
+      }
+    } else if (char === 59 /* ; */) {
+      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
+      valueIdx = -1;
+      keyIdx = i + 1;
+    } else if (char === 44 /* , */) {
+      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
+      return res;
+    } else if (char === 34 /* " */) {
+      if (valueIdx >= 0) {
+        const quoteIdx = i;
+        if (!str.slice(valueIdx, quoteIdx).trim()) {
+          for (i = i + 1; i < str.length; ++i) {
+            const c = str.charCodeAt(i);
+            if (c === 92 /* \ */) ++i;
+            if (c === 34 /* " */) {
+              res[key] = JSON.parse(str.slice(quoteIdx, i + 1).trim());
+              valueIdx = -1;
+              keyIdx = i + 1;
+              break;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  if (valueIdx >= 0) res[key] = str.slice(valueIdx).trim();
+
+  return res;
+}
+
+export function getRequestOrigin(request: IncomingMessage): RequestOrigin {
+  let origin = cache.get(request);
+  if (!origin) {
+    const soc = request.socket;
+    origin = {
+      localAddress: soc.localAddress,
+      localPort: soc.localPort,
+      remoteAddress: soc.remoteAddress,
+      remotePort: soc.remotePort,
+      host: request.headers["host"],
+      encrypted: !!(request.socket as TLSSocket).encrypted,
+    };
+
+    const header = request.headers["forwarded"];
+    if (header) {
+      const ip = parse(soc.remoteAddress) as IPv4;
+      if (cidrs.some((cidr) => ip.match(cidr as [IPv4, number]))) {
+        const parsed = parseForwardedHeader(header);
+
+        if (parsed["proto"] === "https") {
+          origin.encrypted = true;
+          origin.localPort = 443;
+        } else if (parsed["proto"] === "http") {
+          origin.encrypted = false;
+          origin.localPort = 80;
+        }
+
+        if (parsed["host"]) {
+          origin.host = parsed["host"];
+          const [, port] = parsed["host"].split(":", 2);
+          origin.localPort = +port || origin.localPort;
+        }
+
+        if (parsed["for"]) {
+          if (parsed["for"].startsWith("[")) {
+            const i = parsed["for"].lastIndexOf("]");
+            if (i >= 0) {
+              origin.remoteAddress = parsed["for"].slice(1, i);
+              origin.remotePort =
+                parseInt(parsed["for"].slice(i + 2)) || origin.remotePort;
+            }
+          } else {
+            const i = parsed["for"].lastIndexOf(":");
+            if (i >= 0) {
+              origin.remoteAddress = parsed["for"].slice(0, i);
+              origin.remotePort =
+                parseInt(parsed["for"].slice(i + 1)) || origin.remotePort;
+            } else {
+              origin.remoteAddress = parsed["for"];
+            }
+          }
+        }
+
+        if (parsed["by"]) {
+          if (parsed["by"].startsWith("[")) {
+            const i = parsed["by"].lastIndexOf("]");
+            if (i >= 0) {
+              origin.localAddress = parsed["by"].slice(1, i);
+              origin.localPort =
+                parseInt(parsed["by"].slice(i + 2)) || origin.localPort;
+            }
+          } else {
+            const i = parsed["by"].lastIndexOf(":");
+            if (i >= 0) {
+              origin.localAddress = parsed["by"].slice(0, i);
+              origin.localPort =
+                parseInt(parsed["by"].slice(i + 1)) || origin.localPort;
+            } else {
+              origin.localAddress = parsed["by"];
+            }
+          }
+        }
+      }
+    }
+
+    cache.set(request, origin);
+  }
+  return origin;
+}

lib/fs.ts

@@ -19,11 +19,43 @@
 
 import * as url from "url";
 import * as querystring from "querystring";
-import { GridFSBucket } from "mongodb";
-import * as db from "./db";
+import { IncomingMessage, ServerResponse } from "http";
+import { Collection, GridFSBucket, ObjectId } from "mongodb";
+import { onConnect, getOperations } from "./db";
 import * as logger from "./logger";
+import { getRequestOrigin } from "./forwarded";
 
-export function listener(request, response): void {
+let filesCollection: Collection;
+let filesBucket: GridFSBucket;
+let uploadsBucket: GridFSBucket;
+
+onConnect(async (db) => {
+  filesCollection = db.collection("fs.files");
+  filesBucket = new GridFSBucket(db);
+  uploadsBucket = new GridFSBucket(db, { bucketName: "uploads" });
+});
+
+async function canUpload(
+  deviceId: string,
+  fileName: string,
+  timeout = Date.now() + 5000
+): Promise<boolean> {
+  const operations = Object.values(await getOperations(deviceId));
+  for (const operation of operations) {
+    if (operation.name === "Upload" && operation.args.fileName === fileName)
+      return true;
+  }
+
+  if (Date.now() >= timeout) return false;
+  // Need to wait and retry in case upload was initiated before session was closed
+  await new Promise((resolve) => setTimeout(resolve, 1000));
+  return canUpload(deviceId, fileName, timeout);
+}
+
+export async function listener(
+  request: IncomingMessage,
+  response: ServerResponse
+): Promise<void> {
   const urlParts = url.parse(request.url, true);
   if (request.method === "GET") {
     const filename = querystring.unescape(urlParts.pathname.substring(1));
@@ -31,31 +63,61 @@ export function listener(request, response): void {
     const log = {
       message: "Fetch file",
       filename: filename,
-      remoteAddress: request.connection.remoteAddress
+      remoteAddress: getRequestOrigin(request).remoteAddress,
     };
 
-    db.filesCollection.findOne({ _id: filename }, (err, file) => {
-      if (err) throw err;
+    const file = await filesCollection.findOne({ _id: filename });
+    if (!file) {
+      response.writeHead(404);
+      response.end();
+      log.message += " not found";
+      logger.accessError(log);
+      return;
+    }
 
-      if (!file) {
-        response.writeHead(404);
-        response.end();
-        log.message += " not found";
-        logger.accessError(log);
-        return;
-      }
-
-      response.writeHead(200, {
-        "Content-Type": file.contentType || "application/octet-stream",
-        "Content-Length": file.length
-      });
-
-      const bucket = new GridFSBucket(db.client.db());
-      const downloadStream = bucket.openDownloadStreamByName(filename);
-      downloadStream.pipe(response);
-
-      logger.accessInfo(log);
+    response.writeHead(200, {
+      "Content-Type": file.contentType || "application/octet-stream",
+      "Content-Length": file.length,
     });
+
+    const downloadStream = filesBucket.openDownloadStreamByName(filename);
+    downloadStream.pipe(response);
+
+    logger.accessInfo(log);
+  } else if (request.method === "PUT") {
+    const [, deviceId, ...filePath] = urlParts.pathname
+      .split("/")
+      .map(querystring.unescape);
+
+    const fileName = `${deviceId}/${filePath.join("/")}`;
+
+    if (!(await canUpload(deviceId, fileName))) {
+      response.writeHead(403);
+      response.end("403 Forbidden");
+      return;
+    }
+
+    try {
+      await uploadsBucket.delete((fileName as unknown) as ObjectId);
+    } catch (err) {
+      // File not found, do nothing
+    }
+    const uploadStream = uploadsBucket.openUploadStreamWithId(
+      fileName,
+      fileName
+    );
+
+    uploadStream.on("finish", () => {
+      response.writeHead(200);
+      response.end();
+    });
+
+    uploadStream.on("error", (err) => {
+      response.writeHead(500);
+      response.end(err.message);
+    });
+
+    request.pipe(uploadStream);
   } else {
     response.writeHead(405, { Allow: "GET" });
     response.end("405 Method Not Allowed");

lib/gpn-heuristic.ts

@@ -31,7 +31,7 @@ const UNDISCOVERED_DEPTH = 7;
 // point has been discovered.
 export function estimateGpnCount(
   gpnPatterns: [Path, number][],
-  depth: number = 0
+  depth = 0
 ): number {
   const children: { [segment: string]: [Path, number][] } = {};
   const wildcardChildren: [Path, number][] = [];

lib/init.ts

@@ -95,11 +95,11 @@ export async function getStatus(): Promise<Status> {
     filters: !Object.keys(ui["filters"]).length,
     device: !Object.keys(ui["device"]).length,
     index: !Object.keys(ui["index"]).length,
-    overview: !Object.keys(ui["overview"]).length
+    overview: !Object.keys(ui["overview"]).length,
   };
 }
 
-export async function seed(options): Promise<void> {
+export async function seed(options: Record<string, boolean>): Promise<void> {
   const resources = {};
   const proms = [];
 
@@ -108,6 +108,7 @@ export async function seed(options): Promise<void> {
       { role: "admin", resource: "devices", access: 3, validate: "true" },
       { role: "admin", resource: "faults", access: 3, validate: "true" },
       { role: "admin", resource: "files", access: 3, validate: "true" },
+      { role: "admin", resource: "uploads", access: 3, validate: "true" },
       { role: "admin", resource: "presets", access: 3, validate: "true" },
       { role: "admin", resource: "provisions", access: 3, validate: "true" },
       { role: "admin", resource: "config", access: 3, validate: "true" },
@@ -117,12 +118,12 @@ export async function seed(options): Promise<void> {
         role: "admin",
         resource: "virtualParameters",
         access: 3,
-        validate: "true"
-      }
+        validate: "true",
+      },
     ];
 
     resources["users"] = [
-      { username: "admin", password: "admin", roles: ["admin"] }
+      { username: "admin", password: "admin", roles: ["admin"] },
     ];
   }
 
@@ -135,7 +136,7 @@ export async function seed(options): Promise<void> {
       { _id: "ui.filters.1.parameter", value: "DeviceID.ProductClass" },
       { _id: "ui.filters.1.type", value: "'string'" },
       { _id: "ui.filters.2.label", value: "'Tag'" },
-      { _id: "ui.filters.2.type", value: "'tag'" }
+      { _id: "ui.filters.2.type", value: "'tag'" },
     ]);
   }
 
@@ -149,118 +150,118 @@ export async function seed(options): Promise<void> {
       { _id: "ui.device.2.parameters.0.label", value: "'Last inform'" },
       {
         _id: "ui.device.2.parameters.0.components.0.type",
-        value: "'parameter'"
+        value: "'parameter'",
       },
       { _id: "ui.device.2.parameters.0.components.1.chart", value: "'online'" },
       {
         _id: "ui.device.2.parameters.0.components.1.type",
-        value: "'overview-dot'"
+        value: "'overview-dot'",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.type",
-        value: "'summon-button'"
+        value: "'summon-button'",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.0",
-        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.1",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.2",
         value:
-          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress"
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.3",
         value:
-          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress"
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.4",
-        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID"
+        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.5",
         value:
-          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase"
+          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.6",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName"
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.7",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress"
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.8",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress"
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress",
       },
       {
         _id: "ui.device.2.parameters.0.parameter",
-        value: "DATE_STRING(Events.Inform)"
+        value: "DATE_STRING(Events.Inform)",
       },
       { _id: "ui.device.2.parameters.1.label", value: "'Serial number'" },
       {
         _id: "ui.device.2.parameters.1.parameter",
-        value: "DeviceID.SerialNumber"
+        value: "DeviceID.SerialNumber",
       },
       { _id: "ui.device.2.parameters.2.label", value: "'Product class'" },
       {
         _id: "ui.device.2.parameters.2.parameter",
-        value: "DeviceID.ProductClass"
+        value: "DeviceID.ProductClass",
       },
       { _id: "ui.device.2.parameters.3.label", value: "'OUI'" },
       {
         _id: "ui.device.2.parameters.3.parameter",
-        value: "DeviceID.OUI"
+        value: "DeviceID.OUI",
       },
       { _id: "ui.device.2.parameters.4.label", value: "'Manufacturer'" },
       {
         _id: "ui.device.2.parameters.4.parameter",
-        value: "DeviceID.Manufacturer"
+        value: "DeviceID.Manufacturer",
       },
       { _id: "ui.device.2.parameters.5.label", value: "'Hardware version'" },
       {
         _id: "ui.device.2.parameters.5.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
       },
       { _id: "ui.device.2.parameters.6.label", value: "'Software version'" },
       {
         _id: "ui.device.2.parameters.6.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
       },
       { _id: "ui.device.2.parameters.7.label", value: "'MAC'" },
       {
         _id: "ui.device.2.parameters.7.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress"
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress",
       },
       { _id: "ui.device.2.parameters.8.label", value: "'IP'" },
       {
         _id: "ui.device.2.parameters.8.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
       },
       { _id: "ui.device.2.parameters.9.label", value: "'WLAN SSID'" },
       {
         _id: "ui.device.2.parameters.9.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
       },
       { _id: "ui.device.2.parameters.10.label", value: "'WLAN passphrase'" },
       {
         _id: "ui.device.2.parameters.10.parameter",
         value:
-          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase"
+          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
       },
       { _id: "ui.device.3.type", value: "'parameter-table'" },
       { _id: "ui.device.3.label", value: "'WLAN hosts'" },
       {
         _id: "ui.device.3.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host"
+        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host",
       },
       { _id: "ui.device.3.childParameters.0.label", value: "'Host name'" },
       { _id: "ui.device.3.childParameters.0.parameter", value: "HostName" },
@@ -278,12 +279,12 @@ export async function seed(options): Promise<void> {
       { _id: "ui.device.5.element", value: "'div.container-full-width'" },
       {
         _id: "ui.device.5.components.0.components.0",
-        value: "'All parameters'"
+        value: "'All parameters'",
       },
       { _id: "ui.device.5.components.0.element", value: "'h3'" },
       { _id: "ui.device.5.components.0.type", value: "'container'" },
       { _id: "ui.device.5.components.1.type", value: "'all-parameters'" },
-      { _id: "ui.device.6.type", value: "'device-actions'" }
+      { _id: "ui.device.6.type", value: "'device-actions'" },
     ]);
   }
 
@@ -298,18 +299,18 @@ export async function seed(options): Promise<void> {
       { _id: "ui.index.2.label", value: "'Software version'" },
       {
         _id: "ui.index.2.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
       },
       { _id: "ui.index.3.label", value: "'IP'" },
       {
         _id: "ui.index.3.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
       },
       { _id: "ui.index.4.label", value: "'SSID'" },
       {
         _id: "ui.index.4.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
       },
       { _id: "ui.index.5.type", value: "'container'" },
       { _id: "ui.index.5.label", value: "'Last inform'" },
@@ -322,7 +323,7 @@ export async function seed(options): Promise<void> {
       { _id: "ui.index.6.label", value: "'Tags'" },
       { _id: "ui.index.6.parameter", value: "Tags" },
       { _id: "ui.index.6.unsortable", value: "true" },
-      { _id: "ui.index.6.writable", value: "false" }
+      { _id: "ui.index.6.writable", value: "false" },
     ]);
   }
 
@@ -331,43 +332,44 @@ export async function seed(options): Promise<void> {
       { _id: "ui.overview.charts.online.label", value: "'Online status'" },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.color",
-        value: "'#31a354'"
+        value: "'#31a354'",
       },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.filter",
-        value: "Events.Inform > NOW() - 5 * 60 * 1000"
+        value: "Events.Inform > NOW() - 5 * 60 * 1000",
       },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.label",
-        value: "'Online now'"
+        value: "'Online now'",
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.color",
-        value: "'#a1d99b'"
+        value: "'#a1d99b'",
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.filter",
         value:
-          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)"
+          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)",
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.label",
-        value: "'Past 24 hours'"
+        value: "'Past 24 hours'",
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.color",
-        value: "'#e5f5e0'"
+        value: "'#e5f5e0'",
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.filter",
-        value: "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)"
+        value:
+          "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)",
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.label",
-        value: "'Others'"
+        value: "'Others'",
       },
       { _id: "ui.overview.groups.online.label", value: "''" },
-      { _id: "ui.overview.groups.online.charts.0", value: "'online'" }
+      { _id: "ui.overview.groups.online.charts.0", value: "'online'" },
     ]);
   }
 
@@ -378,16 +380,16 @@ export async function seed(options): Promise<void> {
         weight: 0,
         channel: "bootstrap",
         events: "0 BOOTSTRAP",
-        provision: "bootstrap"
+        provision: "bootstrap",
       },
       { _id: "default", weight: 0, channel: "default", provision: "default" },
-      { _id: "inform", weight: 0, channel: "inform", provision: "inform" }
+      { _id: "inform", weight: 0, channel: "inform", provision: "inform" },
     ];
 
     resources["provisions"] = [
       { _id: "bootstrap", script: BOOTSTRAP_SCRIPT },
       { _id: "default", script: DEFAULT_SCRIPT },
-      { _id: "inform", script: INFORM_SCRIPT }
+      { _id: "inform", script: INFORM_SCRIPT },
     ];
   }
 

lib/local-cache.ts

@@ -22,30 +22,39 @@ import * as crypto from "crypto";
 import * as config from "./config";
 import * as db from "./db";
 import * as cache from "./cache";
-import { mongoQueryToFilter } from "./mongodb-functions";
+import { convertOldPrecondition } from "./mongodb-functions";
 import * as logger from "./logger";
 import * as scheduling from "./scheduling";
 import * as expression from "./common/expression";
 import { parse } from "./common/expression-parser";
-import { Preset, Expression } from "./types";
+import {
+  Preset,
+  Expression,
+  Provisions,
+  VirtualParameters,
+  Files,
+  Users,
+  Permissions,
+  Config,
+  UiConfig,
+} from "./types";
 
-interface Permissions {
-  [role: string]: {
-    [access: number]: {
-      [resource: string]: {
-        access: number;
-        filter: Expression;
-        validate?: Expression;
-      };
-    };
-  };
+interface Snapshot {
+  presets: Preset[];
+  provisions: Provisions;
+  virtualParameters: VirtualParameters;
+  files: Files;
+  permissions: Permissions;
+  users: Users;
+  config: Config;
+  ui: UiConfig;
 }
 
 const REFRESH = 3000;
 const EVICT_TIMEOUT = 60000;
 
-const snapshots = new Map();
-let currentSnapshot = null;
+const snapshots = new Map<string, Snapshot>();
+let currentSnapshot: string = null;
 let nextRefresh = 1;
 
 function computeHash(snapshot): string {
@@ -98,11 +107,15 @@ function computeHash(snapshot): string {
   return h.digest("hex");
 }
 
-function flattenObject(src, prefix = "", dst = {}): {} {
+function flattenObject(
+  src: Record<string, unknown>,
+  prefix = "",
+  dst = {}
+): Record<string, unknown> {
   for (const k of Object.keys(src)) {
     const v = src[k];
     if (typeof v === "object" && !Array.isArray(v))
-      flattenObject(v, `${prefix}${k}.`, dst);
+      flattenObject(v as Record<string, unknown>, `${prefix}${k}.`, dst);
     else dst[`${prefix}${k}`] = v;
   }
   return dst;
@@ -112,14 +125,14 @@ async function fetchPresets(): Promise<Preset[]> {
   const res = await db.getPresets();
   let objects = await db.getObjects();
 
-  objects = objects.map(obj => {
+  objects = objects.map((obj) => {
     // Flatten object
     obj = flattenObject(obj);
 
     // If no keys are defined, consider all parameters as keys to keep the
     // same behavior from v1.0
     if (!obj["_keys"] || !obj["_keys"].length)
-      obj["_keys"] = Object.keys(obj).filter(k => !k.startsWith("_"));
+      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
 
     return obj;
   });
@@ -136,12 +149,9 @@ async function fetchPresets(): Promise<Preset[]> {
     if (preset["schedule"]) {
       const parts = preset["schedule"].trim().split(/\s+/);
       schedule = {
-        md5: crypto
-          .createHash("md5")
-          .update(preset["schedule"])
-          .digest("hex"),
+        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
         duration: null,
-        schedule: null
+        schedule: null,
       };
 
       try {
@@ -151,7 +161,7 @@ async function fetchPresets(): Promise<Preset[]> {
         logger.warn({
           message: "Invalid preset schedule",
           preset: preset["_id"],
-          schedule: preset["schedule"]
+          schedule: preset["schedule"],
         });
         schedule.schedule = false;
       }
@@ -163,7 +173,9 @@ async function fetchPresets(): Promise<Preset[]> {
       try {
         precondition = parse(preset["precondition"]);
       } catch (error) {
-        precondition = mongoQueryToFilter(JSON.parse(preset["precondition"]));
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"])
+        );
       }
 
       // Simplify expression
@@ -176,7 +188,7 @@ async function fetchPresets(): Promise<Preset[]> {
     for (const c of preset["configurations"]) {
       switch (c.type) {
         case "age":
-          _provisions.push(["refresh", c.name, c.age]);
+          _provisions.push(["refresh", c.name, +c.age]);
           break;
 
         case "value":
@@ -199,7 +211,7 @@ async function fetchPresets(): Promise<Preset[]> {
           for (const obj of objects) {
             if (obj["_id"] === c.object) {
               const alias = obj["_keys"]
-                .map(k => `${k}:${JSON.stringify(obj[k])}`)
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
                 .join(",");
               const p = `${c.name}.[${alias}]`;
               _provisions.push(["instances", p, 1]);
@@ -217,7 +229,7 @@ async function fetchPresets(): Promise<Preset[]> {
           for (const obj of objects) {
             if (obj["_id"] === c.object) {
               const alias = obj["_keys"]
-                .map(k => `${k}:${JSON.stringify(obj[k])}`)
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
                 .join(",");
               const p = `${c.name}.[${alias}]`;
               _provisions.push(["instances", p, 0]);
@@ -237,14 +249,14 @@ async function fetchPresets(): Promise<Preset[]> {
       schedule: schedule,
       events: events,
       precondition: precondition,
-      provisions: _provisions
+      provisions: _provisions,
     });
   }
 
   return presets;
 }
 
-async function fetchProvisions(): Promise<{}> {
+async function fetchProvisions(): Promise<Provisions> {
   const res = await db.getProvisions();
 
   const provisions = {};
@@ -254,7 +266,9 @@ async function fetchProvisions(): Promise<{}> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    provisions[r["_id"]].script = new vm.Script(
+    provisions[
+      r["_id"]
+    ].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -263,7 +277,7 @@ async function fetchProvisions(): Promise<{}> {
   return provisions;
 }
 
-async function fetchVirtualParameters(): Promise<{}> {
+async function fetchVirtualParameters(): Promise<VirtualParameters> {
   const res = await db.getVirtualParameters();
 
   const virtualParameters = {};
@@ -273,7 +287,9 @@ async function fetchVirtualParameters(): Promise<{}> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    virtualParameters[r["_id"]].script = new vm.Script(
+    virtualParameters[
+      r["_id"]
+    ].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -292,7 +308,7 @@ async function fetchPermissions(): Promise<Permissions> {
 
     permissions[p.role][p.access][p.resource] = {
       access: p.access,
-      filter: parse(p.filter || "true")
+      filter: parse(p.filter || "true"),
     };
     if (p.validate)
       permissions[p.role][p.access][p.resource].validate = parse(p.validate);
@@ -301,7 +317,7 @@ async function fetchPermissions(): Promise<Permissions> {
   return permissions;
 }
 
-async function fetchFiles(): Promise<{}> {
+async function fetchFiles(): Promise<Files> {
   const res = await db.getFiles();
   const files = {};
 
@@ -316,7 +332,7 @@ async function fetchFiles(): Promise<{}> {
   return files;
 }
 
-async function fetchUsers(): Promise<{}> {
+async function fetchUsers(): Promise<Users> {
   const _users = await db.getUsers();
   const users = {};
 
@@ -324,14 +340,14 @@ async function fetchUsers(): Promise<{}> {
     users[user._id] = {
       password: user.password,
       salt: user.salt,
-      roles: user.roles.split(",").map(s => s.trim())
+      roles: user.roles.split(",").map((s) => s.trim()),
     };
   }
 
   return users;
 }
 
-async function fetchConfig(): Promise<[{}, {}]> {
+async function fetchConfig(): Promise<[Config, UiConfig]> {
   const conf = await db.getConfig();
 
   conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
@@ -340,7 +356,7 @@ async function fetchConfig(): Promise<[{}, {}]> {
     filters: {},
     device: {},
     index: {},
-    overview: {}
+    overview: {},
   };
   const _config = {};
 
@@ -367,7 +383,7 @@ async function fetchConfig(): Promise<[{}, {}]> {
 
 async function refresh(): Promise<void> {
   if (!nextRefresh) {
-    await new Promise(resolve => setTimeout(resolve, 20));
+    await new Promise((resolve) => setTimeout(resolve, 20));
     await refresh();
     return;
   }
@@ -382,7 +398,7 @@ async function refresh(): Promise<void> {
     return;
   }
 
-  const unlockOrExtend = await cache.lock("presets_hash_lock", 3);
+  const lockToken = await cache.acquireLock("presets_hash_lock", 4000, 4000);
 
   const res = await Promise.all([
     fetchPresets(),
@@ -391,7 +407,7 @@ async function refresh(): Promise<void> {
     fetchFiles(),
     fetchPermissions(),
     fetchUsers(),
-    fetchConfig()
+    fetchConfig(),
   ]);
 
   const snapshot = {
@@ -402,7 +418,7 @@ async function refresh(): Promise<void> {
     permissions: res[4],
     users: res[5],
     config: res[6][0],
-    ui: res[6][1]
+    ui: res[6][1],
   };
 
   if (currentSnapshot) {
@@ -416,7 +432,7 @@ async function refresh(): Promise<void> {
   currentSnapshot = computeHash(snapshot);
   snapshots.set(currentSnapshot, snapshot);
   await cache.set("presets_hash", currentSnapshot, 300);
-  await unlockOrExtend(0);
+  await cache.releaseLock("presets_hash_lock", lockToken);
 
   nextRefresh = now + (REFRESH - (now % REFRESH));
 }
@@ -426,45 +442,40 @@ export async function getCurrentSnapshot(): Promise<string> {
   return currentSnapshot;
 }
 
-export function hasSnapshot(hash): boolean {
+export function hasSnapshot(hash: string): boolean {
   return snapshots.has(hash);
 }
 
-export function getPresets(snapshotKey): Preset[] {
+export function getPresets(snapshotKey: string): Preset[] {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.presets;
 }
 
-export function getProvisions(
-  snapshotKey
-): { [name: string]: { md5: string; script: vm.Script } } {
+export function getProvisions(snapshotKey: string): Provisions {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.provisions;
 }
 
-export function getVirtualParameters(
-  snapshotKey
-): { [name: string]: { md5: string; script: vm.Script } } {
+export function getVirtualParameters(snapshotKey: string): VirtualParameters {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.virtualParameters;
 }
 
-export function getFiles(
-  snapshotKey
-): { [name: string]: { length: number; md5: string; contentType: string } } {
+export function getFiles(snapshotKey: string): Files {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.files;
 }
 
 export function getConfig(
-  snapshotKey,
-  key,
-  context?,
-  now?
+  snapshotKey: string,
+  key: string,
+  context: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Expression
 ): string | number | boolean | null {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
@@ -484,39 +495,52 @@ export function getConfig(
       "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
     "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
     "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
-    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT"
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
   };
 
   if (!(key in snapshot.config)) {
-    if (key in oldOpts)
-      return config.get(oldOpts[key], context ? context.id : null);
-    else return null;
+    if (key in oldOpts) {
+      let id;
+      if (context && context["id"]) {
+        id = context["id"];
+      } else if (cb) {
+        id = cb(["PARAM", "DeviceID.ID"]);
+        if (Array.isArray(id)) id = null;
+      }
+      return config.get(oldOpts[key], id);
+    }
+    return null;
   }
-  return expression.evaluate(snapshot.config[key], context, now || Date.now());
+
+  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+  return Array.isArray(v) ? null : v;
 }
 
-export function getConfigExpression(snapshotKey, key): Expression {
+export function getConfigExpression(
+  snapshotKey: string,
+  key: string
+): Expression {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 
   return snapshot.config[key];
 }
 
-export function getUsers(snapshotKey): {} {
+export function getUsers(snapshotKey: string): Users {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 
   return snapshot.users;
 }
 
-export function getPermissions(snapshotKey): {} {
+export function getPermissions(snapshotKey: string): Permissions {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 
   return snapshot.permissions;
 }
 
-export function getUiConfig(snapshotKey): {} {
+export function getUiConfig(snapshotKey: string): UiConfig {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 

lib/logger.ts

@@ -21,6 +21,15 @@ import * as fs from "fs";
 import * as os from "os";
 
 import * as config from "./config";
+import { getRequestOrigin } from "./forwarded";
+import {
+  SessionContext,
+  AcsRequest,
+  CpeRequest,
+  CpeFault,
+  InformRequest,
+  Fault,
+} from "./types";
 
 const REOPEN_EVERY = 60000;
 
@@ -66,7 +75,7 @@ function reopen(): void {
       if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
         logStream.end();
         logStream = fs.createWriteStream(null, {
-          fd: fs.openSync(LOG_FILE, "a")
+          fd: fs.openSync(LOG_FILE, "a"),
         });
         logStat = fs.fstatSync(logStream.fd);
       }
@@ -90,7 +99,7 @@ function reopen(): void {
       ) {
         accessLogStream.end();
         accessLogStream = fs.createWriteStream(null, {
-          fd: fs.openSync(ACCESS_LOG_FILE, "a")
+          fd: fs.openSync(ACCESS_LOG_FILE, "a"),
         });
         accessLogStat = fs.fstatSync(accessLogStream.fd);
       }
@@ -104,7 +113,7 @@ function reopen(): void {
     setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
 }
 
-export function init(service, version): void {
+export function init(service: string, version: string): void {
   defaultMeta.hostname = os.hostname();
   defaultMeta.pid = process.pid;
   defaultMeta.name = `genieacs-${service}`;
@@ -120,7 +129,7 @@ export function init(service, version): void {
 
   if (ACCESS_LOG_FILE) {
     accessLogStream = fs.createWriteStream(null, {
-      fd: fs.openSync(ACCESS_LOG_FILE, "a")
+      fd: fs.openSync(ACCESS_LOG_FILE, "a"),
     });
     accessLogStat = fs.fstatSync(accessLogStream.fd);
   }
@@ -146,61 +155,75 @@ export function close(): void {
   logStream.end();
 }
 
-export function flatten(details): {} {
+export function flatten(
+  details: Record<string, unknown>
+): Record<string, unknown> {
   if (details.sessionContext) {
-    details.deviceId = details.sessionContext.deviceId;
-    details.remoteAddress =
-      details.sessionContext.httpRequest.connection.remoteAddress;
+    const sessionContext = details.sessionContext as SessionContext;
+    details.deviceId = sessionContext.deviceId;
+    details.remoteAddress = getRequestOrigin(
+      sessionContext.httpRequest
+    ).remoteAddress;
     delete details.sessionContext;
   }
 
   if (details.exception) {
-    details.exceptionName = details.exception.name;
-    details.exceptionMessage = details.exception.message;
-    details.exceptionStack = details.exception.stack;
+    const err = details.exception as Error;
+    details.exceptionName = err.name;
+    details.exceptionMessage = err.message;
+    details.exceptionStack = err.stack;
     delete details.exception;
   }
 
   if (details.task) {
-    details.taskId = details.task._id;
+    details.taskId = details.task["_id"];
     delete details.task;
   }
 
   if (details.rpc) {
-    if (details.rpc.acsRequest) {
-      details.acsRequestId = details.rpc.id;
-      details.acsRequestName = details.rpc.acsRequest.name;
-      if (details.rpc.acsRequest.commandKey)
-        details.acsRequestCommandKey = details.rpc.acsRequest.commandKey;
-    } else if (details.rpc.cpeRequest) {
-      details.cpeRequestId = details.rpc.id;
-      if (details.rpc.cpeRequest.name === "Inform") {
-        details.informEvent = details.rpc.cpeRequest.event.join(",");
-        details.informRetryCount = details.rpc.cpeRequest.retryCount;
+    const rpc = details.rpc as {
+      id: string;
+      acsRequest?: AcsRequest;
+      cpeRequest?: CpeRequest;
+      cpeFault?: CpeFault;
+    };
+    if (rpc.acsRequest) {
+      details.acsRequestId = rpc.id;
+      details.acsRequestName = rpc.acsRequest.name;
+      if (rpc.acsRequest["commandKey"])
+        details.acsRequestCommandKey = rpc.acsRequest["commandKey"];
+    } else if (rpc.cpeRequest) {
+      details.cpeRequestId = rpc.id;
+      if (rpc.cpeRequest.name === "Inform") {
+        details.informEvent = (rpc.cpeRequest as InformRequest).event.join(",");
+        details.informRetryCount = (rpc.cpeRequest as InformRequest).retryCount;
       } else {
-        details.cpeRequestName = details.rpc.cpeRequest.name;
-        if (details.rpc.cpeRequest.commandKey)
-          details.cpeRequestCommandKey = details.rpc.cpeRequest.commandKey;
+        details.cpeRequestName = rpc.cpeRequest.name;
+        if (rpc.cpeRequest["commandKey"])
+          details.cpeRequestCommandKey = rpc.cpeRequest["commandKey"];
       }
-    } else if (details.rpc.cpeFault) {
-      details.acsRequestId = details.rpc.id;
-      details.cpeFaultCode = details.rpc.cpeFault.detail.faultCode;
-      details.cpeFaultString = details.rpc.cpeFault.detail.faultString;
+    } else if (rpc.cpeFault) {
+      details.acsRequestId = rpc.id;
+      details.cpeFaultCode = rpc.cpeFault.detail.faultCode;
+      details.cpeFaultString = rpc.cpeFault.detail.faultString;
     }
     delete details.rpc;
   }
 
   if (details.fault) {
-    details.faultCode = details.fault.code;
-    details.faultMessage = details.fault.message;
+    const fault = details.fault as Fault;
+    details.faultCode = fault.code;
+    details.faultMessage = fault.message;
     delete details.fault;
   }
 
   // For genieacs-ui
   if (details.context) {
-    details.remoteAddress = details.context.request.ip;
-    if (details.context.state.user)
-      details.user = details.context.state.user.username;
+    details.remoteAddress = getRequestOrigin(
+      details.context["req"]
+    ).remoteAddress;
+    if (details.context["state"].user)
+      details.user = details.context["state"].user.username;
     delete details.context;
   }
 
@@ -210,7 +233,10 @@ export function flatten(details): {} {
   return details;
 }
 
-function formatJson(details, systemd): string {
+function formatJson(
+  details: Record<string, unknown>,
+  systemd: boolean
+): string {
   if (systemd) {
     let severity = "";
     if (details.severity === "info") severity = "<6>";
@@ -223,14 +249,17 @@ function formatJson(details, systemd): string {
   return `${JSON.stringify(flatten(details))}${os.EOL}`;
 }
 
-function formatSimple(details, systemd): string {
+function formatSimple(
+  details: Record<string, unknown>,
+  systemd: boolean
+): string {
   const skip = {
     user: true,
     remoteAddress: true,
     severity: true,
     timestamp: true,
     message: true,
-    deviceId: !!details.sessionContext
+    deviceId: !!details.sessionContext,
   };
 
   flatten(details);
@@ -263,12 +292,14 @@ function formatSimple(details, systemd): string {
     return `${severity}${remote}${details.message}${meta}${os.EOL}`;
   }
 
-  return `${details.timestamp} [${details.severity.toUpperCase()}] ${remote}${
+  return `${
+    details.timestamp
+  } [${(details.severity as string).toUpperCase()}] ${remote}${
     details.message
   }${meta}${os.EOL}`;
 }
 
-function log(details): void {
+function log(details: Record<string, unknown>): void {
   details.timestamp = new Date().toISOString();
   if (LOG_FORMAT === "json") {
     details = Object.assign({}, defaultMeta, details);
@@ -278,22 +309,22 @@ function log(details): void {
   }
 }
 
-export function info(details): void {
+export function info(details: Record<string, unknown>): void {
   details.severity = "info";
   log(details);
 }
 
-export function warn(details): void {
+export function warn(details: Record<string, unknown>): void {
   details.severity = "warn";
   log(details);
 }
 
-export function error(details): void {
+export function error(details: Record<string, unknown>): void {
   details.severity = "error";
   log(details);
 }
 
-export function accessLog(details): void {
+export function accessLog(details: Record<string, unknown>): void {
   details.timestamp = new Date().toISOString();
   if (ACCESS_LOG_FORMAT === "json") {
     Object.assign(details, defaultMeta);
@@ -303,17 +334,17 @@ export function accessLog(details): void {
   }
 }
 
-export function accessInfo(details): void {
+export function accessInfo(details: Record<string, unknown>): void {
   details.severity = "info";
   accessLog(details);
 }
 
-export function accessWarn(details): void {
+export function accessWarn(details: Record<string, unknown>): void {
   details.severity = "warn";
   accessLog(details);
 }
 
-export function accessError(details): void {
+export function accessError(details: Record<string, unknown>): void {
   details.severity = "error";
   accessLog(details);
 }

lib/mongodb-functions.ts

@@ -18,14 +18,15 @@
  */
 
 import { ObjectID } from "mongodb";
-import { map, parse, stringify } from "./common/expression-parser";
-import { likePatternToRegExp, evaluate } from "./common/expression";
+import { map, parse, stringify, parseList } from "./common/expression-parser";
+import { likePatternToRegExp } from "./common/expression";
 import { Expression, Fault, Task } from "./types";
+import { decodeTag, encodeTag } from "./common";
 
 const isArray = Array.isArray;
 
-export function processDeviceFilter(filter): Expression {
-  return map(filter, exp => {
+export function processDeviceFilter(filter: Expression): Expression {
+  return map(filter, (exp) => {
     if (!isArray(exp)) return exp;
 
     if (exp[0] === "PARAM") {
@@ -43,17 +44,26 @@ export function processDeviceFilter(filter): Expression {
       exp[1][0] === "PARAM" &&
       exp[1][1].startsWith("Tags.")
     ) {
-      const t = exp[1][1].slice(5);
-      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
-      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
-      else if (exp[0] === "=" && exp[2] === true)
-        return ["=", ["PARAM", "_tags"], t];
-      else if (exp[0] === "<>" && exp[2] !== true)
-        return ["=", ["PARAM", "_tags"], t];
+      const t = decodeTag(exp[1][1].slice(5));
+      if (exp[0] === "IS NULL") return ["_tags", t, false];
+      else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
+      else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
+      else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
     } else if (
-      ["=", "<>", ">", ">=", "<", "<=", "LIKE", "NOT LIKE"].includes(exp[0])
+      [
+        "=",
+        "<>",
+        ">",
+        ">=",
+        "<",
+        "<=",
+        "LIKE",
+        "NOT LIKE",
+        "IS NULL",
+        "IS NOT NULL",
+      ].includes(exp[0])
     ) {
-      let e = map(exp, ee => {
+      let e = map(exp, (ee) => {
         if (
           isArray(ee) &&
           ee[0] === "PARAM" &&
@@ -75,8 +85,8 @@ export function processDeviceFilter(filter): Expression {
   });
 }
 
-export function processTasksFilter(filter): Expression {
-  return map(filter, exp => {
+export function processTasksFilter(filter: Expression): Expression {
+  return map(filter, (exp) => {
     if (!isArray(exp)) return exp;
     if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
       const e = exp.slice();
@@ -91,8 +101,8 @@ export function processTasksFilter(filter): Expression {
   });
 }
 
-export function processFaultsFilter(filter): Expression {
-  return map(filter, exp => {
+export function processFaultsFilter(filter: Expression): Expression {
+  return map(filter, (exp) => {
     if (!isArray(exp)) return exp;
     if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
       const e = exp.slice();
@@ -105,140 +115,137 @@ export function processFaultsFilter(filter): Expression {
   });
 }
 
-export function filterToMongoQuery(exp: Expression): {} {
-  const ops = {
-    OR: 0,
-    AND: 0,
-    NOT: 0,
-    "=": 1,
-    "<>": 1,
-    ">": 1,
-    ">=": 1,
-    "<": 1,
-    "<=": 1,
-    LIKE: 1,
-    "NOT LIKE": 1,
-    "IS NULL": 1,
-    "IS NOT NULL": 1
-  };
-
-  function recursive(filter, negate, res = {}): {} {
+export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
+  function recursive(filter, negate): Record<string, unknown> {
     const op = filter[0];
 
-    if (ops[op] === 0) {
-      for (let i = 1; i < filter.length; ++i) {
-        if (!isArray(filter[i]) || ops[filter[i][0]] == null)
-          throw new Error(`Invalid expression in ${op} clause`);
-      }
-
-      if ((!negate && op === "AND") || (negate && op === "OR")) {
-        res["$and"] = res["$and"] || [];
-        for (let i = 1; i < filter.length; ++i)
-          res["$and"].push(recursive(filter[i], negate));
-      } else if ((!negate && op === "OR") || (negate && op === "AND")) {
-        res["$or"] = res["$or"] || [];
-
-        for (let i = 1; i < filter.length; ++i)
-          res["$or"].push(recursive(filter[i], negate));
-      } else if (op === "NOT") {
-        recursive(filter[1], !negate, res);
-      }
-    } else if (ops[op] === 1) {
-      if (isArray(filter[2])) {
-        throw new Error(`Right operand of ${op} clause is not a primitive`);
-      } else if (
-        ["LIKE", "NOT LIKE"].includes(op) &&
-        isArray(filter[1]) &&
-        filter[1][0] === "FUNC" &&
-        ["UPPER", "LOWER"].includes(filter[1][1])
-      ) {
-        if (
-          (filter[1][1] === "UPPER" && filter[2] !== filter[2].toUpperCase()) ||
-          (filter[1][1] === "LOWER" && filter[2] !== filter[2].toLowerCase())
-        ) {
-          throw new Error(
-            `Cannot compare ${
-              filter[1][1]
-            }() against non-${filter[1][1].toLowerCase()}case pattern`
-          );
-        }
-      } else if (!isArray(filter[1]) || filter[1][0] !== "PARAM") {
-        throw new Error(`Left operand of ${op} clause is not a parameter`);
-      }
-
-      if (op === "=") {
-        const param = filter[1][1];
-        const p = (res[param] = res[param] || {});
-        if (negate) p["$ne"] = filter[2];
-        else p["$eq"] = filter[2];
-      } else if (op === "<>") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$ne"] = filter[2];
-        if (param !== "_tags") p["$exists"] = true;
-      } else if (op === ">") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gt"] = filter[2];
-      } else if (op === ">=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gte"] = filter[2];
-      } else if (op === "<") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lt"] = filter[2];
-      } else if (op === "<=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lte"] = filter[2];
-      } else if (op === "IS NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: negate };
-      } else if (op === "IS NOT NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: !negate };
-      } else {
-        if (op === "NOT LIKE") negate = !negate;
-        let param;
-        let flags;
-        if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
-          param = filter[1][2][1];
-          flags = "i";
-        } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
-          param = filter[1][2][1];
-          flags = "i";
-        } else {
-          param = filter[1][1];
-          flags = "";
-        }
-        const r = likePatternToRegExp(filter[2], filter[3], flags);
-        if (negate) res[param] = { $not: r };
-        else res[param] = r;
-      }
-    } else {
-      throw new Error(`Unrecognized operator ${op}`);
+    if (op === "AND") {
+      filter = filter.filter((f) => f !== true);
+      if (filter.length === 2) return recursive(filter[1], negate);
     }
 
-    return res;
+    if (op === "OR") {
+      filter = filter.filter((f) => f !== false);
+      if (filter.length === 2) return recursive(filter[1], negate);
+    }
+
+    if ((!negate && op === "AND") || (negate && op === "OR"))
+      return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if ((!negate && op === "OR") || (negate && op === "AND"))
+      return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if (op === "NOT") return recursive(filter[1], !negate);
+
+    if (isArray(filter[2]))
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "LIKE" || op === "NOT LIKE") {
+      if (op === "NOT LIKE") negate = !negate;
+      let param;
+      let flags;
+      if (!Array.isArray(filter[1]))
+        throw new Error(`Invalid LHS operand of ${op} clause`);
+      if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
+        if (filter[2] !== filter[2].toUpperCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
+        param = filter[1][2][1];
+        flags = "i";
+      } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
+        if (filter[2] !== filter[2].toLowerCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
+        param = filter[1][2][1];
+        flags = "i";
+      } else if (filter[1][0] === "PARAM") {
+        param = filter[1][1];
+        flags = "";
+      } else {
+        throw new Error(`Invalid LHS operand of ${op} clause`);
+      }
+      const r = likePatternToRegExp(filter[2], filter[3], flags);
+      if (negate) return { [param]: { $nin: [r, null] } };
+      else return { [param]: r };
+    }
+
+    if (op === "_tags") {
+      let t = filter[2];
+      if (negate) t = !t;
+      if (t) return { _tags: filter[1] };
+      else return { _tags: { $ne: filter[1] } };
+    }
+
+    if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
+      throw new Error(`Invalid LHS operand of ${op} clause`);
+
+    if (op === "IS NULL") {
+      const p = filter[1][1];
+      return { [p]: null };
+    }
+
+    if (op === "IS NOT NULL") {
+      const p = filter[1][1];
+      return { [p]: { $ne: null } };
+    }
+
+    if (isArray(filter[2]) || filter[2] == null)
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (!negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === "<>") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === ">") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lte: v } };
+      else return { [p]: { $gt: v } };
+    }
+
+    if (op === ">=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lt: v } };
+      else return { [p]: { $gte: v } };
+    }
+
+    if (op === "<") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gte: v } };
+      else return { [p]: { $lt: v } };
+    }
+
+    if (op === "<=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gt: v } };
+      else return { [p]: { $lte: v } };
+    }
+
+    throw new Error(`Unrecognized operator ${op}`);
   }
 
-  const _exp = evaluate(exp, null, Date.now());
-
-  if (!isArray(_exp)) {
-    if (_exp === true) return {};
+  if (!isArray(exp)) {
+    if (exp === true) return {};
     throw new Error("Primitives are not valid queries");
   }
 
-  return recursive(_exp, false);
+  return recursive(exp, false);
 }
 
-export function processDeviceProjection(projection: {}): {} {
+export function processDeviceProjection(
+  projection: Record<string, 1>
+): Record<string, 1> {
   if (!projection) return projection;
   const p = {};
   for (const [k, v] of Object.entries(projection)) {
@@ -264,7 +271,9 @@ export function processDeviceProjection(projection: {}): {} {
   return p;
 }
 
-export function processDeviceSort(sort: {}): {} {
+export function processDeviceSort(
+  sort: Record<string, number>
+): Record<string, number> {
   if (!sort) return sort;
   const s = {};
   for (const [k, v] of Object.entries(sort)) {
@@ -280,7 +289,7 @@ export function processDeviceSort(sort: {}): {} {
   return s;
 }
 
-function parseDate(d): number | string {
+function parseDate(d: Date): number | string {
   const n = +d;
   return isNaN(n) ? "" + d : n;
 }
@@ -302,7 +311,7 @@ interface FlatDevice {
   [param: string]: FlatAttributes;
 }
 
-export function flattenDevice(device): FlatDevice {
+export function flattenDevice(device: Record<string, unknown>): FlatDevice {
   function recursive(
     input,
     root: string,
@@ -313,39 +322,39 @@ export function flattenDevice(device): FlatDevice {
       if (!root) {
         if (name === "_lastInform") {
           output["Events.Inform"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
+            value: [parseDate(tree as Date), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_registered") {
           output["Events.Registered"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
+            value: [parseDate(tree as Date), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_lastBoot") {
           output["Events.1_BOOT"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
+            value: [parseDate(tree as Date), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_lastBootstrap") {
           output["Events.0_BOOTSTRAP"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
+            value: [parseDate(tree as Date), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_id") {
           output["DeviceID.ID"] = {
@@ -354,7 +363,7 @@ export function flattenDevice(device): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_deviceId") {
           output["DeviceID.Manufacturer"] = {
@@ -363,7 +372,7 @@ export function flattenDevice(device): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
           output["DeviceID.OUI"] = {
             value: [tree["_OUI"], "xsd:string"],
@@ -371,7 +380,7 @@ export function flattenDevice(device): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
           output["DeviceID.ProductClass"] = {
             value: [tree["_ProductClass"], "xsd:string"],
@@ -379,7 +388,7 @@ export function flattenDevice(device): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
           output["DeviceID.SerialNumber"] = {
             value: [tree["_SerialNumber"], "xsd:string"],
@@ -387,24 +396,24 @@ export function flattenDevice(device): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_tags") {
           output["Tags"] = {
             writable: false,
             writableTimestamp: timestamp,
             object: true,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
 
           for (const t of tree as string[]) {
-            output[`Tags.${t}`] = {
+            output[`Tags.${encodeTag(t)}`] = {
               value: [true, "xsd:boolean"],
               valueTimestamp: timestamp,
               writable: true,
               writableTimestamp: timestamp,
               object: false,
-              objectTimestamp: timestamp
+              objectTimestamp: timestamp,
             };
           }
         }
@@ -422,7 +431,7 @@ export function flattenDevice(device): FlatDevice {
       if (tree["_value"] != null) {
         attrs.value = [
           tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
-          tree["_type"]
+          tree["_type"],
         ];
         attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
         attrs.object = false;
@@ -456,27 +465,29 @@ export function flattenDevice(device): FlatDevice {
   }
 
   const newDevice: FlatDevice = {};
-  const timestamp = new Date(device["_lastInform"] || 1).getTime();
+  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
   recursive(device, "", newDevice, timestamp);
   return newDevice;
 }
 
-export function flattenFault(fault): Fault {
-  const f = Object.assign({}, fault);
+export function flattenFault(fault: unknown): Fault {
+  const f = Object.assign({}, fault) as Fault;
   if (f.timestamp) f.timestamp = +f.timestamp;
-  if (f.expiry) f.expiry = +f.expiry;
-  return f;
+  if (f["expiry"]) f["expiry"] = +f["expiry"];
+  return f as Fault;
 }
 
-export function flattenTask(task): Task {
-  const t = Object.assign({}, task);
+export function flattenTask(task: unknown): Task {
+  const t = Object.assign({}, task) as Task;
   t._id = "" + t._id;
-  if (t.timestamp) t.timestamp = +t.timestamp;
+  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
   if (t.expiry) t.expiry = +t.expiry;
   return t;
 }
 
-export function mongoQueryToFilter(query): Expression {
+export function convertOldPrecondition(
+  query: Record<string, unknown>
+): Expression {
   function recursive(_query): Expression {
     const expressions: Expression[] = [];
     for (const [k, v] of Object.entries(_query)) {
@@ -495,14 +506,27 @@ export function mongoQueryToFilter(query): Expression {
       } else if (k === "_tags") {
         if (typeof v === "object") {
           if (isArray(v)) throw new Error(`Invalid type`);
-
-          if (v.hasOwnProperty("$ne"))
-            expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]);
-          else if (v.hasOwnProperty("$eq"))
-            expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v["$eq"]}`]]);
-          else throw new Error(`Invalid tag query`);
+          const conjs: Expression[] = [];
+          for (const [op, val] of Object.entries(v)) {
+            if (op === "$ne") {
+              if (typeof v["$ne"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else {
+              throw new Error(`Invalid tag query`);
+            }
+          }
+          if (conjs.length === 1) expressions.push(conjs[0]);
+          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
         } else {
-          expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
+          expressions.push([
+            "IS NOT NULL",
+            ["PARAM", `Tags.${encodeTag(v as string)}`],
+          ]);
         }
       } else if (k.startsWith("Tags.")) {
         let exists: boolean;
@@ -518,34 +542,31 @@ export function mongoQueryToFilter(query): Expression {
 
         const exps: Expression[] = [];
         for (const [kk, vv] of Object.entries(v)) {
-          let op;
-          switch (kk) {
-            case "$eq":
-              op = "=";
-              break;
-            case "$ne":
-              op = "<>";
-              break;
-            case "$lt":
-              op = "<";
-              break;
-            case "$lte":
-              op = "<=";
-              break;
-            case "$gt":
-              op = ">";
-              break;
-            case "$gte":
-              op = ">=";
-              break;
-            default:
-              throw new Error(`Operator ${kk} not supported`);
+          if (kk === "$eq") {
+            exps.push(["=", ["PARAM", k], vv]);
+          } else if (kk === "$ne") {
+            exps.push([
+              "OR",
+              ["<>", ["PARAM", k], vv],
+              ["IS NULL", ["PARAM", k]],
+            ]);
+          } else if (kk === "$lt") {
+            exps.push(["<", ["PARAM", k], vv]);
+          } else if (kk === "$lte") {
+            exps.push(["<=", ["PARAM", k], vv]);
+          } else if (kk === "$gt") {
+            exps.push([">", ["PARAM", k], vv]);
+          } else if (kk === "$gte") {
+            exps.push([">=", ["PARAM", k], vv]);
+          } else {
+            throw new Error(`Operator ${kk} not supported`);
           }
-          exps.push([op, ["PARAM", k], vv]);
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
         }
         if (exps.length === 1) {
           expressions.push(exps[0]);
-        } else {
+        } else if (exps.length > 1) {
           const and: Expression = ["AND"];
           expressions.push(and.concat(exps));
         }
@@ -554,8 +575,8 @@ export function mongoQueryToFilter(query): Expression {
       }
     }
     if (expressions.length === 1) return expressions[0];
-    const and: Expression = ["AND"];
-    return and.concat(expressions);
+    if (expressions.length === 0) return true;
+    return ["AND", ...expressions];
   }
 
   // empty filter
@@ -564,15 +585,21 @@ export function mongoQueryToFilter(query): Expression {
   return recursive(query);
 }
 
-export function flattenPreset(preset): {} {
+export function flattenPreset(
+  preset: Record<string, unknown>
+): Record<string, unknown> {
   const p = Object.assign({}, preset);
   if (p.precondition) {
     try {
       // Try parse to check expression validity
-      parse(p.precondition);
+      parse(p.precondition as string);
     } catch (error) {
-      p.precondition = mongoQueryToFilter(JSON.parse(p.precondition));
-      p.precondition = p.precondition.length ? stringify(p.precondition) : "";
+      p.precondition = convertOldPrecondition(
+        JSON.parse(p.precondition as string)
+      );
+      p.precondition = (p.precondition as string).length
+        ? stringify(p.precondition as Expression)
+        : "";
     }
   }
 
@@ -584,14 +611,14 @@ export function flattenPreset(preset): {} {
 
   const provision = p.configurations[0];
   if (
-    p.configurations.length === 1 &&
+    (p.configurations as any[]).length === 1 &&
     provision.type === "provision" &&
     provision.name &&
     provision.name.length
   ) {
     p.provision = provision.name;
     p.provisionArgs = provision.args
-      ? JSON.stringify(provision.args).slice(1, -1)
+      ? provision.args.map((a) => stringify(a)).join(", ")
       : "";
   }
 
@@ -599,30 +626,42 @@ export function flattenPreset(preset): {} {
   return p;
 }
 
-export function flattenFile(file): {} {
+export function flattenFile(
+  file: Record<string, unknown>
+): Record<string, unknown> {
   const f = {};
   f["_id"] = file["_id"];
   if (file.metadata) {
-    f["metadata.fileType"] = file.metadata.fileType || "";
-    f["metadata.oui"] = file.metadata.oui || "";
-    f["metadata.productClass"] = file.metadata.productClass || "";
-    f["metadata.version"] = file.metadata.version || "";
+    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
+    f["metadata.oui"] = file["metadata"]["oui"] || "";
+    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
+    f["metadata.version"] = file["metadata"]["version"] || "";
   }
   return f;
 }
 
-export function preProcessPreset(data): {} {
+export function flattenUpload(
+  file: Record<string, unknown>
+): Record<string, unknown> {
+  const f = {};
+  f["_id"] = file["_id"];
+  return f;
+}
+
+export function preProcessPreset(
+  data: Record<string, unknown>
+): Record<string, unknown> {
   const preset = Object.assign({}, data);
 
   if (!preset.precondition) preset.precondition = "";
   // Try parse to check expression validity
-  parse(preset.precondition);
+  parse(preset.precondition as string);
 
-  preset.weight = parseInt(preset.weight) || 0;
+  preset.weight = parseInt(preset.weight as string) || 0;
 
   const events = {};
   if (preset.events) {
-    for (let e of preset.events.split(",")) {
+    for (let e of (preset.events as string).split(",")) {
       let v = true;
       e = e.trim();
       if (e.startsWith("-")) {
@@ -640,11 +679,11 @@ export function preProcessPreset(data): {} {
   const configuration = {
     type: "provision",
     name: preset.provision,
-    args: null
+    args: null,
   };
 
   if (preset.provisionArgs)
-    configuration.args = JSON.parse(`[${preset.provisionArgs}]`);
+    configuration.args = parseList(preset.provisionArgs as string);
 
   delete preset.provision;
   delete preset.provisionArgs;

lib/nbi.ts

@@ -18,13 +18,14 @@
  */
 
 import * as url from "url";
-import * as mongodb from "mongodb";
+import { Collection, GridFSBucket, ObjectId } from "mongodb";
 import * as querystring from "querystring";
 import * as vm from "vm";
 import * as config from "./config";
-import * as db from "./db";
+import { onConnect } from "./db";
 import * as query from "./query";
 import * as apiFunctions from "./api-functions";
+import { IncomingMessage, ServerResponse } from "http";
 import * as cache from "./cache";
 import { version as VERSION } from "../package.json";
 import { ping } from "./ping";
@@ -42,7 +43,24 @@ const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
 const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
 
-function throwError(err, httpResponse?): never {
+const collections = {
+  tasks: null as Collection,
+  devices: null as Collection,
+  presets: null as Collection,
+  objects: null as Collection,
+  "fs.files": null as Collection,
+  provisions: null as Collection,
+  virtualParameters: null as Collection,
+  faults: null as Collection,
+};
+let filesBucket: GridFSBucket;
+
+onConnect(async (db) => {
+  for (const k of Object.keys(collections)) collections[k] = db.collection(k);
+  filesBucket = new GridFSBucket(db);
+});
+
+function throwError(err: Error, httpResponse?: ServerResponse): never {
   if (httpResponse) {
     httpResponse.writeHead(500, { Connection: "close" });
     httpResponse.end(`${err.name}: ${err.message}`);
@@ -50,7 +68,10 @@ function throwError(err, httpResponse?): never {
   throw err;
 }
 
-export function listener(request, response): void {
+export function listener(
+  request: IncomingMessage,
+  response: ServerResponse
+): void {
   const chunks = [];
   let bytes = 0;
   response.setHeader("GenieACS-Version", VERSION);
@@ -81,11 +102,11 @@ export function listener(request, response): void {
       if (request.method === "PUT") {
         const preset = JSON.parse(body.toString());
         preset._id = presetName;
-        db.presetsCollection.replaceOne(
+        collections.presets.replaceOne(
           { _id: presetName },
           preset,
           { upsert: true },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             cache
@@ -94,7 +115,7 @@ export function listener(request, response): void {
                 response.writeHead(200);
                 response.end();
               })
-              .catch(err => {
+              .catch((err) => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -102,7 +123,7 @@ export function listener(request, response): void {
           }
         );
       } else if (request.method === "DELETE") {
-        db.presetsCollection.deleteOne({ _id: presetName }, err => {
+        collections.presets.deleteOne({ _id: presetName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -111,7 +132,7 @@ export function listener(request, response): void {
               response.writeHead(200);
               response.end();
             })
-            .catch(err => {
+            .catch((err) => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -128,11 +149,11 @@ export function listener(request, response): void {
       if (request.method === "PUT") {
         const object = JSON.parse(body.toString());
         object._id = objectName;
-        db.objectsCollection.replaceOne(
+        collections.objects.replaceOne(
           { _id: objectName },
           object,
           { upsert: true },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             cache
@@ -141,7 +162,7 @@ export function listener(request, response): void {
                 response.writeHead(200);
                 response.end();
               })
-              .catch(err => {
+              .catch((err) => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -149,7 +170,7 @@ export function listener(request, response): void {
           }
         );
       } else if (request.method === "DELETE") {
-        db.objectsCollection.deleteOne({ _id: objectName }, err => {
+        collections.objects.deleteOne({ _id: objectName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -158,7 +179,7 @@ export function listener(request, response): void {
               response.writeHead(200);
               response.end();
             })
-            .catch(err => {
+            .catch((err) => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -175,7 +196,7 @@ export function listener(request, response): void {
       if (request.method === "PUT") {
         const object = {
           _id: provisionName,
-          script: body.toString()
+          script: body.toString(),
         };
 
         try {
@@ -186,11 +207,11 @@ export function listener(request, response): void {
           return;
         }
 
-        db.provisionsCollection.replaceOne(
+        collections.provisions.replaceOne(
           { _id: provisionName },
           object,
           { upsert: true },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             cache
@@ -199,7 +220,7 @@ export function listener(request, response): void {
                 response.writeHead(200);
                 response.end();
               })
-              .catch(err => {
+              .catch((err) => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -207,7 +228,7 @@ export function listener(request, response): void {
           }
         );
       } else if (request.method === "DELETE") {
-        db.provisionsCollection.deleteOne({ _id: provisionName }, err => {
+        collections.provisions.deleteOne({ _id: provisionName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -216,7 +237,7 @@ export function listener(request, response): void {
               response.writeHead(200);
               response.end();
             })
-            .catch(err => {
+            .catch((err) => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -233,7 +254,7 @@ export function listener(request, response): void {
       if (request.method === "PUT") {
         const object = {
           _id: virtualParameterName,
-          script: body.toString()
+          script: body.toString(),
         };
 
         try {
@@ -244,11 +265,11 @@ export function listener(request, response): void {
           return;
         }
 
-        db.virtualParametersCollection.replaceOne(
+        collections.virtualParameters.replaceOne(
           { _id: virtualParameterName },
           object,
           { upsert: true },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             cache
@@ -257,7 +278,7 @@ export function listener(request, response): void {
                 response.writeHead(200);
                 response.end();
               })
-              .catch(err => {
+              .catch((err) => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -265,9 +286,9 @@ export function listener(request, response): void {
           }
         );
       } else if (request.method === "DELETE") {
-        db.virtualParametersCollection.deleteOne(
+        collections.virtualParameters.deleteOne(
           { _id: virtualParameterName },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             cache
@@ -276,7 +297,7 @@ export function listener(request, response): void {
                 response.writeHead(200);
                 response.end();
               })
-              .catch(err => {
+              .catch((err) => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -292,20 +313,20 @@ export function listener(request, response): void {
       const deviceId = querystring.unescape(r[1]);
       const tag = querystring.unescape(r[2]);
       if (request.method === "POST") {
-        db.devicesCollection.updateOne(
+        collections.devices.updateOne(
           { _id: deviceId },
           { $addToSet: { _tags: tag } },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
             response.writeHead(200);
             response.end();
           }
         );
       } else if (request.method === "DELETE") {
-        db.devicesCollection.updateOne(
+        collections.devices.updateOne(
           { _id: deviceId },
           { $pull: { _tags: tag } },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             response.writeHead(200);
@@ -323,12 +344,12 @@ export function listener(request, response): void {
         );
         const deviceId = faultId.split(":", 1)[0];
         const channel = faultId.slice(deviceId.length + 1);
-        db.faultsCollection.deleteOne({ _id: faultId }, err => {
+        collections.faults.deleteOne({ _id: faultId }, (err) => {
           if (err) return void throwError(err, response);
 
           if (channel.startsWith("task_")) {
-            const objId = new mongodb.ObjectID(channel.slice(5));
-            return void db.tasksCollection.deleteOne({ _id: objId }, err => {
+            const objId = new ObjectId(channel.slice(5));
+            return void collections.tasks.deleteOne({ _id: objId }, (err) => {
               if (err) return void throwError(err, response);
 
               cache
@@ -337,7 +358,7 @@ export function listener(request, response): void {
                   response.writeHead(200);
                   response.end();
                 })
-                .catch(err => {
+                .catch((err) => {
                   setTimeout(() => {
                     throwError(err, response);
                   });
@@ -351,7 +372,7 @@ export function listener(request, response): void {
               response.writeHead(200);
               response.end();
             })
-            .catch(err => {
+            .catch((err) => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -371,77 +392,114 @@ export function listener(request, response): void {
           task.device = deviceId;
           apiFunctions
             .insertTasks(task)
-            .then(() => {
-              cache
-                .del(`${deviceId}_tasks_faults_operations`)
-                .then(() => {
-                  if (urlParts.query.connection_request != null) {
-                    apiFunctions
-                      .connectionRequest(deviceId)
-                      .then(() => {
-                        const taskTimeout =
-                          (urlParts.query.timeout &&
-                            parseInt(urlParts.query.timeout as string)) ||
-                          config.get("DEVICE_ONLINE_THRESHOLD", deviceId);
+            .then(async () => {
+              const lastInform = Date.now();
 
-                        apiFunctions
-                          .watchTask(deviceId, task._id, taskTimeout)
-                          .then(status => {
-                            if (status === "timeout") {
-                              response.writeHead(
-                                202,
-                                "Task queued but not processed",
-                                {
-                                  "Content-Type": "application/json"
-                                }
-                              );
-                              response.end(JSON.stringify(task));
-                            } else if (status === "fault") {
-                              db.tasksCollection.findOne(
-                                { _id: task._id },
-                                (err, task2) => {
-                                  if (err)
-                                    return void throwError(err, response);
+              const socketTimeout: number = request.socket["timeout"];
 
-                                  response.writeHead(202, "Task faulted", {
-                                    "Content-Type": "application/json"
-                                  });
-                                  response.end(JSON.stringify(task2));
-                                }
-                              );
-                            } else {
-                              response.writeHead(200, {
-                                "Content-Type": "application/json"
-                              });
-                              response.end(JSON.stringify(task));
-                            }
-                          })
-                          .catch(err => {
-                            setTimeout(() => {
-                              throwError(err, response);
-                            });
-                          });
-                      })
-                      .catch(err => {
-                        response.writeHead(202, err.message, {
-                          "Content-Type": "application/json"
-                        });
-                        response.end(JSON.stringify(task));
-                      });
-                  } else {
-                    response.writeHead(202, {
-                      "Content-Type": "application/json"
-                    });
-                    response.end(JSON.stringify(task));
-                  }
-                })
-                .catch(err => {
-                  setTimeout(() => {
-                    throwError(err, response);
-                  });
+              // Disable socket timeout while waiting for session
+              if (socketTimeout) request.socket.setTimeout(0);
+
+              const notInSession = await apiFunctions.awaitSessionEnd(
+                deviceId,
+                30000
+              );
+              await cache.del(`${deviceId}_tasks_faults_operations`);
+              if (urlParts.query.connection_request == null) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, {
+                  "Content-Type": "application/json",
                 });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              if (!notInSession) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const status = await apiFunctions.connectionRequest(deviceId);
+
+              if (status) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, status, {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const onlineThreshold =
+                (urlParts.query.timeout &&
+                  parseInt(urlParts.query.timeout as string)) ||
+                (config.get("DEVICE_ONLINE_THRESHOLD", deviceId) as number);
+
+              const sessionStarted = await apiFunctions.awaitSessionStart(
+                deviceId,
+                lastInform,
+                onlineThreshold
+              );
+              if (!sessionStarted) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const sessionEnded = await apiFunctions.awaitSessionEnd(
+                deviceId,
+                120000
+              );
+              if (!sessionEnded) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const prom1 = collections.tasks.findOne(
+                { _id: task._id },
+                { projection: { _id: 1 } }
+              );
+              const prom2 = collections.faults.findOne(
+                { _id: `${deviceId}:task_${task._id}` },
+                {
+                  projection: { _id: 1 },
+                }
+              );
+
+              const [t, f] = await Promise.all([prom1, prom2]);
+
+              // Restore socket timeout
+              if (socketTimeout) request.socket.setTimeout(socketTimeout);
+
+              if (f) {
+                response.writeHead(202, "Task faulted", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(t || task));
+              } else if (t) {
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(t));
+              } else {
+                response.writeHead(200, {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+              }
             })
-            .catch(err => {
+            .catch((err) => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -454,7 +512,7 @@ export function listener(request, response): void {
               response.writeHead(200);
               response.end();
             })
-            .catch(err => {
+            .catch((err) => {
               response.writeHead(504);
               response.end(`${err.name}: ${err.message}`);
             });
@@ -472,8 +530,8 @@ export function listener(request, response): void {
       const action = r[2];
       if (!action || action === "/") {
         if (request.method === "DELETE") {
-          db.tasksCollection.findOne(
-            { _id: new mongodb.ObjectID(taskId) },
+          collections.tasks.findOne(
+            { _id: new ObjectId(taskId) },
             { projection: { device: 1 } },
             (err, task) => {
               if (err) return void throwError(err, response);
@@ -485,14 +543,14 @@ export function listener(request, response): void {
               }
 
               const deviceId = task.device;
-              db.tasksCollection.deleteOne(
-                { _id: new mongodb.ObjectID(taskId) },
-                err => {
+              collections.tasks.deleteOne(
+                { _id: new ObjectId(taskId) },
+                (err) => {
                   if (err) return void throwError(err, response);
 
-                  db.faultsCollection.deleteOne(
+                  collections.faults.deleteOne(
                     { _id: `${deviceId}:task_${taskId}` },
-                    err => {
+                    (err) => {
                       if (err) return void throwError(err, response);
 
                       cache
@@ -501,7 +559,7 @@ export function listener(request, response): void {
                           response.writeHead(200);
                           response.end();
                         })
-                        .catch(err => {
+                        .catch((err) => {
                           setTimeout(() => {
                             throwError(err, response);
                           });
@@ -518,16 +576,16 @@ export function listener(request, response): void {
         }
       } else if (action === "/retry") {
         if (request.method === "POST") {
-          db.tasksCollection.findOne(
-            { _id: new mongodb.ObjectID(taskId) },
+          collections.tasks.findOne(
+            { _id: new ObjectId(taskId) },
             { projection: { device: 1 } },
             (err, task) => {
               if (err) return void throwError(err, response);
 
               const deviceId = task.device;
-              db.faultsCollection.deleteOne(
+              collections.faults.deleteOne(
                 { _id: `${deviceId}:task_${taskId}` },
-                err => {
+                (err) => {
                   if (err) return void throwError(err, response);
 
                   cache
@@ -536,7 +594,7 @@ export function listener(request, response): void {
                       response.writeHead(200);
                       response.end();
                     })
-                    .catch(err => {
+                    .catch((err) => {
                       setTimeout(() => {
                         throwError(err, response);
                       });
@@ -562,25 +620,36 @@ export function listener(request, response): void {
           fileType: request.headers.filetype,
           oui: request.headers.oui,
           productClass: request.headers.productclass,
-          version: request.headers.version
+          version: request.headers.version,
         };
-        const bucket = new mongodb.GridFSBucket(db.client.db());
-        const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
-          metadata: metadata
-        });
+        filesBucket.delete((filename as unknown) as ObjectId, () => {
+          const uploadStream = filesBucket.openUploadStreamWithId(
+            filename,
+            filename,
+            {
+              metadata: metadata,
+            }
+          );
 
-        uploadStream.on("error", err => {
-          throwError(err, response);
-        });
+          uploadStream.on("error", (err) => {
+            throwError(err, response);
+          });
 
-        uploadStream.end(body, () => {
-          response.writeHead(201);
-          response.end();
+          uploadStream.end(body, () => {
+            response.writeHead(201);
+            response.end();
+          });
         });
       } else if (request.method === "DELETE") {
-        const bucket = new mongodb.GridFSBucket(db.client.db());
-        bucket.delete((filename as unknown) as mongodb.ObjectId, err => {
-          if (err) return void throwError(err, response);
+        filesBucket.delete((filename as unknown) as ObjectId, (err) => {
+          if (err) {
+            if (err.message.startsWith("FileNotFound")) {
+              response.writeHead(404);
+              response.end("404 Not Found");
+              return;
+            }
+            return void throwError(err, response);
+          }
 
           response.writeHead(200);
           response.end();
@@ -593,7 +662,11 @@ export function listener(request, response): void {
       const host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1]);
       ping(host, (err, res, stdout) => {
         if (err) {
-          if (!res) throwError(err, response);
+          if (!res) {
+            response.writeHead(500, { Connection: "close" });
+            response.end(`${err.name}: ${err.message}`);
+            return;
+          }
           response.writeHead(404, { "Cache-Control": "no-cache" });
           response.end(`${err.name}: ${err.message}`);
           return;
@@ -601,7 +674,7 @@ export function listener(request, response): void {
 
         response.writeHead(200, {
           "Content-Type": "text/plain",
-          "Cache-Control": "no-cache"
+          "Cache-Control": "no-cache",
         });
         response.end(stdout);
       });
@@ -621,7 +694,7 @@ export function listener(request, response): void {
           response.writeHead(200);
           response.end();
         })
-        .catch(err => {
+        .catch((err) => {
           setTimeout(() => {
             throwError(err, response);
           });
@@ -645,7 +718,7 @@ export function listener(request, response): void {
         return;
       }
 
-      const collection = db[`${collectionName}Collection`];
+      const collection = collections[collectionName];
       if (!collection) {
         response.writeHead(404);
         response.end("404 Not Found");
@@ -669,15 +742,15 @@ export function listener(request, response): void {
           break;
         case "tasks":
           q = query.sanitizeQueryTypes(q, {
-            _id: v => new mongodb.ObjectID(v),
-            timestamp: v => new Date(v),
-            retries: Number
+            _id: (v) => new ObjectId(v as string),
+            timestamp: (v) => new Date(v as number),
+            retries: Number,
           });
           break;
         case "faults":
           q = query.sanitizeQueryTypes(q, {
-            timestamp: v => new Date(v),
-            retries: Number
+            timestamp: (v) => new Date(v as number),
+            retries: Number,
           });
       }
 
@@ -713,7 +786,7 @@ export function listener(request, response): void {
 
         response.writeHead(200, {
           "Content-Type": "application/json",
-          total: total
+          total: total,
         });
 
         if (request.method === "HEAD") {
@@ -724,11 +797,11 @@ export function listener(request, response): void {
         response.write("[\n");
         i = 0;
         cur.forEach(
-          item => {
+          (item) => {
             if (i++) response.write(",\n");
             response.write(JSON.stringify(item));
           },
-          err => {
+          (err) => {
             if (err) return void throwError(err);
             response.end("\n]");
           }

lib/ping.ts

@@ -20,7 +20,7 @@
 import { platform } from "os";
 import { exec } from "child_process";
 
-interface Ping {
+export interface PingResult {
   packetsTransmitted: number;
   packetsReceived: number;
   packetLoss: number;
@@ -32,7 +32,7 @@ interface Ping {
 
 export function ping(
   host: string,
-  callback: (err, res?, stdout?) => void
+  callback: (err: Error, res?: PingResult, stdout?: string) => void
 ): void {
   let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
   switch (platform()) {
@@ -55,7 +55,7 @@ export function ping(
   }
 
   exec(cmd, (err, stdout) => {
-    let parsed: Ping;
+    let parsed: PingResult;
     if (stdout) {
       const m1 = stdout.match(parseRegExp1);
       if (m1) {
@@ -66,7 +66,7 @@ export function ping(
           min: +m1[4],
           avg: +m1[5],
           max: +m1[6],
-          mdev: +m1[7]
+          mdev: +m1[7],
         };
       } else {
         const m2 = stdout.match(parseRegExp2);
@@ -78,7 +78,7 @@ export function ping(
             min: null,
             avg: null,
             max: null,
-            mdev: null
+            mdev: null,
           };
         }
       }

lib/query.ts

@@ -17,7 +17,9 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import * as common from "./common";
+function isObject(obj: any): boolean {
+  return Object.prototype.toString.call(obj) === "[object Object]";
+}
 
 function stringToRegexp(input, flags?): RegExp | false {
   if (input.indexOf("*") === -1) return false;
@@ -34,8 +36,8 @@ function stringToRegexp(input, flags?): RegExp | false {
 }
 
 function normalize(input): any {
-  if (common.typeOf(input) === common.STRING_TYPE) {
-    const vals = [input];
+  if (typeof input === "string") {
+    const vals: any = [input];
     const m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input);
     if (m) vals.push({ $regex: new RegExp(m[1], m[2]) });
 
@@ -52,14 +54,14 @@ function normalize(input): any {
   return input;
 }
 
-export function expandValue(value): any[] {
-  if (common.typeOf(value) === common.ARRAY_TYPE) {
+function expandValue(value: unknown): unknown[] {
+  if (Array.isArray(value)) {
     let a = [];
     for (const j of value) a = a.concat(expandValue(j));
     return [a];
-  } else if (common.typeOf(value) !== common.OBJECT_TYPE) {
+  } else if (!isObject(value)) {
     const n = normalize(value);
-    if (common.typeOf(n) !== common.ARRAY_TYPE) return [n];
+    if (!Array.isArray(n)) return [n];
     else return n;
   }
 
@@ -103,12 +105,14 @@ function permute(param, val): any[] {
   return conditions;
 }
 
-export function expand(query): {} {
+export function expand(
+  query: Record<string, unknown>
+): Record<string, unknown> {
   const newQuery = {};
   for (const [k, v] of Object.entries(query)) {
     if (k[0] === "$") {
       // Operator
-      newQuery[k] = (v as any[]).map(e => expand(e));
+      newQuery[k] = (v as any[]).map((e) => expand(e));
     } else {
       const conditions = permute(k, v);
       if (conditions.length > 1) {
@@ -129,100 +133,16 @@ export function expand(query): {} {
   return newQuery;
 }
 
-function testExpressions(params, expressions, lop): boolean {
-  for (const f of expressions) {
-    const res = test(params, f);
-    switch (lop) {
-      case "$and":
-        if (!res) return false;
-        break;
-      case "$or":
-        if (res) return true;
-        break;
-      case "$nor":
-        if (res) return false;
-    }
-  }
-
-  switch (lop) {
-    case "$and":
-      return true;
-    case "$or":
-      return false;
-    case "$nor":
-      return true;
-    default:
-      throw new Error("Unknown logical operator");
-  }
-}
-
-export function test(params, query): boolean {
-  let res;
-  for (const [k, v] of Object.entries(query)) {
-    if (k.charAt(0) === "$") {
-      // Logical operator
-      res = testExpressions(params, v, k);
-    } else {
-      const value = params[k];
-      if (common.typeOf(v) !== common.OBJECT_TYPE) {
-        if (common.typeOf(value) === common.ARRAY_TYPE) {
-          // TODO comparing array to regex, array to array, and object to object
-          res = value.indexOf(v) !== -1;
-        } else {
-          if (common.typeOf(v) === common.REGEXP_TYPE)
-            res = (v as RegExp).test(value);
-          else res = v === value;
-        }
-      } else {
-        for (const [k2, v2] of Object.entries(v)) {
-          switch (k2) {
-            case "$ne":
-              if (common.typeOf(value) === common.ARRAY_TYPE)
-                res = value.indexOf(v2) === -1;
-              else res = value !== v2;
-              break;
-            case "$lt":
-              res = value < v2;
-              break;
-            case "$lte":
-              res = value <= v2;
-              break;
-            case "$gt":
-              res = value > v2;
-              break;
-            case "$gte":
-              res = value >= v2;
-              break;
-            case "$regex":
-              res = v2.test(value);
-              break;
-            case "$in":
-              throw new Error("Operator not supported");
-            case "$nin":
-              throw new Error("Operator not supported");
-            case "$all":
-              throw new Error("Operator not supported");
-            case "$exists":
-              throw new Error("Operator not supported");
-            default:
-              throw new Error("Operator not supported");
-          }
-        }
-      }
-    }
-
-    if (!res) return false;
-  }
-  return true;
-}
-
-export function sanitizeQueryTypes(query, types): {} {
+export function sanitizeQueryTypes(
+  query: Record<string, unknown>,
+  types: Record<string, (v: unknown) => unknown>
+): Record<string, unknown> {
   for (const [k, v] of Object.entries(query)) {
     if (k[0] === "$") {
       // Logical operator
       for (const vv of v as any[]) sanitizeQueryTypes(vv, types);
     } else if (k in types) {
-      if (common.typeOf(v) === common.OBJECT_TYPE) {
+      if (isObject(v)) {
         for (const [kk, vv] of Object.entries(v)) {
           switch (kk) {
             case "$in":

lib/sandbox.ts

@@ -109,6 +109,16 @@ class SandboxDate {
 
     return t;
   }
+
+  public static parse(dateString: string): number {
+    return Date.parse(dateString);
+  }
+
+  public static UTC(
+    ...args: [number, number?, number?, number?, number?, number?, number?]
+  ): number {
+    return Date.UTC(...args);
+  }
 }
 
 function random(): number {
@@ -117,7 +127,7 @@ function random(): number {
   return state.rng();
 }
 
-random.seed = function(s) {
+random.seed = function (s) {
   state.rng = seedrandom(s);
 };
 
@@ -125,7 +135,7 @@ class ParameterWrapper {
   public constructor(path: Path, attributes, unpacked?, unpackedRevision?) {
     for (const attrName of attributes) {
       Object.defineProperty(this, attrName, {
-        get: function() {
+        get: function () {
           if (state.uncommitted) commit();
 
           if (state.revision !== unpackedRevision) {
@@ -147,12 +157,12 @@ class ParameterWrapper {
           if (!attr) return UNDEFINED;
 
           return attr[1];
-        }
+        },
       });
     }
 
     Object.defineProperty(this, "path", {
-      get: function() {
+      get: function () {
         if (state.uncommitted) commit();
 
         if (state.revision !== unpackedRevision) {
@@ -167,11 +177,11 @@ class ParameterWrapper {
         if (!unpacked.length) return UNDEFINED;
 
         return unpacked[0].toString();
-      }
+      },
     });
 
     Object.defineProperty(this, "size", {
-      get: function() {
+      get: function () {
         if (state.uncommitted) commit();
 
         if (state.revision !== unpackedRevision) {
@@ -186,10 +196,10 @@ class ParameterWrapper {
         if (!unpacked.length) return UNDEFINED;
 
         return unpacked.length;
-      }
+      },
     });
 
-    this[Symbol.iterator] = function*() {
+    this[Symbol.iterator] = function* () {
       if (state.uncommitted) commit();
 
       if (state.revision !== unpackedRevision) {
@@ -225,7 +235,7 @@ function declare(
     pathSet: null,
     attrGet: null,
     attrSet: null,
-    defer: true
+    defer: true,
   };
 
   const attrs = new Set();
@@ -280,9 +290,9 @@ function commit(): void {
   }
 }
 
-function ext(): any {
+function ext(...args: unknown[]): any {
   ++state.extCounter;
-  const extCall = Array.from(arguments).map(String);
+  const extCall = args.map(String);
   const key = `${state.revision}: ${JSON.stringify(extCall)}`;
 
   if (key in state.sessionContext.extensionsCache)
@@ -292,11 +302,11 @@ function ext(): any {
   throw EXT;
 }
 
-function log(msg: string, meta: {}): void {
+function log(msg: string, meta: Record<string, unknown>): void {
   if (state.revision === state.maxRevision && state.extCounter >= 0) {
     const details = Object.assign({}, meta, {
       sessionContext: state.sessionContext,
-      message: `Script: ${msg}`
+      message: `Script: ${msg}`,
     });
 
     delete details["hostname"];
@@ -332,8 +342,8 @@ function errorToFault(err: Error): Fault {
     message: err.message,
     detail: {
       name: err.name,
-      message: err.message
-    }
+      message: err.message,
+    },
   };
 
   if (err.stack) {
@@ -355,7 +365,7 @@ function errorToFault(err: Error): Fault {
 
 export async function run(
   script: vm.Script,
-  globals: {},
+  globals: Record<string, unknown>,
   sessionContext: SessionContext,
   startRevision: number,
   maxRevision: number,
@@ -370,7 +380,7 @@ export async function run(
     extensions: {},
     clear: [],
     rng: null,
-    extCounter: extCounter
+    extCounter: extCounter,
   };
 
   for (const n of Object.keys(context)) delete context[n];
@@ -393,7 +403,7 @@ export async function run(
         clear: null,
         declare: null,
         done: false,
-        returnValue: null
+        returnValue: null,
       };
     }
   }
@@ -413,7 +423,7 @@ export async function run(
       clear: null,
       declare: null,
       done: false,
-      returnValue: null
+      returnValue: null,
     };
   }
 
@@ -433,6 +443,6 @@ export async function run(
     clear: _state.clear,
     declare: _state.declarations,
     done: status === 0,
-    returnValue: ret
+    returnValue: ret,
   };
 }

lib/scheduling.ts

@@ -21,10 +21,7 @@ import * as crypto from "crypto";
 import * as later from "later";
 
 function md532(str): number {
-  const digest = crypto
-    .createHash("md5")
-    .update(str)
-    .digest();
+  const digest = crypto.createHash("md5").update(str).digest();
   return (
     digest.readUInt32LE(0) ^
     digest.readUInt32LE(4) ^
@@ -33,29 +30,37 @@ function md532(str): number {
   );
 }
 
-export function variance(deviceId, vrnc): number {
+export function variance(deviceId: string, vrnc: number): number {
   return (md532(deviceId) >>> 0) % vrnc;
 }
 
-export function interval(timestamp, intrvl, offset = 0): number {
+export function interval(
+  timestamp: number,
+  intrvl: number,
+  offset = 0
+): number {
   return Math.trunc((timestamp + offset) / intrvl) * intrvl - offset;
 }
 
-export function parseCron(cronExp): any {
+export function parseCron(cronExp: string): any {
   const parts = cronExp.trim().split(/\s+/);
   if (parts.length === 5) parts.unshift("*");
 
   return later.schedule(later.parse.cron(parts.join(" "), true));
 }
 
-export function cron(timestamp, schedule, offset = 0): number[] {
+export function cron(
+  timestamp: number,
+  schedule: unknown,
+  offset = 0
+): number[] {
   // TODO later.js doesn't throw erorr if expression is invalid!
   const ret = [0, 0];
 
-  const prev = schedule.prev(1, new Date(timestamp + offset));
+  const prev = (schedule as any).prev(1, new Date(timestamp + offset));
   if (prev) ret[0] = prev.setMilliseconds(0) - offset;
 
-  const next = schedule.next(1, new Date(timestamp + offset + 1000));
+  const next = (schedule as any).next(1, new Date(timestamp + offset + 1000));
   if (next) ret[1] = next.setMilliseconds(0) - offset;
 
   return ret;

lib/server.ts

@@ -20,6 +20,7 @@
 import * as fs from "fs";
 import * as http from "http";
 import * as https from "https";
+import { Socket } from "net";
 import * as path from "path";
 import { ROOT_DIR } from "./config";
 
@@ -51,35 +52,43 @@ function closeServer(timeout, callback): void {
   });
 }
 
+interface ServerOptions {
+  port?: number;
+  host?: string;
+  ssl?: { key: string; cert: string };
+  timeout?: number;
+  keepAliveTimeout?: number;
+  onConnection?: (socket: Socket) => void;
+}
+
 export function start(
-  port,
-  networkInterface,
-  ssl,
-  _listener,
-  onConnection?,
-  keepAliveTimeout: number = -1
+  options: ServerOptions,
+  _listener: http.RequestListener
 ): void {
   listener = _listener;
 
-  if (ssl && ssl.key && ssl.cert) {
-    const options = {
-      key: ssl.key
+  if (options.ssl) {
+    const opts = {
+      key: options.ssl.key
         .split(":")
-        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
-      cert: ssl.cert
+        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
+      cert: options.ssl.cert
         .split(":")
-        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim())))
+        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
     };
 
-    server = https.createServer(options, listener);
-    if (onConnection != null) server.on("secureConnection", onConnection);
+    server = https.createServer(opts, listener);
+    if (options.onConnection)
+      server.on("secureConnection", options.onConnection);
   } else {
     server = http.createServer(listener);
-    if (onConnection != null) server.on("connection", onConnection);
+    if (options.onConnection) server.on("connection", options.onConnection);
   }
 
-  if (keepAliveTimeout >= 0) server.keepAliveTimeout = keepAliveTimeout;
-  server.listen(port, networkInterface);
+  server.timeout = options.timeout || 0;
+  if (options.keepAliveTimeout != null)
+    server.keepAliveTimeout = options.keepAliveTimeout;
+  server.listen({ port: options.port, host: options.host });
 }
 
 export function stop(): Promise<void> {

lib/soap-libxmljs.ts

@@ -1,745 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { parseXml, Document, Element, ParserOptions } from "libxmljs";
-import * as config from "./config";
-import { version as VERSION } from "../package.json";
-import {
-  CpeGetResponse,
-  CpeSetResponse,
-  InformRequest,
-  AcsResponse,
-  CpeRequest,
-  FaultStruct,
-  SpvFault,
-  CpeFault,
-  SoapMessage,
-  TransferCompleteRequest
-} from "./types";
-
-const SERVER_NAME = `GenieACS/${VERSION}`;
-
-const NAMESPACES = {
-  "1.0": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-0"
-  },
-  "1.1": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-1"
-  },
-  "1.2": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
-  },
-  "1.3": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
-  },
-  "1.4": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-3"
-  }
-};
-
-const LIBXMLJS_PARSE_OPTIONS: ParserOptions = {
-  nocdata: true,
-  recover: !!config.get("XML_RECOVER"),
-  /* eslint-disable-next-line @typescript-eslint/camelcase */
-  ignore_enc: !!config.get("XML_IGNORE_ENC")
-};
-
-interface WriterOptions {
-  declaration?: boolean;
-  format?: boolean;
-  selfCloseEmpty?: boolean;
-  whitespace?: boolean;
-  type?: "XML" | "HTML" | "XHTML";
-}
-
-declare module "libxmljs" {
-  interface Document {
-    toString(formatting: boolean | WriterOptions): string;
-  }
-}
-
-const LIBXMLJS_SAVE_OPTIONS: WriterOptions = {
-  format: !!config.get("XML_FORMAT"),
-  declaration: !config.get("XML_NO_DECL"),
-  selfCloseEmpty: !config.get("XML_NO_EMPTY")
-};
-
-let warnings;
-
-function parseBool(v): boolean {
-  v = "" + v;
-  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
-  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
-    return false;
-  else return null;
-}
-
-function event(xml): string[] {
-  return xml
-    .childNodes()
-    .filter(n => n.name() === "EventStruct")
-    .map(c =>
-      c
-        .childNodes()
-        .find(n => n.name() === "EventCode")
-        .text()
-        .trim()
-    );
-}
-
-function parameterInfoList(xml): [string, boolean][] {
-  return xml
-    .childNodes()
-    .filter(e => e.name() === "ParameterInfoStruct")
-    .map(e => {
-      let param, value;
-      for (const c of e.childNodes()) {
-        switch (c.name()) {
-          case "Name":
-            param = c.text();
-            break;
-          case "Writable":
-            value = c.text();
-            break;
-        }
-      }
-
-      let parsed = parseBool(value);
-
-      if (parsed === null) {
-        warnings.push({
-          message: "Invalid writable attribute",
-          parameter: param
-        });
-        parsed = false;
-      }
-
-      return [param, parsed];
-    });
-}
-
-function parameterValueList(
-  xml
-): [string, string | number | boolean, string][] {
-  return xml
-    .childNodes()
-    .filter(e => e.name() === "ParameterValueStruct")
-    .map(e => {
-      let valueElement, param;
-      for (const c of e.childNodes()) {
-        switch (c.name()) {
-          case "Name":
-            param = c.text();
-            break;
-          case "Value":
-            valueElement = c;
-            break;
-        }
-      }
-
-      const valueType = valueElement.attr("type").value();
-      const value = valueElement.text();
-      let parsed = value;
-      if (valueType === "xsd:boolean") {
-        parsed = parseBool(value);
-        if (parsed === null) {
-          warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
-          });
-          parsed = value;
-        }
-      } else if (valueType === "xsd:int" || valueType === "xsd:unsignedInt") {
-        parsed = parseInt(value);
-        if (isNaN(parsed)) {
-          warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
-          });
-          parsed = value;
-        }
-      } else if (valueType === "xsd:dateTime") {
-        parsed = Date.parse(value);
-        if (isNaN(parsed)) {
-          warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
-          });
-          parsed = value;
-        }
-      }
-
-      return [param, parsed, valueType];
-    });
-}
-
-function GetParameterNames(xml, methodRequest): void {
-  const el = xml.node("cwmp:GetParameterNames");
-  el.node("ParameterPath").text(methodRequest.parameterPath);
-  el.node("NextLevel").text(+methodRequest.nextLevel);
-}
-
-function GetParameterNamesResponse(xml): CpeGetResponse {
-  return {
-    name: "GetParameterNamesResponse",
-    parameterList: parameterInfoList(
-      xml.childNodes().find(n => n.name() === "ParameterList")
-    )
-  };
-}
-
-function GetParameterValues(xml, methodRequest): void {
-  const el = xml.node("cwmp:GetParameterValues").node("ParameterNames");
-  el.attr({
-    "soap-enc:arrayType": `xsd:string[${methodRequest.parameterNames.length}]`
-  });
-
-  for (const p of methodRequest.parameterNames) el.node("string").text(p);
-}
-
-function GetParameterValuesResponse(xml): CpeGetResponse {
-  return {
-    name: "GetParameterValuesResponse",
-    parameterList: parameterValueList(
-      xml.childNodes().find(n => n.name() === "ParameterList")
-    )
-  };
-}
-
-function SetParameterValues(xml, methodRequest): void {
-  const el = xml.node("cwmp:SetParameterValues");
-  const paramList = el.node("ParameterList");
-  paramList.attr({
-    "soap-enc:arrayType": `cwmp:ParameterValueStruct[${methodRequest.parameterList.length}]`
-  });
-
-  for (const p of methodRequest.parameterList) {
-    const pvs = paramList.node("ParameterValueStruct");
-    pvs.node("Name").text(p[0]);
-    let val = p[1];
-    if (p[2] === "xsd:dateTime" && typeof val === "number") {
-      val = new Date(val).toISOString();
-      if (methodRequest.DATETIME_MILLISECONDS === false)
-        val = val.replace(".000", "");
-    }
-    if (p[2] === "xsd:boolean" && typeof val === "boolean")
-      if (methodRequest.BOOLEAN_LITERAL === false) val = +val;
-
-    pvs
-      .node("Value")
-      .attr({ "xsi:type": p[2] })
-      .text(val);
-  }
-
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-function SetParameterValuesResponse(xml): CpeSetResponse {
-  return {
-    name: "SetParameterValuesResponse",
-    status: parseInt(
-      xml
-        .childNodes()
-        .find(n => n.name() === "Status")
-        .text()
-    )
-  };
-}
-
-function AddObject(xml, methodRequest): void {
-  const el = xml.node("cwmp:AddObject");
-  el.node("ObjectName").text(methodRequest.objectName);
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-function AddObjectResponse(xml): CpeSetResponse {
-  let instanceNumber, status;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "InstanceNumber":
-        instanceNumber = parseInt(c.text());
-        break;
-      case "Status":
-        status = parseInt(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "AddObjectResponse",
-    instanceNumber: instanceNumber,
-    status: status
-  };
-}
-
-function DeleteObject(xml, methodRequest): void {
-  const el = xml.node("cwmp:DeleteObject");
-  el.node("ObjectName").text(methodRequest.objectName);
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-function DeleteObjectResponse(xml): CpeSetResponse {
-  return {
-    name: "DeleteObjectResponse",
-    status: parseInt(
-      xml
-        .childNodes()
-        .find(n => n.name() === "Status")
-        .text()
-    )
-  };
-}
-
-function Reboot(xml, methodRequest): void {
-  const el = xml.node("cwmp:Reboot");
-  el.node("CommandKey").text(methodRequest.commandKey || "");
-}
-
-function RebootResponse(): CpeSetResponse {
-  return {
-    name: "RebootResponse"
-  };
-}
-
-function FactoryReset(xml): void {
-  xml.node("cwmp:FactoryReset");
-}
-
-function FactoryResetResponse(): CpeSetResponse {
-  return {
-    name: "FactoryResetResponse"
-  };
-}
-
-function Download(xml, methodRequest): void {
-  const el = xml.node("cwmp:Download");
-  el.node("CommandKey").text(methodRequest.commandKey || "");
-  el.node("FileType").text(methodRequest.fileType);
-  el.node("URL").text(methodRequest.url);
-  el.node("Username").text(methodRequest.username || "");
-  el.node("Password").text(methodRequest.password || "");
-  el.node("FileSize").text(methodRequest.fileSize || "0");
-  el.node("TargetFileName").text(methodRequest.targetFileName || "");
-  el.node("DelaySeconds").text(methodRequest.delaySeconds || "0");
-  el.node("SuccessURL").text(methodRequest.successUrl || "");
-  el.node("FailureURL").text(methodRequest.failureUrl || "");
-}
-
-function DownloadResponse(xml): CpeSetResponse {
-  let status, startTime, completeTime;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "Status":
-        status = parseInt(c.text());
-        break;
-      case "StartTime":
-        startTime = Date.parse(c.text());
-        break;
-      case "CompleteTime":
-        completeTime = Date.parse(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "DownloadResponse",
-    status: status,
-    startTime: startTime,
-    completeTime: completeTime
-  };
-}
-
-function Inform(xml): InformRequest {
-  let retryCount, evnt, parameterList;
-  const deviceId = {
-    Manufacturer: null,
-    OUI: null,
-    ProductClass: null,
-    SerialNumber: null
-  };
-
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "ParameterList":
-        parameterList = parameterValueList(c);
-        break;
-      case "DeviceId":
-        for (const cc of c.childNodes()) {
-          const n = cc.name();
-          if (n in deviceId) deviceId[n] = cc.text();
-        }
-        break;
-      case "Event":
-        evnt = event(c);
-        break;
-      case "RetryCount":
-        retryCount = parseInt(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "Inform",
-    parameterList: parameterList,
-    deviceId: deviceId,
-    event: evnt,
-    retryCount: retryCount
-  };
-}
-
-function InformResponse(xml): void {
-  xml
-    .node("cwmp:InformResponse")
-    .node("MaxEnvelopes")
-    .text(1);
-}
-
-function GetRPCMethods(): AcsResponse {
-  return { name: "GetRPCMethods" };
-}
-
-function GetRPCMethodsResponse(xml, methodResponse): void {
-  const el = xml.node("cwmp:GetRPCMethodsResponse").node("MethodList");
-  el.attr({
-    "soap-enc:arrayType": `xsd:string[${methodResponse.methodList.length}]`
-  });
-
-  for (const m of methodResponse.methodList) el.node("string").text(m);
-}
-
-function TransferComplete(xml): TransferCompleteRequest {
-  let commandKey, _faultStruct, startTime, completeTime;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "CommandKey":
-        commandKey = c.text();
-        break;
-      case "FaultStruct":
-        _faultStruct = faultStruct(c);
-        break;
-      case "StartTime":
-        startTime = Date.parse(c.text());
-        break;
-      case "CompleteTime":
-        completeTime = Date.parse(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "TransferComplete",
-    commandKey: commandKey,
-    faultStruct: _faultStruct,
-    startTime: startTime,
-    completeTime: completeTime
-  };
-}
-
-function TransferCompleteResponse(xml): void {
-  xml.node("cwmp:TransferCompleteResponse").text("");
-}
-
-function RequestDownload(xml): CpeRequest {
-  return {
-    name: "RequestDownload",
-    fileType: xml
-      .childNodes()
-      .find(n => n.name() === "FileType")
-      .text()
-  };
-}
-
-function RequestDownloadResponse(xml): void {
-  xml.node("cwmp:RequestDownloadResponse").text("");
-}
-
-function faultStruct(xml): FaultStruct {
-  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "FaultCode":
-        faultCode = c.text();
-        break;
-      case "FaultString":
-        faultString = c.text();
-        break;
-      case "SetParameterValuesFault":
-        setParameterValuesFault = setParameterValuesFault || [];
-        for (const cc of c.childNodes()) {
-          switch (cc.name()) {
-            case "ParameterName":
-              pn = cc.text();
-              break;
-            case "FaultCode":
-              fc = cc.text();
-              break;
-            case "FaultString":
-              fs = cc.text();
-              break;
-          }
-        }
-        setParameterValuesFault.push({
-          parameterName: pn,
-          faultCode: fc,
-          faultString: fs
-        });
-    }
-  }
-
-  return { faultCode, faultString, setParameterValuesFault };
-}
-
-function fault(xml): CpeFault {
-  let faultCode, faultString, detail;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "faultcode":
-        faultCode = c.text();
-        break;
-      case "faultstring":
-        faultString = c.text();
-        break;
-      case "detail":
-        detail = faultStruct(c.childNodes().find(n => n.name() === "Fault"));
-        break;
-    }
-  }
-
-  return { faultCode, faultString, detail };
-}
-
-export function request(data, cwmpVersion, warn): SoapMessage {
-  warnings = warn;
-
-  const rpc = {
-    id: null,
-    cwmpVersion: cwmpVersion,
-    sessionTimeout: null,
-    cpeRequest: null,
-    cpeFault: null,
-    cpeResponse: null
-  };
-
-  if (!data.length) return rpc;
-
-  const xml = parseXml(data, LIBXMLJS_PARSE_OPTIONS);
-
-  let headerElement, bodyElement;
-
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "Header":
-        headerElement = c;
-        break;
-      case "Body":
-        bodyElement = c;
-        break;
-    }
-  }
-
-  if (headerElement) {
-    for (const c of headerElement.childNodes()) {
-      switch (c.name()) {
-        case "ID":
-          rpc.id = c.text();
-          break;
-        case "sessionTimeout":
-          rpc.sessionTimeout = parseInt(c.text());
-          break;
-      }
-    }
-  }
-
-  for (const methodElement of bodyElement.childNodes()) {
-    const methodName = methodElement.name();
-    if (methodName === "text") continue;
-
-    if (!rpc.cwmpVersion && methodName !== "Fault") {
-      switch (methodElement.namespace().href()) {
-        case "urn:dslforum-org:cwmp-1-0":
-          rpc.cwmpVersion = "1.0";
-          break;
-        case "urn:dslforum-org:cwmp-1-1":
-          rpc.cwmpVersion = "1.1";
-          break;
-        case "urn:dslforum-org:cwmp-1-2":
-          if (rpc.sessionTimeout) rpc.cwmpVersion = "1.3";
-          else rpc.cwmpVersion = "1.2";
-
-          break;
-        case "urn:dslforum-org:cwmp-1-3":
-          rpc.cwmpVersion = "1.4";
-          break;
-        default:
-          throw new Error("Unrecognized CWMP version");
-      }
-    }
-
-    switch (methodName) {
-      case "Inform":
-        rpc.cpeRequest = Inform(methodElement);
-        break;
-      case "GetRPCMethods":
-        rpc.cpeRequest = GetRPCMethods();
-        break;
-      case "TransferComplete":
-        rpc.cpeRequest = TransferComplete(methodElement);
-        break;
-      case "RequestDownload":
-        rpc.cpeRequest = RequestDownload(methodElement);
-        break;
-      case "GetParameterNamesResponse":
-        rpc.cpeResponse = GetParameterNamesResponse(methodElement);
-        break;
-      case "GetParameterValuesResponse":
-        rpc.cpeResponse = GetParameterValuesResponse(methodElement);
-        break;
-      case "SetParameterValuesResponse":
-        rpc.cpeResponse = SetParameterValuesResponse(methodElement);
-        break;
-      case "AddObjectResponse":
-        rpc.cpeResponse = AddObjectResponse(methodElement);
-        break;
-      case "DeleteObjectResponse":
-        rpc.cpeResponse = DeleteObjectResponse(methodElement);
-        break;
-      case "RebootResponse":
-        rpc.cpeResponse = RebootResponse();
-        break;
-      case "FactoryResetResponse":
-        rpc.cpeResponse = FactoryResetResponse();
-        break;
-      case "DownloadResponse":
-        rpc.cpeResponse = DownloadResponse(methodElement);
-        break;
-      case "Fault":
-        rpc.cpeFault = fault(methodElement);
-        break;
-      default:
-        throw new Error(`8000 Method not supported ${methodName}`);
-    }
-  }
-
-  return rpc;
-}
-
-function createSoapEnv(cwmpVersion: string): Element {
-  const xml = new Document();
-  const env = xml.node("soap-env:Envelope");
-
-  for (const prefix of Object.keys(NAMESPACES[cwmpVersion]))
-    env.defineNamespace(prefix, NAMESPACES[cwmpVersion][prefix]);
-
-  return env;
-}
-
-export function response(rpc): { code: number; headers: {}; data: string } {
-  const headers = {
-    Server: SERVER_NAME,
-    SOAPServer: SERVER_NAME
-  };
-
-  if (!rpc) return { code: 204, headers: headers, data: "" };
-
-  const env = createSoapEnv(rpc.cwmpVersion);
-  const header = env.node("soap-env:Header");
-  header
-    .node("cwmp:ID")
-    .attr({
-      "soap-env:mustUnderstand": "1"
-    })
-    .text(rpc.id);
-
-  const body = env.node("soap-env:Body");
-
-  if (rpc.acsResponse) {
-    switch (rpc.acsResponse.name) {
-      case "InformResponse":
-        InformResponse(body);
-        break;
-      case "GetRPCMethodsResponse":
-        GetRPCMethodsResponse(body, rpc.acsResponse);
-        break;
-      case "TransferCompleteResponse":
-        TransferCompleteResponse(body);
-        break;
-      case "RequestDownloadResponse":
-        RequestDownloadResponse(body);
-        break;
-      default:
-        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
-    }
-  } else if (rpc.acsRequest) {
-    switch (rpc.acsRequest.name) {
-      case "GetParameterNames":
-        GetParameterNames(body, rpc.acsRequest);
-        break;
-      case "GetParameterValues":
-        GetParameterValues(body, rpc.acsRequest);
-        break;
-      case "SetParameterValues":
-        SetParameterValues(body, rpc.acsRequest);
-        break;
-      case "AddObject":
-        AddObject(body, rpc.acsRequest);
-        break;
-      case "DeleteObject":
-        DeleteObject(body, rpc.acsRequest);
-        break;
-      case "Reboot":
-        Reboot(body, rpc.acsRequest);
-        break;
-      case "FactoryReset":
-        FactoryReset(body);
-        break;
-      case "Download":
-        Download(body, rpc.acsRequest);
-        break;
-      default:
-        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
-    }
-  }
-
-  headers["Content-Type"] = 'text/xml; charset="utf-8"';
-  return {
-    code: 200,
-    headers: headers,
-    data: env.doc().toString(LIBXMLJS_SAVE_OPTIONS)
-  };
-}

lib/soap.ts

@@ -22,22 +22,35 @@ import {
   Element,
   parseAttrs,
   encodeEntities,
-  decodeEntities
+  decodeEntities,
 } from "./xml-parser";
 import memoize from "./common/memoize";
 import { version as VERSION } from "../package.json";
 import {
-  CpeGetResponse,
-  CpeSetResponse,
   InformRequest,
-  AcsResponse,
-  CpeRequest,
   FaultStruct,
   SpvFault,
   CpeFault,
   SoapMessage,
-  TransferCompleteRequest
+  TransferCompleteRequest,
+  AcsRequest,
+  GetParameterNamesResponse,
+  GetParameterValuesResponse,
+  GetParameterAttributesResponse,
+  SetParameterValuesResponse,
+  SetParameterAttributesResponse,
+  AddObjectResponse,
+  DeleteObject,
+  DeleteObjectResponse,
+  RebootResponse,
+  FactoryResetResponse,
+  DownloadResponse,
+  UploadResponse,
+  GetRPCMethodsRequest,
+  RequestDownloadRequest,
+  AcsResponse,
 } from "./types";
+import Path from "./common/path";
 
 const SERVER_NAME = `GenieACS/${VERSION}`;
 
@@ -47,60 +60,58 @@ const NAMESPACES = {
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-0"
+    cwmp: "urn:dslforum-org:cwmp-1-0",
   },
   "1.1": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-1"
+    cwmp: "urn:dslforum-org:cwmp-1-1",
   },
   "1.2": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
+    cwmp: "urn:dslforum-org:cwmp-1-2",
   },
   "1.3": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
+    cwmp: "urn:dslforum-org:cwmp-1-2",
   },
   "1.4": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-3"
-  }
+    cwmp: "urn:dslforum-org:cwmp-1-3",
+  },
 };
 
-let warnings;
+let warnings: Record<string, unknown>[];
 
 const memoizedParseAttrs = memoize(parseAttrs);
 
-function parseBool(v): boolean {
-  v = "" + v;
-  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
-  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
-    return false;
-  else return null;
+function parseBool(v: string): boolean {
+  if (v === "true" || v === "1") return true;
+  if (v === "false" || v === "0") return false;
+  return null;
 }
 
 function event(xml: Element): string[] {
   return xml.children
-    .filter(n => n.localName === "EventStruct")
-    .map(c => c.children.find(n => n.localName === "EventCode").text.trim());
+    .filter((n) => n.localName === "EventStruct")
+    .map((c) => c.children.find((n) => n.localName === "EventCode").text);
 }
 
-function parameterInfoList(xml: Element): [string, boolean][] {
+function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
   return xml.children
-    .filter(e => e.localName === "ParameterInfoStruct")
-    .map<[string, boolean]>(e => {
+    .filter((e) => e.localName === "ParameterInfoStruct")
+    .map<[Path, boolean, boolean]>((e) => {
       let param: string, value: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -115,30 +126,34 @@ function parameterInfoList(xml: Element): [string, boolean][] {
 
       let parsed: boolean = parseBool(value);
 
-      if (parsed === null) {
+      if (parsed == null) {
         warnings.push({
-          message: "Invalid writable attribute",
-          parameter: param
+          message: "Missing or invalid XML node",
+          element: "Writable",
+          parameter: param,
         });
         parsed = false;
       }
 
-      return [param, parsed];
+      if (param && !param.endsWith("."))
+        return [Path.parse(param), false, parsed];
+      else return [Path.parse(param.slice(0, -1)), true, parsed];
     });
 }
 
-const getValueType = memoize((str: string) =>
-  parseAttrs(str)
-    .find(s => s.localName === "type")
-    .value.trim()
-);
+const getValueType = memoize((str: string) => {
+  const attrs = parseAttrs(str);
+  for (const attr of attrs) if (attr.localName === "type") return attr.value;
+
+  return null;
+});
 
 function parameterValueList(
   xml: Element
-): [string, string | number | boolean, string][] {
+): [Path, string | number | boolean, string][] {
   return xml.children
-    .filter(e => e.localName === "ParameterValueStruct")
-    .map<[string, string | number | boolean, string]>(e => {
+    .filter((e) => e.localName === "ParameterValueStruct")
+    .map<[Path, string | number | boolean, string]>((e) => {
       let valueElement: Element, param: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -151,16 +166,25 @@ function parameterValueList(
         }
       }
 
-      const valueType = getValueType(valueElement.attrs);
+      let valueType = getValueType(valueElement.attrs);
+      if (!valueType) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          attribute: "type",
+          parameter: param,
+        });
+        valueType = "xsd:string";
+      }
 
       const value = decodeEntities(valueElement.text);
       let parsed: string | number | boolean = value;
       if (valueType === "xsd:boolean") {
         parsed = parseBool(value);
-        if (parsed === null) {
+        if (parsed == null) {
           warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
+            message: "Missing or invalid XML node",
+            element: "Value",
+            parameter: param,
           });
           parsed = value;
         }
@@ -168,8 +192,9 @@ function parameterValueList(
         parsed = parseInt(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
+            message: "Missing or invalid XML node",
+            element: "Value",
+            parameter: param,
           });
           parsed = value;
         }
@@ -177,21 +202,22 @@ function parameterValueList(
         parsed = Date.parse(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
+            message: "Missing or invalid XML node",
+            element: "Value",
+            parameter: param,
           });
           parsed = value;
         }
       }
 
-      return [param, parsed, valueType];
+      return [Path.parse(param), parsed, valueType];
     });
 }
 
-function parameterAttributeList(xml: Element): [string, number, string[]][] {
+function parameterAttributeList(xml: Element): [Path, number, string[]][] {
   return xml.children
-    .filter(e => e.localName === "ParameterAttributeStruct")
-    .map<[string, number, string[]]>(e => {
+    .filter((e) => e.localName === "ParameterAttributeStruct")
+    .map<[Path, number, string[]]>((e) => {
       let notificationElement: Element,
         accessListElement: Element,
         param: string;
@@ -209,21 +235,21 @@ function parameterAttributeList(xml: Element): [string, number, string[]][] {
         }
       }
 
-      const notification = decodeEntities(notificationElement.text);
-      const notif = +notification;
-
-      if (notif !== parseInt(notification)) {
+      let notification = parseInt(notificationElement.text);
+      if (isNaN(notification)) {
         warnings.push({
-          message: "Invalid notification attribute",
-          parameter: param
+          message: "Missing or invalid XML node",
+          element: "Notification",
+          parameter: param,
         });
+        notification = 0;
       }
 
       const accessList = accessListElement.children
-        .filter(c => c.localName === "string")
-        .map(c => decodeEntities(c.text));
+        .filter((c) => c.localName === "string")
+        .map((c) => decodeEntities(c.text));
 
-      return [param, notif, accessList];
+      return [Path.parse(param), notification, accessList];
     });
 }
 
@@ -233,12 +259,12 @@ function GetParameterNames(methodRequest): string {
   }</ParameterPath><NextLevel>${+methodRequest.nextLevel}</NextLevel></cwmp:GetParameterNames>`;
 }
 
-function GetParameterNamesResponse(xml): CpeGetResponse {
+function GetParameterNamesResponse(xml): GetParameterNamesResponse {
   return {
     name: "GetParameterNamesResponse",
     parameterList: parameterInfoList(
-      xml.children.find(n => n.localName === "ParameterList")
-    )
+      xml.children.find((n) => n.localName === "ParameterList")
+    ),
   };
 }
 
@@ -246,16 +272,16 @@ function GetParameterValues(methodRequest): string {
   return `<cwmp:GetParameterValues><ParameterNames soap-enc:arrayType="xsd:string[${
     methodRequest.parameterNames.length
   }]">${methodRequest.parameterNames
-    .map(p => `<string>${p}</string>`)
+    .map((p) => `<string>${p}</string>`)
     .join("")}</ParameterNames></cwmp:GetParameterValues>`;
 }
 
-function GetParameterValuesResponse(xml: Element): CpeGetResponse {
+function GetParameterValuesResponse(xml: Element): GetParameterValuesResponse {
   return {
     name: "GetParameterValuesResponse",
     parameterList: parameterValueList(
-      xml.children.find(n => n.localName === "ParameterList")
-    )
+      xml.children.find((n) => n.localName === "ParameterList")
+    ),
   };
 }
 
@@ -263,21 +289,23 @@ function GetParameterAttributes(methodRequest): string {
   return `<cwmp:GetParameterAttributes><ParameterNames soap-enc:arrayType="xsd:string[${
     methodRequest.parameterNames.length
   }]">${methodRequest.parameterNames
-    .map(p => `<string>${p}</string>`)
+    .map((p) => `<string>${p}</string>`)
     .join("")}</ParameterNames></cwmp:GetParameterAttributes>`;
 }
 
-function GetParameterAttributesResponse(xml: Element): CpeGetResponse {
+function GetParameterAttributesResponse(
+  xml: Element
+): GetParameterAttributesResponse {
   return {
     name: "GetParameterAttributesResponse",
     parameterList: parameterAttributeList(
-      xml.children.find(n => n.localName === "ParameterList")
-    )
+      xml.children.find((n) => n.localName === "ParameterList")
+    ),
   };
 }
 
 function SetParameterValues(methodRequest): string {
-  const params = methodRequest.parameterList.map(p => {
+  const params = methodRequest.parameterList.map((p) => {
     let val = p[1];
     if (p[2] === "xsd:dateTime" && typeof val === "number") {
       val = new Date(val).toISOString();
@@ -293,21 +321,38 @@ function SetParameterValues(methodRequest): string {
 
   return `<cwmp:SetParameterValues><ParameterList soap-enc:arrayType="cwmp:ParameterValueStruct[${
     methodRequest.parameterList.length
-  }]">${params.join(
-    ""
-  )}</ParameterList><ParameterKey>${methodRequest.parameterKey ||
-    ""}</ParameterKey></cwmp:SetParameterValues>`;
+  }]">${params.join("")}</ParameterList><ParameterKey>${
+    methodRequest.parameterKey || ""
+  }</ParameterKey></cwmp:SetParameterValues>`;
 }
 
-function SetParameterValuesResponse(xml: Element): CpeSetResponse {
+function SetParameterValuesResponse(xml: Element): SetParameterValuesResponse {
+  let status: number;
+
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "Status":
+        status = parseInt(c.text);
+        break;
+    }
+  }
+
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
   return {
     name: "SetParameterValuesResponse",
-    status: parseInt(xml.children.find(n => n.localName === "Status").text)
+    status: status,
   };
 }
 
 function SetParameterAttributes(methodRequest): string {
-  const params = methodRequest.parameterList.map(p => {
+  const params = methodRequest.parameterList.map((p) => {
     return `<SetParameterAttributesStruct><Name>${
       p[0]
     }</Name><NotificationChange>${
@@ -321,7 +366,7 @@ function SetParameterAttributes(methodRequest): string {
     }]">${
       p[2] == null
         ? ""
-        : p[2].map(s => `<string>${encodeEntities(s)}</string>`).join("")
+        : p[2].map((s) => `<string>${encodeEntities(s)}</string>`).join("")
     }</AccessList></SetParameterAttributesStruct>`;
   });
 
@@ -330,25 +375,26 @@ function SetParameterAttributes(methodRequest): string {
   }]">${params.join("")}</ParameterList></cwmp:SetParameterAttributes>`;
 }
 
-function SetParameterAttributesResponse(): CpeSetResponse {
+function SetParameterAttributesResponse(): SetParameterAttributesResponse {
   return {
-    name: "SetParameterAttributesResponse"
+    name: "SetParameterAttributesResponse",
   };
 }
 
 function AddObject(methodRequest): string {
   return `<cwmp:AddObject><ObjectName>${
     methodRequest.objectName
-  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
-    ""}</ParameterKey></cwmp:AddObject>`;
+  }</ObjectName><ParameterKey>${
+    methodRequest.parameterKey || ""
+  }</ParameterKey></cwmp:AddObject>`;
 }
 
-function AddObjectResponse(xml: Element): CpeSetResponse {
-  let instanceNumber, status;
+function AddObjectResponse(xml: Element): AddObjectResponse {
+  let instanceNumber: string, status: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "InstanceNumber":
-        instanceNumber = parseInt(c.text);
+        instanceNumber = c.text;
         break;
       case "Status":
         status = parseInt(c.text);
@@ -356,35 +402,66 @@ function AddObjectResponse(xml: Element): CpeSetResponse {
     }
   }
 
+  if (!/^[0-9]+$/.test(instanceNumber))
+    throw new Error("Missing or invalid instance number");
+
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
   return {
     name: "AddObjectResponse",
     instanceNumber: instanceNumber,
-    status: status
+    status: status,
   };
 }
 
 function DeleteObject(methodRequest): string {
   return `<cwmp:DeleteObject><ObjectName>${
     methodRequest.objectName
-  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
-    ""}</ParameterKey></cwmp:DeleteObject>`;
+  }</ObjectName><ParameterKey>${
+    methodRequest.parameterKey || ""
+  }</ParameterKey></cwmp:DeleteObject>`;
 }
 
-function DeleteObjectResponse(xml: Element): CpeSetResponse {
+function DeleteObjectResponse(xml: Element): DeleteObjectResponse {
+  let status: number;
+
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "Status":
+        status = parseInt(c.text);
+        break;
+    }
+  }
+
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
   return {
     name: "DeleteObjectResponse",
-    status: parseInt(xml.children.find(n => n.localName === "Status").text)
+    status: status,
   };
 }
 
 function Reboot(methodRequest): string {
-  return `<cwmp:Reboot><CommandKey>${methodRequest.commandKey ||
-    ""}</CommandKey></cwmp:Reboot>`;
+  return `<cwmp:Reboot><CommandKey>${
+    methodRequest.commandKey || ""
+  }</CommandKey></cwmp:Reboot>`;
 }
 
-function RebootResponse(): CpeSetResponse {
+function RebootResponse(): RebootResponse {
   return {
-    name: "RebootResponse"
+    name: "RebootResponse",
   };
 }
 
@@ -392,33 +469,54 @@ function FactoryReset(): string {
   return "<cwmp:FactoryReset></cwmp:FactoryReset>";
 }
 
-function FactoryResetResponse(): CpeSetResponse {
+function FactoryResetResponse(): FactoryResetResponse {
   return {
-    name: "FactoryResetResponse"
+    name: "FactoryResetResponse",
   };
 }
 
 function Download(methodRequest): string {
-  return `<cwmp:Download><CommandKey>${methodRequest.commandKey ||
-    ""}</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
+  return `<cwmp:Download><CommandKey>${
+    methodRequest.commandKey || ""
+  }</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
     methodRequest.url
   }</URL><Username>${encodeEntities(
     methodRequest.username || ""
   )}</Username><Password>${encodeEntities(
     methodRequest.password || ""
-  )}</Password><FileSize>${methodRequest.fileSize ||
-    "0"}</FileSize><TargetFileName>${encodeEntities(
+  )}</Password><FileSize>${
+    methodRequest.fileSize || "0"
+  }</FileSize><TargetFileName>${encodeEntities(
     methodRequest.targetFileName || ""
-  )}</TargetFileName><DelaySeconds>${methodRequest.delaySeconds ||
-    "0"}</DelaySeconds><SuccessURL>${encodeEntities(
+  )}</TargetFileName><DelaySeconds>${
+    methodRequest.delaySeconds || "0"
+  }</DelaySeconds><SuccessURL>${encodeEntities(
     methodRequest.successUrl || ""
   )}</SuccessURL><FailureURL>${encodeEntities(
     methodRequest.failureUrl || ""
   )}</FailureURL></cwmp:Download>`;
 }
 
-function DownloadResponse(xml: Element): CpeSetResponse {
-  let status, startTime, completeTime;
+function Upload(methodRequest): string {
+  return `<cwmp:Upload><CommandKey>${
+    methodRequest.commandKey || ""
+  }</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
+    methodRequest.url
+  }</URL><Username>${encodeEntities(
+    methodRequest.username || ""
+  )}</Username><Password>${encodeEntities(
+    methodRequest.password || ""
+  )}</Password><DelaySeconds>${
+    methodRequest.delaySeconds || "0"
+  }</DelaySeconds><SuccessURL>${encodeEntities(
+    methodRequest.successUrl || ""
+  )}</SuccessURL><FailureURL>${encodeEntities(
+    methodRequest.failureUrl || ""
+  )}</FailureURL></cwmp:Upload>`;
+}
+
+function DownloadResponse(xml: Element): DownloadResponse {
+  let status: number, startTime: number, completeTime: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "Status":
@@ -433,21 +531,75 @@ function DownloadResponse(xml: Element): CpeSetResponse {
     }
   }
 
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "DownloadResponse",
     status: status,
     startTime: startTime,
-    completeTime: completeTime
+    completeTime: completeTime,
+  };
+}
+
+function UploadResponse(xml: Element): UploadResponse {
+  let status: number, startTime: number, completeTime: number;
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "Status":
+        status = parseInt(c.text);
+        break;
+      case "StartTime":
+        startTime = Date.parse(c.text);
+        break;
+      case "CompleteTime":
+        completeTime = Date.parse(c.text);
+        break;
+    }
+  }
+
+  if (!(status >= 0)) {
+    warnings.push({ message: "Invalid UploadResponse status" });
+    status = 0;
+  }
+
+  return {
+    name: "UploadResponse",
+    status: status,
+    startTime: startTime,
+    completeTime: completeTime,
   };
 }
 
 function Inform(xml: Element): InformRequest {
-  let retryCount, evnt, parameterList;
+  let retryCount: number, evnt: string[];
+  let parameterList: [Path, string | number | boolean, string][];
   const deviceId = {
     Manufacturer: null,
     OUI: null,
     ProductClass: null,
-    SerialNumber: null
+    SerialNumber: null,
   };
 
   for (const c of xml.children) {
@@ -470,12 +622,36 @@ function Inform(xml: Element): InformRequest {
     }
   }
 
+  if (!deviceId || !deviceId.SerialNumber || !deviceId.OUI)
+    throw new Error("Missing or invalid DeviceId element");
+
+  if (!parameterList) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "ParameterList",
+    });
+    parameterList = [];
+  }
+
+  if (!evnt) {
+    warnings.push({ message: "Missing or invalid XML node", element: "Event" });
+    evnt = [];
+  }
+
+  if (retryCount == null || isNaN(retryCount)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "RetryCount",
+    });
+    retryCount = 0;
+  }
+
   return {
     name: "Inform",
     parameterList: parameterList,
     deviceId: deviceId,
     event: evnt,
-    retryCount: retryCount
+    retryCount: retryCount,
   };
 }
 
@@ -483,7 +659,7 @@ function InformResponse(): string {
   return "<cwmp:InformResponse><MaxEnvelopes>1</MaxEnvelopes></cwmp:InformResponse>";
 }
 
-function GetRPCMethods(): AcsResponse {
+function GetRPCMethods(): GetRPCMethodsRequest {
   return { name: "GetRPCMethods" };
 }
 
@@ -491,12 +667,15 @@ function GetRPCMethodsResponse(methodResponse): string {
   return `<cwmp:GetRPCMethodsResponse><MethodList soap-enc:arrayType="xsd:string[${
     methodResponse.methodList.length
   }]">${methodResponse.methodList
-    .map(m => `<string>${m}</string>`)
+    .map((m) => `<string>${m}</string>`)
     .join("")}</MethodList></cwmp:GetRPCMethodsResponse>`;
 }
 
 function TransferComplete(xml: Element): TransferCompleteRequest {
-  let commandKey, _faultStruct, startTime, completeTime;
+  let commandKey: string,
+    _faultStruct: FaultStruct,
+    startTime: number,
+    completeTime: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "CommandKey":
@@ -514,12 +693,44 @@ function TransferComplete(xml: Element): TransferCompleteRequest {
     }
   }
 
+  if (commandKey == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CommandKey",
+    });
+    commandKey = "";
+  }
+
+  if (!_faultStruct) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultStruct",
+    });
+    _faultStruct = { faultCode: "0", faultString: "" };
+  }
+
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "TransferComplete",
     commandKey: commandKey,
     faultStruct: _faultStruct,
     startTime: startTime,
-    completeTime: completeTime
+    completeTime: completeTime,
   };
 }
 
@@ -527,10 +738,10 @@ function TransferCompleteResponse(): string {
   return "<cwmp:TransferCompleteResponse></cwmp:TransferCompleteResponse>";
 }
 
-function RequestDownload(xml: Element): CpeRequest {
+function RequestDownload(xml: Element): RequestDownloadRequest {
   return {
     name: "RequestDownload",
-    fileType: xml.children.find(n => n.localName === "FileType").text
+    fileType: xml.children.find((n) => n.localName === "FileType").text,
   };
 }
 
@@ -539,7 +750,12 @@ function RequestDownloadResponse(): string {
 }
 
 function faultStruct(xml: Element): FaultStruct {
-  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
+  let faultCode: string,
+    faultString: string,
+    setParameterValuesFault: SpvFault[],
+    pn: string,
+    fc: string,
+    fs: string;
   for (const c of xml.children) {
     switch (c.localName) {
       case "FaultCode":
@@ -550,6 +766,7 @@ function faultStruct(xml: Element): FaultStruct {
         break;
       case "SetParameterValuesFault":
         setParameterValuesFault = setParameterValuesFault || [];
+        pn = fc = fs = null;
         for (const cc of c.children) {
           switch (cc.localName) {
             case "ParameterName":
@@ -566,16 +783,32 @@ function faultStruct(xml: Element): FaultStruct {
         setParameterValuesFault.push({
           parameterName: pn,
           faultCode: fc,
-          faultString: fs
+          faultString: fs,
         });
     }
   }
 
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultCode",
+    });
+    faultCode = "";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultString",
+    });
+    faultString = "";
+  }
+
   return { faultCode, faultString, setParameterValuesFault };
 }
 
 function fault(xml: Element): CpeFault {
-  let faultCode, faultString, detail;
+  let faultCode: string, faultString: string, detail: FaultStruct;
   for (const c of xml.children) {
     switch (c.localName) {
       case "faultcode":
@@ -585,15 +818,37 @@ function fault(xml: Element): CpeFault {
         faultString = decodeEntities(c.text);
         break;
       case "detail":
-        detail = faultStruct(c.children.find(n => n.localName === "Fault"));
+        detail = faultStruct(c.children.find((n) => n.localName === "Fault"));
         break;
     }
   }
 
-  return { faultCode, faultString, detail };
+  if (!detail) throw new Error("Missing detail element");
+
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultcode",
+    });
+    faultCode = "Client";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultstring",
+    });
+    faultString = "CWMP fault";
+  }
+
+  return { faultCode, faultString, detail } as CpeFault;
 }
 
-export function request(body: string, cwmpVersion, warn): SoapMessage {
+export function request(
+  body: string,
+  cwmpVersion: string,
+  warn: Record<string, unknown>[]
+): SoapMessage {
   warnings = warn;
 
   const rpc = {
@@ -602,7 +857,7 @@ export function request(body: string, cwmpVersion, warn): SoapMessage {
     sessionTimeout: null,
     cpeRequest: null,
     cpeFault: null,
-    cpeResponse: null
+    cpeResponse: null,
   };
 
   if (!body.length) return rpc;
@@ -649,9 +904,9 @@ export function request(body: string, cwmpVersion, warn): SoapMessage {
         const attrs = memoizedParseAttrs(e.attrs);
         const attr = namespace
           ? attrs.find(
-              s => s.namespace === "xmlns" && s.localName === namespace
+              (s) => s.namespace === "xmlns" && s.localName === namespace
             )
-          : attrs.find(s => s.name === "xmlns");
+          : attrs.find((s) => s.name === "xmlns");
 
         if (attr) namespaceHref = attr.value;
       }
@@ -720,6 +975,9 @@ export function request(body: string, cwmpVersion, warn): SoapMessage {
     case "DownloadResponse":
       rpc.cpeResponse = DownloadResponse(methodElement);
       break;
+    case "UploadResponse":
+      rpc.cpeResponse = UploadResponse(methodElement);
+      break;
     case "Fault":
       rpc.cpeFault = fault(methodElement);
       break;
@@ -745,13 +1003,18 @@ const namespacesAttrs = {
     .join(" "),
   "1.4": Object.entries(NAMESPACES["1.4"])
     .map(([k, v]) => `xmlns:${k}="${v}"`)
-    .join(" ")
+    .join(" "),
 };
 
-export function response(rpc): { code: number; headers: {}; data: string } {
+export function response(rpc: {
+  id: string;
+  acsRequest?: AcsRequest;
+  acsResponse?: AcsResponse;
+  cwmpVersion?: string;
+}): { code: number; headers: Record<string, string>; data: string } {
   const headers = {
     Server: SERVER_NAME,
-    SOAPServer: SERVER_NAME
+    SOAPServer: SERVER_NAME,
   };
 
   if (!rpc) return { code: 204, headers: headers, data: "" };
@@ -772,7 +1035,11 @@ export function response(rpc): { code: number; headers: {}; data: string } {
         body = RequestDownloadResponse();
         break;
       default:
-        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
+        throw new Error(
+          `Unknown method response type ${
+            (rpc.acsResponse as AcsResponse).name
+          }`
+        );
     }
   } else if (rpc.acsRequest) {
     switch (rpc.acsRequest.name) {
@@ -806,8 +1073,13 @@ export function response(rpc): { code: number; headers: {}; data: string } {
       case "Download":
         body = Download(rpc.acsRequest);
         break;
+      case "Upload":
+        body = Upload(rpc.acsRequest);
+        break;
       default:
-        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
+        throw new Error(
+          `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`
+        );
     }
   }
 
@@ -819,6 +1091,6 @@ export function response(rpc): { code: number; headers: {}; data: string } {
       namespacesAttrs[rpc.cwmpVersion]
     }><soap-env:Header><cwmp:ID soap-env:mustUnderstand="1">${
       rpc.id
-    }</cwmp:ID></soap-env:Header><soap-env:Body>${body}</soap-env:Body></soap-env:Envelope>`
+    }</cwmp:ID></soap-env:Header><soap-env:Body>${body}</soap-env:Body></soap-env:Envelope>`,
   };
 }

lib/types.ts

@@ -22,6 +22,7 @@ import PathSet from "./common/path-set";
 import VersionedMap from "./versioned-map";
 import InstanceSet from "./instance-set";
 import { IncomingMessage, ServerResponse } from "http";
+import { Script } from "vm";
 
 export type Expression = string | number | boolean | null | any[];
 
@@ -120,6 +121,10 @@ export interface SyncState {
   downloadsToCreate: InstanceSet;
   downloadsValues: Map<Path, string | number>;
   downloadsDownload: Map<Path, number>;
+  uploadsToDelete: Set<Path>;
+  uploadsToCreate: InstanceSet;
+  uploadsValues: Map<Path, string | number>;
+  uploadsUpload: Map<Path, number>;
   reboot: number;
   factoryReset: number;
 }
@@ -149,7 +154,6 @@ export interface SessionContext {
   faults?: { [channel: string]: SessionFault };
   retries?: { [channel: string]: number };
   cacheSnapshot?: string;
-  configContext?: { [name: string]: any };
   httpResponse?: ServerResponse;
   httpRequest?: IncomingMessage;
   faultsTouched?: { [channel: string]: boolean };
@@ -157,11 +161,13 @@ export interface SessionContext {
   new?: boolean;
   debug?: boolean;
   state: number;
+  authState: number;
   tasks?: Task[];
   operations?: { [commandKey: string]: Operation };
   cacheUntil?: number;
   syncState?: SyncState;
   lastActivity?: number;
+  extendLock?: number;
   rpcRequest?: AcsRequest;
   operationsTouched?: { [commandKey: string]: 1 | 0 };
   provisionsRet?: any[];
@@ -169,7 +175,7 @@ export interface SessionContext {
 }
 
 export interface Task {
-  _id: string;
+  _id?: string;
   name: string;
   parameterNames?: string[];
   parameterValues?: [string, string | number | boolean, string?][];
@@ -191,51 +197,103 @@ export interface Operation {
     instance: string;
     fileType: string;
     fileName: string;
-    targetFileName: string;
+    targetFileName?: string;
   };
 }
 
-export interface AcsRequest {
-  name: string;
-  next?: string;
+export type AcsRequest =
+  | GetParameterNames
+  | GetParameterValues
+  | GetParameterAttributes
+  | SetParameterValues
+  | SetParameterAttributes
+  | AddObject
+  | DeleteObject
+  | FactoryReset
+  | Reboot
+  | Download
+  | Upload;
+
+export interface GetParameterNames {
+  name: "GetParameterNames";
+  parameterPath: string;
+  nextLevel: boolean;
 }
 
-export interface GetAcsRequest extends AcsRequest {
-  name: "GetParameterNames" | "GetParameterValues" | "GetParameterAttributes";
-  objectName?: string;
-  parameterNames?: string[];
-  parameterPath?: string;
-  nextLevel?: boolean;
-  instanceValues?: { [name: string]: string };
+export interface GetParameterValues {
+  name: "GetParameterValues";
+  parameterNames: string[];
 }
 
-export interface SetAcsRequest extends AcsRequest {
-  name:
-    | "SetParameterValues"
-    | "SetParameterAttributes"
-    | "AddObject"
-    | "DeleteObject"
-    | "FactoryReset"
-    | "Reboot"
-    | "Download";
-  parameterList?:
-    | [string, string | number | boolean, string][]
-    | [string, number, string[]][];
-  instanceValues?: { [param: string]: string | number | boolean };
-  objectName?: string;
+export interface GetParameterAttributes {
+  name: "GetParameterAttributes";
+  parameterNames: string[];
+}
+
+export interface SetParameterValues {
+  name: "SetParameterValues";
+  parameterList: [string, boolean | number | string, string][];
+  parameterKey?: string;
   DATETIME_MILLISECONDS?: boolean;
   BOOLEAN_LITERAL?: boolean;
-  commandKey?: string;
-  instance?: string;
-  fileType?: string;
-  fileSize?: number;
-  url?: string;
-  fileName?: string;
-  targetFileName?: string;
 }
 
-export interface CpeResponse {
-  name: string;
+export interface SetParameterAttributes {
+  name: "SetParameterAttributes";
+  parameterList: [string, number, string[]][];
+}
+
+export interface AddObject {
+  name: "AddObject";
+  objectName: string;
+  parameterKey?: string;
+  instanceValues: Record<string, string>;
+}
+
+export interface DeleteObject {
+  name: "DeleteObject";
+  objectName: string;
+  parameterKey?: string;
+}
+
+export interface FactoryReset {
+  name: "FactoryReset";
+  commandKey?: string;
+}
+
+export interface Reboot {
+  name: "Reboot";
+  commandKey?: string;
+}
+
+export interface Download {
+  name: "Download";
+  commandKey: string;
+  instance: string;
+  fileType: string;
+  fileName?: string;
+  url?: string;
+  username?: string;
+  password?: string;
+  fileSize?: number;
+  targetFileName?: string;
+  delaySecods?: number;
+  successUrl?: string;
+  failureUrl?: string;
+}
+
+export interface Upload {
+  name: "Upload";
+  commandKey: string;
+  instance: string;
+  fileType: string;
+  fileName?: string;
+  url?: string;
+  username?: string;
+  password?: string;
+  delaySecods?: number;
+  successUrl?: string;
+  failureUrl?: string;
 }
 
 export interface SpvFault {
@@ -251,43 +309,88 @@ export interface FaultStruct {
 }
 
 export interface CpeFault {
-  faultCode: string;
-  faultString: string;
+  faultCode: "Client" | "Server";
+  faultString: "CWMP fault";
   detail?: FaultStruct;
 }
 
-export interface CpeGetResponse extends CpeResponse {
-  name:
-    | "GetParameterNamesResponse"
-    | "GetParameterValuesResponse"
-    | "GetParameterAttributesResponse";
-  parameterList?:
-    | [string, boolean][]
-    | [string, string | number | boolean, string][]
-    | [string, number, string[]][];
+export type CpeResponse =
+  | GetParameterNamesResponse
+  | GetParameterValuesResponse
+  | GetParameterAttributesResponse
+  | SetParameterValuesResponse
+  | SetParameterAttributesResponse
+  | AddObjectResponse
+  | DeleteObjectResponse
+  | RebootResponse
+  | FactoryResetResponse
+  | DownloadResponse
+  | UploadResponse;
+
+export interface GetParameterNamesResponse {
+  name: "GetParameterNamesResponse";
+  parameterList: [Path, boolean, boolean][];
 }
 
-export interface CpeSetResponse extends CpeResponse {
-  name:
-    | "SetParameterValuesResponse"
-    | "SetParameterAttributesResponse"
-    | "AddObjectResponse"
-    | "DeleteObjectResponse"
-    | "RebootResponse"
-    | "FactoryResetResponse"
-    | "DownloadResponse";
-  status?: number;
-  instanceNumber?: string;
+export interface GetParameterValuesResponse {
+  name: "GetParameterValuesResponse";
+  parameterList: [Path, string | number | boolean, string][];
+}
+
+export interface GetParameterAttributesResponse {
+  name: "GetParameterAttributesResponse";
+  parameterList: [Path, number, string[]][];
+}
+
+export interface SetParameterValuesResponse {
+  name: "SetParameterValuesResponse";
+  status: number;
+}
+
+export interface SetParameterAttributesResponse {
+  name: "SetParameterAttributesResponse";
+}
+
+export interface AddObjectResponse {
+  name: "AddObjectResponse";
+  instanceNumber: string;
+  status: number;
+}
+
+export interface DeleteObjectResponse {
+  name: "DeleteObjectResponse";
+  status: number;
+}
+
+export interface RebootResponse {
+  name: "RebootResponse";
+}
+
+export interface FactoryResetResponse {
+  name: "FactoryResetResponse";
+}
+
+export interface DownloadResponse {
+  name: "DownloadResponse";
+  status: number;
   startTime?: number;
   completeTime?: number;
 }
 
-export interface CpeRequest {
-  name: string;
-  fileType?: string;
+export interface UploadResponse {
+  name: "UploadResponse";
+  status: number;
+  startTime?: number;
+  completeTime?: number;
 }
 
-export interface InformRequest extends CpeRequest {
+export type CpeRequest =
+  | InformRequest
+  | TransferCompleteRequest
+  | GetRPCMethodsRequest
+  | RequestDownloadRequest;
+
+export interface InformRequest {
   name: "Inform";
   deviceId: {
     Manufacturer: string;
@@ -297,10 +400,10 @@ export interface InformRequest extends CpeRequest {
   };
   event: string[];
   retryCount: number;
-  parameterList: [string, string | number | boolean, string][];
+  parameterList: [Path, string | number | boolean, string][];
 }
 
-export interface TransferCompleteRequest extends CpeRequest {
+export interface TransferCompleteRequest {
   name: "TransferComplete";
   commandKey?: string;
   faultStruct?: FaultStruct;
@@ -308,10 +411,36 @@ export interface TransferCompleteRequest extends CpeRequest {
   completeTime?: number;
 }
 
-export interface AcsResponse {
-  name: string;
-  commandKey?: string;
-  faultStruct?: FaultStruct;
+export interface GetRPCMethodsRequest {
+  name: "GetRPCMethods";
+}
+
+export interface RequestDownloadRequest {
+  name: "RequestDownload";
+  fileType: string;
+}
+
+export type AcsResponse =
+  | InformResponse
+  | GetRPCMethodsResponse
+  | TransferCompleteResponse
+  | RequestDownloadResponse;
+
+export interface InformResponse {
+  name: "InformResponse";
+}
+
+export interface GetRPCMethodsResponse {
+  name: "GetRPCMethodsResponse";
+  methodList: string[];
+}
+
+export interface TransferCompleteResponse {
+  name: "TransferCompleteResponse";
+}
+
+export interface RequestDownloadResponse {
+  name: "RequestDownloadResponse";
 }
 
 export interface QueryOptions {
@@ -326,7 +455,7 @@ export interface QueryOptions {
 export interface Declaration {
   path: Path;
   pathGet: number;
-  pathSet?: [number, number];
+  pathSet?: number | [number, number];
   attrGet?: {
     object?: number;
     writable?: number;
@@ -362,8 +491,36 @@ export interface Preset {
   channel: string;
   schedule?: { md5: string; duration: number; schedule: any };
   events?: { [event: string]: boolean };
-  precondition?: {};
-  provisions: string[][];
+  precondition?: Expression;
+  provisions: [string, ...Expression[]][];
+}
+
+export interface Provisions {
+  [name: string]: { md5: string; script: Script };
+}
+
+export interface VirtualParameters {
+  [name: string]: { md5: string; script: Script };
+}
+
+export interface Files {
+  [name: string]: { length: number; md5: string; contentType: string };
+}
+
+export interface Users {
+  [name: string]: { password: string; salt: string; roles: string[] };
+}
+
+export interface Permissions {
+  [role: string]: {
+    [access: number]: {
+      [resource: string]: {
+        access: number;
+        filter: Expression;
+        validate?: Expression;
+      };
+    };
+  };
 }
 
 export type PermissionSet = {
@@ -374,6 +531,21 @@ export type PermissionSet = {
   };
 }[];
 
+export interface Config {
+  [name: string]: Expression;
+}
+
+export interface UiConfig {
+  filters: Record<string, unknown>;
+  device: Record<string, unknown>;
+  index: Record<string, unknown>;
+  overview: {
+    charts?: Record<string, unknown>;
+    groups?: Record<string, unknown>;
+  };
+  pageSize?: Expression;
+}
+
 export interface SoapMessage {
   id: string;
   cwmpVersion: string;

lib/ui.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Z_SYNC_FLUSH } from "zlib";
+import { constants } from "zlib";
 import Koa from "koa";
 import Router from "koa-router";
 import * as jwt from "jsonwebtoken";
@@ -34,6 +34,7 @@ import { PermissionSet } from "./types";
 import { authLocal } from "./ui/api-functions";
 import * as init from "./init";
 import { version as VERSION } from "../package.json";
+import memoize from "./common/memoize";
 
 declare module "koa" {
   interface Request {
@@ -47,8 +48,21 @@ const router = new Router();
 const JWT_SECRET = "" + config.get("UI_JWT_SECRET");
 const JWT_COOKIE = "genieacs-ui-jwt";
 
-koa.on("error", async err => {
-  throw err;
+const getAuthorizer = memoize(
+  (snapshot: string, rolesStr: string): Authorizer => {
+    const roles: string[] = JSON.parse(rolesStr);
+    const allPermissions = localCache.getPermissions(snapshot);
+    const permissionSets: PermissionSet[] = roles.map((r) =>
+      Object.values(allPermissions[r] || {})
+    );
+    return new Authorizer(permissionSets);
+  }
+);
+
+koa.on("error", async (err) => {
+  setTimeout(() => {
+    throw err;
+  });
 });
 
 koa.use(async (ctx, next) => {
@@ -72,16 +86,14 @@ koa.use(
       }
 
       return true;
-    }
+    },
   })
 );
 
 koa.use(async (ctx, next) => {
-  const permissionSets: PermissionSet[] = [];
+  let roles: string[] = [];
 
   if (ctx.state.user && ctx.state.user.username) {
-    const allPermissions = localCache.getPermissions(ctx.state.configSnapshot);
-
     let user;
     if (ctx.state.user.authMethod === "local") {
       user = localCache.getUsers(ctx.state.configSnapshot)[
@@ -90,17 +102,18 @@ koa.use(async (ctx, next) => {
     } else {
       throw new Error("Invalid auth method");
     }
-
-    for (const role of (user.roles || []).sort())
-      permissionSets.push(Object.values(allPermissions[role] || {}));
+    roles = user.roles || [];
   }
 
-  ctx.state.authorizer = new Authorizer(permissionSets);
+  ctx.state.authorizer = getAuthorizer(
+    ctx.state.configSnapshot,
+    JSON.stringify(roles)
+  );
 
   return next();
 });
 
-router.post("/login", async ctx => {
+router.post("/login", async (ctx) => {
   if (!JWT_SECRET) {
     ctx.status = 500;
     ctx.body = "UI_JWT_SECRET is not set";
@@ -115,7 +128,7 @@ router.post("/login", async ctx => {
     message: "Log in",
     context: ctx,
     username: username,
-    method: null
+    method: null,
   };
 
   function success(authMethod): void {
@@ -139,13 +152,13 @@ router.post("/login", async ctx => {
   failure();
 });
 
-router.post("/logout", async ctx => {
+router.post("/logout", async (ctx) => {
   ctx.cookies.set(JWT_COOKIE); // Delete cookie
   ctx.body = "";
 
   logger.accessInfo({
     message: "Log out",
-    context: ctx
+    context: ctx,
   });
 });
 
@@ -159,11 +172,11 @@ koa.use(async (ctx, next) => {
 koa.use(koaBodyParser());
 router.use("/api", api.routes(), api.allowedMethods());
 
-router.get("/status", ctx => {
+router.get("/status", (ctx) => {
   ctx.body = "OK";
 });
 
-router.get("/init", async ctx => {
+router.get("/init", async (ctx) => {
   const status = await init.getStatus();
   if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
     if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
@@ -184,7 +197,7 @@ router.get("/init", async ctx => {
   ctx.body = status;
 });
 
-router.post("/init", async ctx => {
+router.post("/init", async (ctx) => {
   const status = ctx.request.body;
   if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
     if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
@@ -205,8 +218,8 @@ router.post("/init", async ctx => {
   ctx.body = "";
 });
 
-router.get("/", async ctx => {
-  const permissionSets = ctx.state.authorizer.getPermissionSets();
+router.get("/", async (ctx) => {
+  const permissionSets: PermissionSet[] = ctx.state.authorizer.getPermissionSets();
 
   let wizard = "";
   if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
@@ -223,7 +236,7 @@ router.get("/", async ctx => {
     <noscript>GenieACS UI requires JavaScript to work. Please enable JavaScript in your browser.</noscript>
       <script>
         window.clientConfig = ${JSON.stringify({
-          ui: localCache.getUiConfig(ctx.state.configSnapshot)
+          ui: localCache.getUiConfig(ctx.state.configSnapshot),
         })};
         window.configSnapshot = ${JSON.stringify(ctx.state.configSnapshot)};
         window.genieacsVersion = ${JSON.stringify(VERSION)};
@@ -240,7 +253,12 @@ router.get("/", async ctx => {
 
 koa.use(
   koaCompress({
-    flush: Z_SYNC_FLUSH
+    gzip: {
+      flush: constants.Z_SYNC_FLUSH,
+    },
+    deflate: {
+      flush: constants.Z_SYNC_FLUSH,
+    },
   })
 );
 

lib/ui/api-functions.ts

@@ -24,10 +24,12 @@ import { getUsers } from "../local-cache";
 import { hashPassword } from "../auth";
 import {
   insertTasks,
-  watchTask,
+  awaitSessionEnd,
+  awaitSessionStart,
   connectionRequest,
-  deleteDevice
+  deleteDevice,
 } from "../api-functions";
+import { Task } from "../types";
 
 async function deleteFault(id): Promise<void> {
   const deviceId = id.split(":", 1)[0];
@@ -37,13 +39,16 @@ async function deleteFault(id): Promise<void> {
     db.deleteFault(id),
     channel.startsWith("task_")
       ? db.deleteTask(new ObjectID(channel.slice(5)))
-      : null
+      : null,
   ]);
 
   await del(`${deviceId}_tasks_faults_operations`);
 }
 
-export async function deleteResource(resource, id): Promise<void> {
+export async function deleteResource(
+  resource: string,
+  id: string
+): Promise<void> {
   switch (resource) {
     case "devices":
       await deleteDevice(id);
@@ -53,6 +58,10 @@ export async function deleteResource(resource, id): Promise<void> {
       await db.deleteFile(id);
       break;
 
+    case "uploads":
+      await db.deleteUpload(id);
+      break;
+
     case "faults":
       await deleteFault(id);
       break;
@@ -86,35 +95,59 @@ export async function deleteResource(resource, id): Promise<void> {
 }
 
 export async function postTasks(
-  deviceId,
-  tasks,
-  timeout
+  deviceId: string,
+  tasks: Task[],
+  timeout: number,
+  device: Record<string, { value?: [boolean | number | string, string] }>
 ): Promise<{ connectionRequest: string; tasks: any[] }> {
   for (const task of tasks) {
     delete task._id;
-    task.device = deviceId;
+    task["device"] = deviceId;
   }
 
   tasks = await insertTasks(tasks);
-  const statuses = tasks.map(t => {
+  const statuses = tasks.map((t) => {
     return { _id: t._id, status: "pending" };
   });
-
-  await del(`${deviceId}_tasks_faults_operations`);
-
-  try {
-    await connectionRequest(deviceId);
-  } catch (err) {
+  const lastInform = Date.now();
+  const notInSession = await awaitSessionEnd(deviceId, 30000);
+  if (!notInSession) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
     return {
-      connectionRequest: err.message,
-      tasks: statuses
+      connectionRequest: "Session took too long to complete",
+      tasks: statuses,
     };
   }
 
-  const sample = tasks[tasks.length - 1];
+  await del(`${deviceId}_tasks_faults_operations`);
 
-  // Waiting for session to finish or timeout
-  await watchTask(deviceId, sample._id, timeout);
+  const status = await connectionRequest(deviceId, device);
+
+  if (status) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: status,
+      tasks: statuses,
+    };
+  }
+
+  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
+  if (!sessionStarted) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: "No contact from CPE",
+      tasks: statuses,
+    };
+  }
+
+  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
+  if (!sessionEnded) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: "Session took too long to complete",
+      tasks: statuses,
+    };
+  }
 
   const promises = [];
   for (const s of statuses) {
@@ -130,25 +163,20 @@ export async function postTasks(
       r.status = "done";
     } else if (res[i * 2 + 1].length === 1) {
       r.status = "fault";
-      r.fault = res[i * 2 + 1][0];
+      r["fault"] = res[i * 2 + 1][0];
     }
-    db.deleteTask(r._id);
   }
 
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+
   return { connectionRequest: "OK", tasks: statuses };
 }
 
-interface PingResponse {
-  packetsTransmitted: number;
-  packetsReceived: number;
-  packetLoss: string;
-  min: number;
-  avg: number;
-  max: number;
-  mdev: number;
-}
-
-export async function putResource(resource, id, data): Promise<void> {
+export async function putResource(
+  resource: string,
+  id: string,
+  data: Record<string, unknown>
+): Promise<void> {
   if (resource === "presets") {
     await db.putPreset(id, data);
   } else if (resource === "provisions") {
@@ -160,21 +188,25 @@ export async function putResource(resource, id, data): Promise<void> {
   } else if (resource === "permissions") {
     await db.putPermission(id, data);
   } else if (resource === "users") {
-    delete data.password;
-    delete data.salt;
+    delete data["password"];
+    delete data["salt"];
     await db.putUser(id, data);
   }
 
   await del("presets_hash");
 }
 
-export function authLocal(snapshot, username, password): Promise<boolean> {
+export function authLocal(
+  snapshot: string,
+  username: string,
+  password: string
+): Promise<boolean> {
   return new Promise((resolve, reject) => {
     const users = getUsers(snapshot);
     const user = users[username];
     if (!user || !user.password) return void resolve(null);
     hashPassword(password, user.salt)
-      .then(hash => {
+      .then((hash) => {
         if (hash === user.password) resolve(true);
         else resolve(false);
       })

lib/ui/api.ts

@@ -30,6 +30,8 @@ import { generateSalt, hashPassword } from "../auth";
 import { del } from "../cache";
 import Authorizer from "../common/authorizer";
 import { ping } from "../ping";
+import * as url from "url";
+import { decodeTag } from "../common";
 
 const router = new Router();
 export default router;
@@ -47,12 +49,13 @@ const RESOURCE_IDS = {
   presets: "_id",
   provisions: "_id",
   files: "_id",
+  uploads: "_id",
   virtualParameters: "_id",
   config: "_id",
   permissions: "_id",
   users: "_id",
   faults: "_id",
-  tasks: "_id"
+  tasks: "_id",
 };
 
 const resources = {
@@ -60,31 +63,32 @@ const resources = {
   presets: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   provisions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   files: 0 | RESOURCE_DELETE,
+  uploads: 0 | RESOURCE_DELETE,
   virtualParameters: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   config: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   permissions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   users: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   faults: 0 | RESOURCE_DELETE,
-  tasks: 0
+  tasks: 0,
 };
 
-router.get(`/devices/:id.csv`, async ctx => {
+router.get(`/devices/:id.csv`, async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Query device (CSV)",
     context: ctx,
-    id: ctx.params.id
+    id: ctx.params.id,
   };
 
   const filter = and(authorizer.getFilter("devices", 2), [
     "=",
     ["PARAM", RESOURCE_IDS.devices],
-    ctx.params.id
+    ctx.params.id,
   ]);
 
   if (!authorizer.hasAccess("devices", 2)) {
     logUnauthorizedWarning(log);
-    return void (ctx.status = 404);
+    return void (ctx.status = 403);
   }
 
   const res = await db.query("devices", filter);
@@ -116,16 +120,16 @@ router.get(`/devices/:id.csv`, async ctx => {
       p.notification,
       p.notificationTimestamp,
       p.accessList ? p.accessList.join(", ") : "",
-      p.accessListTimestamp
+      p.accessListTimestamp,
     ];
-    ctx.body.write(row.map(r => (r != null ? r : "")).join(",") + "\n");
+    ctx.body.write(row.map((r) => (r != null ? r : "")).join(",") + "\n");
   }
   ctx.body.end();
   logger.accessInfo(log);
 });
 
 for (const [resource, flags] of Object.entries(resources)) {
-  router.head(`/${resource}`, async (ctx, next) => {
+  router.head(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     let filter: Expression = authorizer.getFilter(resource, 1);
     if (ctx.request.query.filter)
@@ -135,19 +139,20 @@ for (const [resource, flags] of Object.entries(resources)) {
       message: `Count ${resource}`,
       context: ctx,
       filter: ctx.request.query.filter,
-      count: null
+      count: null,
     };
 
     if (!authorizer.hasAccess(resource, 1)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -160,7 +165,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.get(`/${resource}`, async (ctx, next) => {
+  router.get(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = {};
     let filter: Expression = authorizer.getFilter(resource, 2);
@@ -183,19 +188,20 @@ for (const [resource, flags] of Object.entries(resources)) {
       limit: options.limit,
       skip: options.skip,
       sort: options.sort,
-      projection: options.projection
+      projection: options.projection,
     };
 
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -204,7 +210,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     let c = 0;
     ctx.body.write("[\n");
-    await db.query(resource, filter, options, obj => {
+    await db.query(resource, filter, options, (obj) => {
       ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
     });
     ctx.body.end("]");
@@ -213,7 +219,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   });
 
   // CSV download
-  router.get(`/${resource}.csv`, async (ctx, next) => {
+  router.get(`/${resource}.csv`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = { projection: {} };
     let filter: Expression = authorizer.getFilter(resource, 2);
@@ -230,15 +236,17 @@ for (const [resource, flags] of Object.entries(resources)) {
       filter: ctx.request.query.filter,
       limit: options.limit,
       skip: options.skip,
-      sort: options.sort
+      sort: options.sort,
     };
 
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
-    const columns = JSON.parse(ctx.request.query.columns);
+    const columns: Record<string, Expression> = JSON.parse(
+      ctx.request.query.columns
+    );
     const now = Date.now();
 
     for (const [k, v] of Object.entries(columns)) {
@@ -251,8 +259,9 @@ for (const [resource, flags] of Object.entries(resources)) {
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+        "OR",
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -263,18 +272,18 @@ for (const [resource, flags] of Object.entries(resources)) {
     );
 
     ctx.body.write(
-      Object.keys(columns).map(k => `"${k.replace(/"/, '""')}"`) + "\n"
+      Object.keys(columns).map((k) => `"${k.replace(/"/, '""')}"`) + "\n"
     );
-    await db.query(resource, filter, options, obj => {
-      const arr = Object.values(columns).map(exp => {
-        const v = evaluate(exp, obj, null, e => {
+    await db.query(resource, filter, options, (obj) => {
+      const arr = Object.values(columns).map((exp) => {
+        const v = evaluate(exp, obj, null, (e) => {
           if (Array.isArray(e)) {
             if (e[0] === "PARAM") {
               if (resource === "devices") {
                 if (e[1] === "Tags") {
                   const tags = [];
                   for (const p in obj)
-                    if (p.startsWith("Tags.")) tags.push(p.slice(5));
+                    if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
 
                   return tags.join(", ");
                 }
@@ -301,78 +310,78 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.head(`/${resource}/:id`, async (ctx, next) => {
+  router.head(`/${resource}/:id`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Count ${resource}`,
       context: ctx,
-      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`,
     };
 
     const filter = and(authorizer.getFilter(resource, 2), [
       "=",
       ["PARAM", RESOURCE_IDS[resource]],
-      ctx.params.id
+      ctx.params.id,
     ]);
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void next();
+    if (!res.length) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = "";
   });
 
-  router.get(`/${resource}/:id`, async (ctx, next) => {
+  router.get(`/${resource}/:id`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Query ${resource}`,
       context: ctx,
-      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`,
     };
 
     const filter = and(authorizer.getFilter(resource, 2), [
       "=",
       ["PARAM", RESOURCE_IDS[resource]],
-      ctx.params.id
+      ctx.params.id,
     ]);
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void next();
+    if (!res.length) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = res[0];
   });
 
   if (flags & RESOURCE_DELETE) {
-    router.delete(`/${resource}/:id`, async (ctx, next) => {
+    router.delete(`/${resource}/:id(.+)`, async (ctx) => {
       const authorizer: Authorizer = ctx.state.authorizer;
       const log = {
         message: `Delete ${resource}`,
         context: ctx,
-        id: ctx.params.id
+        id: ctx.params.id,
       };
 
       const filter = and(authorizer.getFilter(resource, 3), [
         "=",
         ["PARAM", RESOURCE_IDS[resource]],
-        ctx.params.id
+        ctx.params.id,
       ]);
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
       const res = await db.query(resource, filter);
-      if (!res.length) return void next();
+      if (!res.length) return void (ctx.status = 404);
 
       const validate = authorizer.getValidator(resource, res[0]);
       if (!validate("delete")) {
@@ -389,19 +398,19 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 
   if (flags & RESOURCE_PUT) {
-    router.put(`/${resource}/:id`, async (ctx, next) => {
+    router.put(`/${resource}/:id`, async (ctx) => {
       const authorizer: Authorizer = ctx.state.authorizer;
       const id = ctx.params.id;
 
       const log = {
         message: `Put ${resource}`,
         context: ctx,
-        id: id
+        id: id,
       };
 
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
 
       const obj = ctx.request.body;
@@ -428,7 +437,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 }
 
-router.put("/files/:id", async (ctx, next) => {
+router.put("/files/:id", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const resource = "files";
   const id = ctx.params.id;
@@ -437,19 +446,19 @@ router.put("/files/:id", async (ctx, next) => {
     message: `Upload ${resource}`,
     context: ctx,
     id: id,
-    metadata: null
+    metadata: null,
   };
 
   if (!authorizer.hasAccess(resource, 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
   const metadata = {
     fileType: ctx.request.headers["metadata.filetype"] || "",
     oui: ctx.request.headers["metadata.oui"] || "",
     productClass: ctx.request.headers["metadata.productclass"] || "",
-    version: ctx.request.headers["metadata.version"] || ""
+    version: ctx.request.headers["metadata.version"] || "",
   };
 
   const validate = authorizer.getValidator(resource, metadata);
@@ -458,6 +467,12 @@ router.put("/files/:id", async (ctx, next) => {
     return void (ctx.status = 403);
   }
 
+  try {
+    await db.deleteFile(id);
+  } catch (err) {
+    // File doesn't exist, ignore
+  }
+
   await db.putFile(id, metadata, ctx.req);
   log.metadata = metadata;
   logger.accessInfo(log);
@@ -465,28 +480,64 @@ router.put("/files/:id", async (ctx, next) => {
   ctx.body = "";
 });
 
-router.post("/devices/:id/tasks", async (ctx, next) => {
+router.get("/uploads/blob/:id(.+)", async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
+  const resource = "uploads";
+  const id = ctx.params.id;
+
+  const log = {
+    message: `Download ${resource} blob`,
+    context: ctx,
+    id: id,
+  };
+  1;
+
+  const filter = and(authorizer.getFilter(resource, 2), [
+    "=",
+    ["PARAM", RESOURCE_IDS[resource]],
+    id,
+  ]);
+
+  if (!authorizer.hasAccess(resource, 2)) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 403);
+  }
+
+  const res = await db.query(resource, filter);
+  if (!res[0]) return void (ctx.status = 404);
+
+  ctx.type = "application/octet-stream";
+  ctx.attachment(id.replace(/\//g, "_"));
+  ctx.body = new stream.PassThrough();
+
+  const downloadStream = await db.getUploadBlob(id);
+  downloadStream.pipe(ctx.body);
+  logger.accessInfo(log);
+});
+
+router.post("/devices/:id/tasks", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Commit tasks",
     context: ctx,
     deviceId: ctx.params.id,
-    tasks: null
+    tasks: null,
   };
 
   const filter = and(authorizer.getFilter("devices", 3), [
     "=",
     ["PARAM", "DeviceID.ID"],
-    ctx.params.id
+    ctx.params.id,
   ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
   const devices = await db.query("devices", filter);
-  if (!devices.length) return void next();
+  if (!devices.length) return void (ctx.status = 404);
+  const device = devices[0];
 
-  const validate = authorizer.getValidator("devices", devices[0]);
+  const validate = authorizer.getValidator("devices", device);
   for (const t of ctx.request.body) {
     if (!validate("task", t)) {
       logUnauthorizedWarning(log);
@@ -494,19 +545,45 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
     }
   }
 
-  // TODO Pass context and timestamp
   const onlineThreshold = getConfig(
     ctx.state.configSnapshot,
-    "cwmp.deviceOnlineThreshold"
-  );
+    "cwmp.deviceOnlineThreshold",
+    {},
+    Date.now(),
+    (exp) => {
+      if (!Array.isArray(exp)) return exp;
+      if (exp[0] === "PARAM") {
+        const p = device[exp[1]];
+        if (p && p.value) return p.value[0];
+      } else if (exp[0] === "FUNC") {
+        if (exp[1] === "REMOTE_ADDRESS") {
+          for (const root of ["InternetGatewayDevice", "Device"]) {
+            const p = device[`${root}.ManagementServer.ConnectionRequestURL`];
+            if (p && p.value) return url.parse(p.value[0]).host;
+          }
+          return null;
+        }
+      }
+      return exp;
+    }
+  ) as number;
+
+  const socketTimeout: number = ctx.socket["timeout"];
+
+  // Disable socket timeout while waiting for postTasks()
+  if (socketTimeout) ctx.socket.setTimeout(0);
 
   const res = await apiFunctions.postTasks(
     ctx.params.id,
     ctx.request.body,
-    onlineThreshold
+    onlineThreshold,
+    device
   );
 
-  log.tasks = res.tasks.map(t => t._id).join(",");
+  // Restore socket timeout
+  if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
+
+  log.tasks = res.tasks.map((t) => t._id).join(",");
 
   logger.accessInfo(log);
 
@@ -514,26 +591,26 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
   ctx.body = res.tasks;
 });
 
-router.post("/devices/:id/tags", async (ctx, next) => {
+router.post("/devices/:id/tags", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Update tags",
     context: ctx,
     deviceId: ctx.params.id,
-    tags: ctx.request.body
+    tags: ctx.request.body,
   };
 
   const filter = and(authorizer.getFilter("devices", 3), [
     "=",
     ["PARAM", "DeviceID.ID"],
-    ctx.params.id
+    ctx.params.id,
   ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
   const res = await db.query("devices", filter);
-  if (!res.length) return void next();
+  if (!res.length) return void (ctx.status = 404);
 
   const validate = authorizer.getValidator("devices", res[0]);
   if (!validate("tags", ctx.request.body)) {
@@ -555,8 +632,8 @@ router.post("/devices/:id/tags", async (ctx, next) => {
   ctx.body = "";
 });
 
-router.get("/ping/:host", async ctx => {
-  return new Promise(resolve => {
+router.get("/ping/:host", async (ctx) => {
+  return new Promise((resolve) => {
     ping(ctx.params.host, (err, parsed) => {
       if (parsed) {
         ctx.body = parsed;
@@ -569,13 +646,13 @@ router.get("/ping/:host", async ctx => {
   });
 });
 
-router.put("/users/:id/password", async (ctx, next) => {
+router.put("/users/:id/password", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const username = ctx.params.id;
   const log = {
     message: "Change password",
     context: ctx,
-    username: username
+    username: username,
   };
 
   if (!ctx.state.user) {
@@ -594,16 +671,16 @@ router.put("/users/:id/password", async (ctx, next) => {
     }
   } else if (!authorizer.hasAccess("users", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
   const filter = and(authorizer.getFilter("users", 3), [
     "=",
     ["PARAM", RESOURCE_IDS.users],
-    username
+    username,
   ]);
   const res = await db.query("users", filter);
-  if (!res.length) return void next();
+  if (!res.length) return void (ctx.status = 404);
 
   const newPassword = ctx.request.body.newPassword;
   if (ctx.state.user) {

lib/ui/db.ts

@@ -17,89 +17,36 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, GridFSBucket } from "mongodb";
+import { Db, GridFSBucket, ObjectID } from "mongodb";
 import { Script } from "vm";
-import * as config from "../config";
+import { onConnect } from "../db";
 import * as mongodbFunctions from "../mongodb-functions";
 import * as expression from "../common/expression";
-import { QueryOptions } from "../types";
-
-const CACHE_TTL = 300000;
-
-let clientPromise: Promise<MongoClient>;
+import { QueryOptions, Expression } from "../types";
+import { Readable } from "stream";
+import { minimize } from "../common/boolean-expression";
 
 const RESOURCE_COLLECTION = {
-  files: "fs.files"
+  files: "fs.files",
+  uploads: "uploads.files",
 };
 
-function ensureIndexes(client): void {
-  client
-    .db()
-    .collection("cache")
-    .createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
-}
+let db: Db;
 
-function getClient(): Promise<MongoClient> {
-  if (!clientPromise) {
-    clientPromise = new Promise((resolve, reject) => {
-      const CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
-      MongoClient.connect(
-        CONNECTION_URL,
-        { useNewUrlParser: true },
-        (err, client) => {
-          if (err) return void reject(err);
-          ensureIndexes(client);
-          resolve(client);
-        }
-      );
-    });
-  }
-
-  return clientPromise;
-}
-
-export function cache<T>(key, valueGetter: () => Promise<T>, ttl): Promise<T> {
-  return new Promise((resolve, reject) => {
-    getClient()
-      .then(client => {
-        const collection = client.db().collection("cache");
-        collection.findOne({ _id: key }, (err, doc) => {
-          if (err) return void reject(err);
-          if (doc != null) return void resolve(JSON.parse(doc.value) as T);
-          valueGetter()
-            .then(res => {
-              const expire = Date.now() + (ttl || CACHE_TTL);
-              const cacheDoc = {
-                _id: key,
-                value: JSON.stringify(res),
-                expire: new Date(expire)
-              };
-              collection.updateOne(
-                { _id: key },
-                { $set: cacheDoc },
-                { upsert: true },
-                err => {
-                  if (err) reject(err);
-                  else resolve(res);
-                }
-              );
-            })
-            .catch(reject);
-        });
-      })
-      .catch(reject);
-  });
-}
+onConnect(async (_db) => {
+  db = _db;
+});
 
 export function query(
-  resource,
-  filter,
+  resource: string,
+  filter: Expression,
   options?: QueryOptions,
   callback?: (doc: any) => void
 ): Promise<any[]> {
   options = options || {};
   let q;
   filter = expression.evaluate(filter, null, Date.now());
+  filter = minimize(filter, true);
 
   if (Array.isArray(filter)) {
     if (resource === "devices") {
@@ -116,89 +63,89 @@ export function query(
     }
 
     q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (filter != null && !filter) {
+  } else if (!filter) {
     return Promise.resolve([]);
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then(client => {
-      const collection = client
-        .db()
-        .collection(RESOURCE_COLLECTION[resource] || resource);
-      const cursor = collection.find(q);
-      if (options.projection) {
-        let projection = options.projection;
-        if (resource === "devices") {
-          projection = mongodbFunctions.processDeviceProjection(
-            options.projection
-          );
-        }
-
-        if (resource === "presets") projection.configurations = 1;
-        cursor.project(projection);
-      }
-
-      if (resource === "users") cursor.project({ password: 0, salt: 0 });
-
-      if (options.skip) cursor.skip(options.skip);
-      if (options.limit) cursor.limit(options.limit);
-
-      if (options.sort) {
-        let s = Object.entries(options.sort)
-          .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
-          .reduce(
-            (obj, [k, v]) =>
-              Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
-            {}
-          );
-
-        if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
-        cursor.sort(s);
-      }
-
-      if (!callback) {
-        cursor.toArray((err, docs) => {
-          if (err) return reject(err);
-          if (resource === "devices")
-            docs = docs.map(d => mongodbFunctions.flattenDevice(d));
-          else if (resource === "faults")
-            docs = docs.map(d => mongodbFunctions.flattenFault(d));
-          else if (resource === "tasks")
-            docs = docs.map(d => mongodbFunctions.flattenTask(d));
-          else if (resource === "presets")
-            docs = docs.map(d => mongodbFunctions.flattenPreset(d));
-          else if (resource === "files")
-            docs = docs.map(d => mongodbFunctions.flattenFile(d));
-          return resolve(docs);
-        });
-      } else {
-        cursor.forEach(
-          doc => {
-            if (resource === "devices")
-              doc = mongodbFunctions.flattenDevice(doc);
-            else if (resource === "faults")
-              doc = mongodbFunctions.flattenFault(doc);
-            else if (resource === "tasks")
-              doc = mongodbFunctions.flattenTask(doc);
-            else if (resource === "presets")
-              doc = mongodbFunctions.flattenPreset(doc);
-            else if (resource === "files")
-              doc = mongodbFunctions.flattenFile(doc);
-            callback(doc);
-          },
-          err => {
-            if (err) reject(err);
-            else resolve();
-          }
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
+    const cursor = collection.find(q);
+    if (options.projection) {
+      let projection = options.projection;
+      if (resource === "devices") {
+        projection = mongodbFunctions.processDeviceProjection(
+          options.projection
         );
       }
-    });
+
+      if (resource === "presets") projection.configurations = 1;
+      cursor.project(projection);
+    }
+
+    if (resource === "users") cursor.project({ password: 0, salt: 0 });
+
+    if (options.skip) cursor.skip(options.skip);
+    if (options.limit) cursor.limit(options.limit);
+
+    if (options.sort) {
+      let s = Object.entries(options.sort)
+        .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
+        .reduce(
+          (obj, [k, v]) =>
+            Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
+          {}
+        );
+
+      if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
+      cursor.sort(s);
+    }
+
+    if (!callback) {
+      cursor.toArray((err, docs) => {
+        if (err) return reject(err);
+        if (resource === "devices")
+          docs = docs.map((d) => mongodbFunctions.flattenDevice(d));
+        else if (resource === "faults")
+          docs = docs.map((d) => mongodbFunctions.flattenFault(d));
+        else if (resource === "tasks")
+          docs = docs.map((d) => mongodbFunctions.flattenTask(d));
+        else if (resource === "presets")
+          docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
+        else if (resource === "files")
+          docs = docs.map((d) => mongodbFunctions.flattenFile(d));
+        else if (resource === "uploads")
+          docs = docs.map((d) => mongodbFunctions.flattenUpload(d));
+        return resolve(docs);
+      });
+    } else {
+      cursor.forEach(
+        (doc) => {
+          if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc);
+          else if (resource === "faults")
+            doc = mongodbFunctions.flattenFault(doc);
+          else if (resource === "tasks")
+            doc = mongodbFunctions.flattenTask(doc);
+          else if (resource === "presets")
+            doc = mongodbFunctions.flattenPreset(doc);
+          else if (resource === "files")
+            doc = mongodbFunctions.flattenFile(doc);
+          else if (resource === "uploads")
+            doc = mongodbFunctions.flattenUpload(doc);
+          callback(doc);
+        },
+        (err) => {
+          if (err) reject(err);
+          else resolve();
+        }
+      );
+    }
   });
 }
 
-export function count(resource, filter): Promise<number> {
+export function count(resource: string, filter: Expression): Promise<number> {
   let q;
   filter = expression.evaluate(filter, null, Date.now());
+  filter = minimize(filter, true);
 
   if (Array.isArray(filter)) {
     if (resource === "devices")
@@ -208,97 +155,80 @@ export function count(resource, filter): Promise<number> {
     else if (resource === "faults")
       filter = mongodbFunctions.processFaultsFilter(filter);
     q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (filter != null && !filter) {
+  } else if (!filter) {
     return Promise.resolve(0);
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then(client => {
-      const collection = client
-        .db()
-        .collection(RESOURCE_COLLECTION[resource] || resource);
-      collection.find(q).count((err, c) => {
-        if (err) reject(err);
-        else resolve(c);
-      });
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
+    collection.find(q).count((err, c) => {
+      if (err) reject(err);
+      else resolve(c);
     });
   });
 }
 
-export function updateDeviceTags(deviceId, tags): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const add = [];
-    const pull = [];
+export async function updateDeviceTags(
+  deviceId: string,
+  tags: Record<string, boolean>
+): Promise<void> {
+  const add = [];
+  const pull = [];
 
-    const regex = /^[0-9a-zA-Z_]+$/;
-    for (let [tag, onOff] of Object.entries(tags)) {
-      tag = tag.trim();
-      if (onOff) {
-        if (!tag.match(regex))
-          return void reject(new Error(`Invalid tag '${tag}'`));
-        add.push(tag);
-      } else {
-        pull.push(tag);
-      }
-    }
-    getClient()
-      .then(client => {
-        const collection = client.db().collection("devices");
+  for (let [tag, onOff] of Object.entries(tags)) {
+    tag = tag.trim();
+    if (onOff) add.push(tag);
+    else pull.push(tag);
+  }
 
-        const object = {};
-        if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
-        if (pull && pull.length) object["$pullAll"] = { _tags: pull };
-        collection.updateOne({ _id: deviceId }, object, err => {
-          if (err) return void reject(err);
-          resolve();
-        });
-      })
-      .catch(reject);
-  });
+  const collection = db.collection("devices");
+  const object = {};
+
+  if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
+  if (pull && pull.length) object["$pullAll"] = { _tags: pull };
+
+  await collection.updateOne({ _id: deviceId }, object);
 }
 
 function putResource(resource, id, object): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
-      .then(client => {
-        const collection = client
-          .db()
-          .collection(RESOURCE_COLLECTION[resource] || resource);
-        collection.replaceOne({ _id: id }, object, { upsert: true }, err => {
-          if (err) return void reject(err);
-          resolve();
-        });
-      })
-      .catch(reject);
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
+    collection.replaceOne({ _id: id }, object, { upsert: true }, (err) => {
+      if (err) return void reject(err);
+      resolve();
+    });
   });
 }
 
-function deleteResource(resource, id): Promise<void> {
+function deleteResource(
+  resource: string,
+  id: string | ObjectID
+): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
-      .then(client => {
-        const collection = client
-          .db()
-          .collection(RESOURCE_COLLECTION[resource] || resource);
-        collection.deleteOne({ _id: id }, err => {
-          if (err) return void reject(err);
-          resolve();
-        });
-      })
-      .catch(reject);
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
+    collection.deleteOne({ _id: id }, (err) => {
+      if (err) return void reject(err);
+      resolve();
+    });
   });
 }
 
-export function putPreset(id, object): Promise<void> {
+export function putPreset(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   object = mongodbFunctions.preProcessPreset(object);
   return putResource("presets", id, object);
 }
 
-export function deletePreset(id): Promise<void> {
+export function deletePreset(id: string): Promise<void> {
   return deleteResource("presets", id);
 }
 
-export function putProvision(id, object): Promise<void> {
+export function putProvision(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   if (!object.script) object.script = "";
   try {
     new Script(`"use strict";(function(){\n${object.script}\n})();`);
@@ -308,11 +238,14 @@ export function putProvision(id, object): Promise<void> {
   return putResource("provisions", id, object);
 }
 
-export function deleteProvision(id): Promise<void> {
+export function deleteProvision(id: string): Promise<void> {
   return deleteResource("provisions", id);
 }
 
-export function putVirtualParameter(id, object): Promise<void> {
+export function putVirtualParameter(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   if (!object.script) object.script = "";
   try {
     new Script(`"use strict";(function(){\n${object.script}\n})();`);
@@ -322,89 +255,96 @@ export function putVirtualParameter(id, object): Promise<void> {
   return putResource("virtualParameters", id, object);
 }
 
-export function deleteVirtualParameter(id): Promise<void> {
+export function deleteVirtualParameter(id: string): Promise<void> {
   return deleteResource("virtualParameters", id);
 }
 
-export function putConfig(id, object): Promise<void> {
+export function putConfig(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   return putResource("config", id, object);
 }
 
-export function deleteConfig(id): Promise<void> {
+export function deleteConfig(id: string): Promise<void> {
   return deleteResource("config", id);
 }
 
-export function putPermission(id, object): Promise<void> {
+export function putPermission(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   return putResource("permissions", id, object);
 }
 
-export function deletePermission(id): Promise<void> {
+export function deletePermission(id: string): Promise<void> {
   return deleteResource("permissions", id);
 }
 
-export function putUser(id, object): Promise<void> {
+export function putUser(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
-      .then(client => {
-        const collection = client.db().collection("users");
-        // update instead of replace to keep the password if not set by user
-        collection.updateOne(
-          { _id: id },
-          { $set: object },
-          { upsert: true },
-          err => {
-            if (err) return void reject(err);
-            resolve();
-          }
-        );
-      })
-      .catch(reject);
+    const collection = db.collection("users");
+    // update instead of replace to keep the password if not set by user
+    collection.updateOne(
+      { _id: id },
+      { $set: object },
+      { upsert: true },
+      (err) => {
+        if (err) return void reject(err);
+        resolve();
+      }
+    );
   });
 }
 
-export function deleteUser(id): Promise<void> {
+export function deleteUser(id: string): Promise<void> {
   return deleteResource("users", id);
 }
 
-export function putFile(filename, metadata, contentStream): Promise<void> {
+export function putFile(
+  filename: string,
+  metadata: Record<string, string>,
+  contentStream: Readable
+): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
-      .then(client => {
-        const bucket = new GridFSBucket(client.db());
-        const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
-          metadata: metadata
-        });
-        uploadStream.on("error", reject);
-        contentStream.on("error", reject);
-        uploadStream.on("finish", resolve);
-        contentStream.pipe(uploadStream);
-      })
-      .catch(reject);
+    const bucket = new GridFSBucket(db);
+    const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
+      metadata: metadata,
+    });
+    uploadStream.on("error", reject);
+    contentStream.on("error", reject);
+    uploadStream.on("finish", resolve);
+    contentStream.pipe(uploadStream);
   });
 }
 
-export function deleteFile(filename): Promise<void> {
+export function deleteFile(filename: string): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
-      .then(client => {
-        const bucket = new GridFSBucket(client.db());
-        bucket.delete(filename, err => {
-          if (err) return void reject(err);
-          resolve();
-        });
-      })
-      .catch(reject);
+    const bucket = new GridFSBucket(db);
+    bucket.delete(filename as any, (err) => {
+      if (err) return void reject(err);
+      resolve();
+    });
   });
 }
 
-export function deleteFault(id): Promise<void> {
+export async function deleteUpload(filename: string): Promise<void> {
+  const bucket = new GridFSBucket(db, { bucketName: "uploads" });
+  await bucket.delete(filename as any);
+}
+
+export async function getUploadBlob(filename: string): Promise<Readable> {
+  const bucket = new GridFSBucket(db, { bucketName: "uploads" });
+  return bucket.openDownloadStreamByName(filename);
+}
+
+export function deleteFault(id: string): Promise<void> {
   return deleteResource("faults", id);
 }
 
-export function deleteTask(id): Promise<void> {
+export function deleteTask(id: ObjectID): Promise<void> {
   return deleteResource("tasks", id);
 }
-
-export async function disconnect(): Promise<void> {
-  if (clientPromise) await (await clientPromise).close();
-}

lib/versioned-map.ts

@@ -46,7 +46,7 @@ export default class VersionedMap<K, V> {
     return this._revision;
   }
 
-  public set revision(rev) {
+  public set revision(rev: number) {
     for (let i = this._sizeDiff.length; i <= rev; ++i)
       this._sizeDiff[i] = this._sizeDiff[i - 1];
 
@@ -193,7 +193,7 @@ export default class VersionedMap<K, V> {
       const last = v[l];
       v.splice(revision, l - revision);
 
-      if (last === NONEXISTENT && !v.some(vv => vv !== NONEXISTENT))
+      if (last === NONEXISTENT && !v.some((vv) => vv !== NONEXISTENT))
         this.map.delete(k);
     }
   }

lib/xml-parser.ts

@@ -53,12 +53,13 @@ export interface Element {
 }
 
 export function parseXmlDeclaration(buffer: Buffer): Attribute[] {
-  for (const enc of ["utf16le", "utf8", "latin1", "ascii"]) {
+  const encodings: BufferEncoding[] = ["utf16le", "utf8", "latin1", "ascii"];
+  for (const enc of encodings) {
     let str = buffer.toString(enc, 0, 150);
     if (str.startsWith("<?xml")) {
-      str = str.split("\n")[0].trim();
+      str = str.slice(0, str.indexOf("?>"));
       try {
-        return parseAttrs(str.slice(5, -2));
+        return parseAttrs(str.slice(5));
       } catch (err) {
         // Ignore
       }
@@ -90,7 +91,7 @@ export function parseAttrs(string: string): Attribute[] {
               name: name,
               namespace: namespace,
               localName: localName,
-              value: value
+              value: value,
             };
             attrs.push(e);
             name = "";
@@ -103,10 +104,12 @@ export function parseAttrs(string: string): Attribute[] {
         continue;
 
       case CHAR_COLON:
+        if (state) continue;
         if (idx >= colonIdx) colonIdx = i;
         continue;
 
       case CHAR_EQUAL:
+        if (state) continue;
         if (name) throw new Error(`Unexpected character at ${i}`);
         name = string.slice(idx, i).trim();
         // TODO validate name
@@ -128,8 +131,8 @@ export function parseAttrs(string: string): Attribute[] {
   return attrs;
 }
 
-export function decodeEntities(string): string {
-  return string.replace(/&[0-9a-z#]+;/gi, match => {
+export function decodeEntities(string: string): string {
+  return string.replace(/&[0-9a-z#]+;/gi, (match) => {
     switch (match) {
       case "&quot;":
         return '"';
@@ -161,15 +164,15 @@ export function decodeEntities(string): string {
   });
 }
 
-export function encodeEntities(string): string {
+export function encodeEntities(string: string): string {
   const entities = {
     "&": "&amp;",
     '"': "&quot;",
     "'": "&apos;",
     "<": "&lt;",
-    ">": "&gt;"
+    ">": "&gt;",
   };
-  return string.replace(/[&"'<>]/g, m => entities[m]);
+  return string.replace(/[&"'<>]/g, (m) => entities[m]);
 }
 
 export function parseXml(string: string): Element {
@@ -186,7 +189,7 @@ export function parseXml(string: string): Element {
     attrs: "",
     text: "",
     bodyIndex: 0,
-    children: []
+    children: [],
   };
 
   const stack: Element[] = [root];
@@ -330,7 +333,7 @@ export function parseXml(string: string): Element {
                   : "",
                 text: "",
                 bodyIndex: i + 1,
-                children: []
+                children: [],
               };
               parent.children.push(e);
               if (!selfClosing) stack.push(e);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.0-beta.0",
+  "version": "1.2.3",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -20,69 +20,69 @@
     "tools": "tools"
   },
   "dependencies": {
-    "iconv-lite": "^0.5.0",
+    "espresso-iisojs": "^1.0.3",
+    "iconv-lite": "^0.6.2",
+    "ipaddr.js": "^2.0.0",
     "jsonwebtoken": "^8.5.1",
-    "koa": "^2.7.0",
-    "koa-bodyparser": "^4.2.1",
-    "koa-compress": "^3.0.0",
-    "koa-jwt": "^3.6.0",
-    "koa-router": "^7.4.0",
+    "koa": "^2.13.0",
+    "koa-bodyparser": "^4.3.0",
+    "koa-compress": "^5.0.1",
+    "koa-jwt": "^4.0.0",
+    "koa-router": "^9.4.0",
     "koa-static": "^5.0.0",
     "later": "^1.2.0",
-    "mongodb": "^3.2.6",
-    "parsimmon": "^1.13.0",
-    "seedrandom": "^3.0.1",
-    "yaml": "^1.6.0"
-  },
-  "optionalDependencies": {
-    "libxmljs": "^0.19.5"
+    "mongodb": "^3.6.2",
+    "parsimmon": "^1.16.0",
+    "seedrandom": "^3.0.5",
+    "yaml": "^1.10.0"
   },
   "devDependencies": {
-    "@types/jsonwebtoken": "^8.3.2",
-    "@types/koa": "^2.0.49",
-    "@types/koa-compress": "^2.0.9",
-    "@types/koa-router": "^7.0.42",
+    "@rollup/plugin-commonjs": "^16.0.0",
+    "@rollup/plugin-json": "^4.1.0",
+    "@rollup/plugin-node-resolve": "^10.0.0",
+    "@rollup/plugin-typescript": "^6.0.0",
+    "@types/jsonwebtoken": "^8.5.0",
+    "@types/koa": "^2.11.6",
+    "@types/koa-compress": "^4.0.1",
+    "@types/koa-router": "^7.4.1",
     "@types/koa-static": "^4.0.1",
-    "@types/libxmljs": "^0.18.3",
-    "@types/mithril": "^2.0.0",
-    "@types/mongodb": "^3.1.31",
-    "@types/parsimmon": "^1.10.0",
+    "@types/mithril": "^2.0.5",
+    "@types/mongodb": "^3.5.31",
+    "@types/node": "^14.14.5",
+    "@types/parsimmon": "^1.10.3",
     "@types/seedrandom": "^2.4.28",
-    "@types/svgo": "^1.2.1",
-    "@typescript-eslint/eslint-plugin": "^1.13.0",
-    "@typescript-eslint/parser": "^1.13.0",
-    "ava": "^2.2.0",
-    "codemirror": "^5.48.2",
+    "@types/svgo": "^1.3.3",
+    "@typescript-eslint/eslint-plugin": "^4.6.0",
+    "@typescript-eslint/parser": "^4.6.0",
+    "ava": "^3.13.0",
+    "codemirror": "^5.58.2",
     "cssnano": "^4.1.10",
-    "eslint": "^6.1.0",
-    "eslint-config-prettier": "^6.0.0",
+    "eslint": "^7.12.1",
+    "eslint-config-prettier": "^6.14.0",
     "esm": "^3.2.25",
-    "mithril": "^2.0.3",
-    "postcss": "^7.0.17",
-    "postcss-import": "^12.0.1",
+    "jsbi": "^3.1.4",
+    "mithril": "^2.0.4",
+    "postcss": "^8.1.4",
+    "postcss-import": "^13.0.0",
     "postcss-preset-env": "^6.7.0",
-    "prettier": "^1.18.2",
-    "rollup": "^1.17.0",
-    "rollup-plugin-json": "^4.0.0",
-    "rollup-plugin-replace": "^2.2.0",
-    "rollup-plugin-terser": "^5.1.1",
-    "rollup-plugin-typescript": "^1.0.1",
-    "source-map-loader": "^0.2.4",
-    "svgo": "^1.3.0",
-    "ts-node": "^8.3.0",
-    "typescript": "^3.5.3",
-    "webpack": "^4.38.0"
+    "prettier": "^2.1.2",
+    "rollup": "^2.32.1",
+    "rollup-plugin-terser": "^7.0.2",
+    "source-map-loader": "^1.1.1",
+    "svgo": "^1.3.2",
+    "ts-node": "^9.0.0",
+    "typescript": "^4.0.5",
+    "webpack": "^5.2.0"
   },
   "engines": {
-    "node": ">=8.14.0"
+    "node": ">=12.13.0"
   },
   "scripts": {
     "test": "ava --verbose --color",
-    "lint-ln": "find bin -type f -exec sh -c 'ln -fs \"$(sh -c pwd)/$1\" \"bin/.${1#bin/}.ts\"' _ {} \\;; find tools -type f -exec sh -c 'ln -fs \"$(sh -c pwd)/$1\" \"tools/.${1#tools/}.ts\"' _ {} \\;",
-    "lint-eslint": "eslint --fix \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
+    "lint-eslint": "eslint --fix \"bin/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/*.ts\"",
     "lint-prettier": "prettier --parser typescript --write \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
     "lint-tsc": "tsc --noEmit",
-    "lint": "npm run lint-ln && npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
+    "lint": "npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
     "build": "node -r esm -r ts-node/register/transpile-only build/build"
   }
 }

test/auth.ts

@@ -2,7 +2,7 @@ import ava from "ava";
 import { randomBytes } from "crypto";
 import * as auth from "../lib/auth";
 
-ava("digest", t => {
+ava("digest", (t) => {
   const username = "test";
   const password = "test";
   const uri = "/";
@@ -14,7 +14,7 @@ ava("digest", t => {
   const challenges = [
     `Digest realm="${realm}",nonce="${nonce}"`,
     `Digest realm="${realm}",nonce="${nonce}",qop="auth"`,
-    `Digest realm="${realm}",nonce="${nonce}",qop="auth-int"`
+    `Digest realm="${realm}",nonce="${nonce}",qop="auth-int"`,
   ];
 
   t.plan(challenges.length);

test/common.ts

@@ -1,35 +1,7 @@
 import ava from "ava";
 import * as common from "../lib/common";
 
-ava("typeOf", t => {
-  const cases = [
-    [undefined, common.UNDEFINED_TYPE],
-    [null, common.NULL_TYPE],
-    [true, common.BOOLEAN_TYPE],
-    [false, common.BOOLEAN_TYPE],
-    [0, common.NUMBER_TYPE],
-    [1, common.NUMBER_TYPE],
-    [-1, common.NUMBER_TYPE],
-    [0.0, common.NUMBER_TYPE],
-    [1.0, common.NUMBER_TYPE],
-    [1.1, common.NUMBER_TYPE],
-    [-1.0, common.NUMBER_TYPE],
-    [-1.1, common.NUMBER_TYPE],
-    ["", common.STRING_TYPE],
-    ["TEST", common.STRING_TYPE],
-    [[], common.ARRAY_TYPE],
-    [[1, 2, 3], common.ARRAY_TYPE],
-    [{}, common.OBJECT_TYPE],
-    [{ test: 1 }, common.OBJECT_TYPE],
-    [/[^A-Za-z0-9_]/g, common.REGEXP_TYPE],
-    [new RegExp("//"), common.REGEXP_TYPE],
-    [new Date(), common.DATE_TYPE]
-  ];
-  t.plan(cases.length);
-  for (const c of cases) t.is(common.typeOf(c[0]), c[1]);
-});
-
-ava("generateDeviceId", t => {
+ava("generateDeviceId", (t) => {
   const space = [" ", "%20"];
   const special = [";", "%3B"];
   const cases = [
@@ -37,30 +9,31 @@ ava("generateDeviceId", t => {
       {
         ProductClass: "TestProductClass",
         OUI: "TestOUI",
-        SerialNumber: "TestSerialNumber"
+        SerialNumber: "TestSerialNumber",
       },
-      "TestOUI-TestProductClass-TestSerialNumber"
+      "TestOUI-TestProductClass-TestSerialNumber",
     ],
     [
       {
         OUI: "TestOUI",
-        SerialNumber: "TestSerialNumber"
+        SerialNumber: "TestSerialNumber",
       },
-      "TestOUI-TestSerialNumber"
+      "TestOUI-TestSerialNumber",
     ],
     [
       {
         OUI: `TestOUIWith${space[0]}_${special[0]}2912`,
-        SerialNumber: `TestSerialNumberWith${space[0]}_${special[0]}2912`
+        SerialNumber: `TestSerialNumberWith${space[0]}_${special[0]}2912`,
       },
-      `TestOUIWith${space[1]}_${special[1]}2912-TestSerialNumberWith${space[1]}_${special[1]}2912`
-    ]
+      `TestOUIWith${space[1]}_${special[1]}2912-TestSerialNumberWith${space[1]}_${special[1]}2912`,
+    ],
   ];
   t.plan(cases.length);
-  for (const c of cases) t.is(common.generateDeviceId(c[0]), c[1]);
+  for (const c of cases)
+    t.is(common.generateDeviceId(c[0] as Record<string, string>), c[1]);
 });
 
-ava("escapeRegExp", t => {
+ava("escapeRegExp", (t) => {
   t.is(
     common.escapeRegExp("\\ ^ $ * + ? . ( ) | { } [ ]"),
     "\\\\ \\^ \\$ \\* \\+ \\? \\. \\( \\) \\| \\{ \\} \\[ \\]"

test/device.ts

@@ -5,7 +5,7 @@ import PathSet from "../lib/common/path-set";
 import VersionedMap from "../lib/versioned-map";
 import { Attributes } from "../lib/types";
 
-ava("getAliasDeclarations", t => {
+ava("getAliasDeclarations", (t) => {
   const path = Path.parse("a.[aa:10,bb.[aaa:100].cc:1].b");
   const decs = device.getAliasDeclarations(path, 99);
 
@@ -22,64 +22,64 @@ ava("getAliasDeclarations", t => {
   }
 });
 
-ava("unpack", t => {
+ava("unpack", (t) => {
   const now = Date.now();
   const deviceData = {
     paths: new PathSet(),
     timestamps: new VersionedMap<Path, number>(),
     attributes: new VersionedMap<Path, Attributes>(),
     trackers: new Map<Path, { [name: string]: number }>(),
-    changes: new Set<string>()
+    changes: new Set<string>(),
   };
 
   device.set(deviceData, Path.parse("a.1.b"), now, {
-    value: [now, ["b", "xsd:string"]]
+    value: [now, ["b", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.c"), now, {
-    value: [now, ["c", "xsd:string"]]
+    value: [now, ["c", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.1.a"), now, {
-    value: [now, ["", "xsd:string"]]
+    value: [now, ["", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.1.b"), now, {
-    value: [now, ["b1", "xsd:string"]]
+    value: [now, ["b1", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.1.c"), now, {
-    value: [now, ["c1", "xsd:string"]]
+    value: [now, ["c1", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.2.a"), now, {
-    value: [now, ["", "xsd:string"]]
+    value: [now, ["", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.2.b"), now, {
-    value: [now, ["b2", "xsd:string"]]
+    value: [now, ["b2", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.2.c"), now, {
-    value: [now, ["c2", "xsd:string"]]
+    value: [now, ["c2", "xsd:string"]],
   });
 
   device.set(deviceData, Path.parse("a.2.b"), now, {
-    value: [now, ["b", "xsd:string"]]
+    value: [now, ["b", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.c"), now, {
-    value: [now, ["c", "xsd:string"]]
+    value: [now, ["c", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.1.a"), now, {
-    value: [now, ["", "xsd:string"]]
+    value: [now, ["", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.1.b"), now, {
-    value: [now, ["b1", "xsd:string"]]
+    value: [now, ["b1", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.1.c"), now, {
-    value: [now, ["c1", "xsd:string"]]
+    value: [now, ["c1", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.2.a"), now, {
-    value: [now, ["", "xsd:string"]]
+    value: [now, ["", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.2.b"), now, {
-    value: [now, ["c1", "xsd:string"]]
+    value: [now, ["c1", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.2.c"), now, {
-    value: [now, ["b1", "xsd:string"]]
+    value: [now, ["b1", "xsd:string"]],
   });
 
   let unpacked: Path[];

test/mongodb-functions.ts

@@ -1,13 +1,13 @@
 import ava from "ava";
 import * as mongodbFunctions from "../lib/mongodb-functions";
-import { stringify } from "../lib/common/expression-parser";
+import { stringify, parse } from "../lib/common/expression-parser";
 
-ava("mongoQueryToFilter", t => {
+ava("convertOldPrecondition", (t) => {
   const tests = [
     [{}, "true"],
     [{ test: "test" }, 'test = "test"'],
     [{ test: { $eq: "test" } }, 'test = "test"'],
-    [{ test: { $ne: "test" } }, 'test <> "test"'],
+    [{ test: { $ne: "test" } }, 'test <> "test" OR test IS NULL'],
     [{ test: { $gte: "test" } }, 'test >= "test"'],
     [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
     [{ "Tags.test": false }, "Tags.test IS NULL"],
@@ -22,21 +22,21 @@ ava("mongoQueryToFilter", t => {
     [{ _tags: { $eq: "test" } }, "Tags.test IS NOT NULL"],
     [
       { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" AND test <> "test"'
+      'test = "test" AND (test <> "test" OR test IS NULL)',
     ],
     [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
     [
       { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" OR test <> "test"'
+      'test = "test" OR (test <> "test" OR test IS NULL)',
     ],
     [
       { test: { $gte: "test1", $ne: "test2" } },
-      'test >= "test1" AND test <> "test2"'
+      'test >= "test1" AND (test <> "test2" OR test IS NULL)',
     ],
     [
       { test: "test", test2: { $ne: "test2" } },
-      'test = "test" AND test2 <> "test2"'
-    ]
+      'test = "test" AND (test2 <> "test2" OR test2 IS NULL)',
+    ],
   ];
 
   const shouldFailTests = [
@@ -45,18 +45,88 @@ ava("mongoQueryToFilter", t => {
     [{ "Tags.test": { $gt: true } }, "Invalid tag query"],
     [{ _tags: [] }, "Invalid type"],
     [{ _tags: { $gt: "test" } }, "Invalid tag query"],
-    [{ $nor: [] }, "Operator $nor not supported"]
+    [{ $nor: [] }, "Operator $nor not supported"],
   ];
 
   t.plan(tests.length + 2 * shouldFailTests.length);
   for (const test of tests)
-    t.is(stringify(mongodbFunctions.mongoQueryToFilter(test[0])), test[1]);
+    t.is(stringify(mongodbFunctions.convertOldPrecondition(test[0])), test[1]);
 
   for (const test of shouldFailTests) {
     const func = (): void => {
-      mongodbFunctions.mongoQueryToFilter(test[0]);
+      mongodbFunctions.convertOldPrecondition(
+        test[0] as Record<string, unknown>
+      );
     };
-    const error = t.throws(func, Error);
+    const error = t.throws(func, { instanceOf: Error });
     t.is(error.message, test[1]);
   }
 });
+
+ava("filterToMongoQuery", async (t) => {
+  const queries: [string, Record<string, unknown>][] = [
+    ["true", {}],
+    ["Tags.tag1 = true", { _tags: "tag1" }],
+    ["Tags.tag1 <> false", { _tags: "tag1" }],
+    ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }],
+    ["Tags.tag1 = 123", { "Tags.tag1": 123 }],
+    ["Param1 = 'value1'", { "Param1._value": "value1" }],
+    ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }],
+    ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
+    ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
+    ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }],
+    ["LOWER(Param1) LIKE 'value'", { "Param1._value": /^value$/i }],
+    [
+      "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          {
+            $and: [
+              { "Param2._value": { $nin: ["value1", null] } },
+              { "Param1._value": { $gte: "value2" } },
+            ],
+          },
+        ],
+      },
+    ],
+    [
+      "Param1 <> 'value2' OR Param1 IS NULL",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          { "Param1._value": null },
+        ],
+      },
+    ],
+  ];
+
+  for (const [expStr, expect] of queries) {
+    const exp = parse(expStr);
+    const query = mongodbFunctions.filterToMongoQuery(
+      mongodbFunctions.processDeviceFilter(exp)
+    );
+    t.deepEqual(query, expect);
+  }
+
+  const failQueries: [any, string][] = [
+    ["Param1 = Param2", "Invalid RHS operand of = clause"],
+    ["Param1 LIKE Param2", "Invalid RHS operand of LIKE clause"],
+    ["1 LIKE '1'", "Invalid LHS operand of LIKE clause"],
+    ["1 = 2", "Invalid LHS operand of = clause"],
+    ["1 IS NULL", "Invalid LHS operand of IS NULL clause"],
+    ["false", "Primitives are not valid queries"],
+    ["UPPER(Param1) LIKE 'value'", "Invalid RHS operand of LIKE clause"],
+  ];
+
+  for (const [expStr, err] of failQueries) {
+    const exp = parse(expStr);
+    t.throws(
+      () =>
+        mongodbFunctions.filterToMongoQuery(
+          mongodbFunctions.processDeviceFilter(exp)
+        ),
+      { message: err }
+    );
+  }
+});

test/path-set.ts

@@ -2,7 +2,7 @@ import ava from "ava";
 import Path from "../lib/common/path";
 import PathSet from "../lib/common/path-set";
 
-ava("depth", t => {
+ava("depth", (t) => {
   const pathSet = new PathSet();
   t.is(pathSet.depth, 0);
   pathSet.add(Path.parse(""));
@@ -11,14 +11,14 @@ ava("depth", t => {
   t.is(pathSet.depth, 2);
 });
 
-ava("add", t => {
+ava("add", (t) => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a"));
   pathSet.add(Path.parse("a"));
   t.is(pathSet.find(Path.parse("a"), true, true, 99).length, 1);
 });
 
-ava("get", t => {
+ava("get", (t) => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a.*"));
   pathSet.add(Path.parse("a.a"));
@@ -28,7 +28,7 @@ ava("get", t => {
   t.is(pathSet.get(Path.parse("*.a")), null);
 });
 
-ava("find", t => {
+ava("find", (t) => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a"));
   pathSet.add(Path.parse("a.*"));
@@ -37,32 +37,32 @@ ava("find", t => {
   pathSet.add(Path.parse("*.*"));
 
   t.deepEqual(
-    pathSet.find(Path.parse(""), true, true, 1).map(p => p.toString()),
+    pathSet.find(Path.parse(""), true, true, 1).map((p) => p.toString()),
     ["a"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse(""), false, false, 2).map(p => p.toString()),
+    pathSet.find(Path.parse(""), false, false, 2).map((p) => p.toString()),
     ["a", "a.*", "a.a", "*.a", "*.*"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), false, false).map(p => p.toString()),
+    pathSet.find(Path.parse("a.*"), false, false).map((p) => p.toString()),
     ["a.*"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), false, true).map(p => p.toString()),
+    pathSet.find(Path.parse("a.*"), false, true).map((p) => p.toString()),
     ["a.*", "a.a"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), true, false).map(p => p.toString()),
+    pathSet.find(Path.parse("a.*"), true, false).map((p) => p.toString()),
     ["a.*", "*.*"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), true, true).map(p => p.toString()),
+    pathSet.find(Path.parse("a.*"), true, true).map((p) => p.toString()),
     ["a.*", "a.a", "*.a", "*.*"]
   );
 });

test/path.ts

@@ -1,13 +1,13 @@
 import ava from "ava";
 import Path from "../lib/common/path";
 
-ava("toString", t => {
+ava("toString", (t) => {
   const path1 = Path.parse('abc.[ abc : 123 , 123 : abc , 123: " abc "].123');
   const path2 = Path.parse('abc.[123:  " abc ",abc:123,123:abc].123');
   t.is(path1.toString(), path2.toString());
 });
 
-ava("slice", t => {
+ava("slice", (t) => {
   const path = Path.parse("a.*.b.[x:y].c");
   const sliced = path.slice(1, -1);
   t.is(sliced.toString(), '*.b.[x:"y"]');
@@ -15,7 +15,7 @@ ava("slice", t => {
   t.is(sliced.wildcard, 0b1);
 });
 
-ava("concat", t => {
+ava("concat", (t) => {
   const path1 = Path.parse("a");
   const path2 = Path.parse("*.[a:b]");
   const concat = path1.concat(path2);

test/xml-parser.ts

@@ -2,10 +2,10 @@ import ava from "ava";
 import {
   parseXmlDeclaration,
   decodeEntities,
-  parseXml
+  parseXml,
 } from "../lib/xml-parser";
 
-ava("parseXmlDeclaration", t => {
+ava("parseXmlDeclaration", (t) => {
   const buf = Buffer.from(
     '<?xml version="1.0" encoding="UTF-8"?>\n<soap-env:Envelope />'
   );
@@ -15,25 +15,25 @@ ava("parseXmlDeclaration", t => {
       name: "version",
       namespace: "",
       localName: "version",
-      value: "1.0"
+      value: "1.0",
     },
     {
       name: "encoding",
       namespace: "",
       localName: "encoding",
-      value: "UTF-8"
-    }
+      value: "UTF-8",
+    },
   ]);
 });
 
-ava("decodeEntities", t => {
+ava("decodeEntities", (t) => {
   t.is(
     decodeEntities("&&amp;&lt;&gt;&quot;&apos;&gt;&#167;&#xd842;&#xDFB7;;"),
     "&&<>\"'>§𠮷;"
   );
 });
 
-ava("parse", t => {
+ava("parse", (t) => {
   const xml =
     '<?xml version="1.0"?>\n<a-b:c><d f="1<g>"/><!-- comment --><h >i</h></a-b:c>';
   const parsed = parseXml(xml);
@@ -60,7 +60,7 @@ ava("parse", t => {
             attrs: 'f="1<g>"',
             text: "",
             bodyIndex: 42,
-            children: []
+            children: [],
           },
           {
             name: "h",
@@ -69,10 +69,10 @@ ava("parse", t => {
             attrs: "",
             text: "i",
             bodyIndex: 62,
-            children: []
-          }
-        ]
-      }
-    ]
+            children: [],
+          },
+        ],
+      },
+    ],
   });
 });

tools/dump-data-model.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -60,7 +58,7 @@ connect()
   .then(() => {
     devicesCollection.findOne(
       {
-        _id: process.argv[2]
+        _id: process.argv[2],
       },
       {},
       (err, device) => {
@@ -79,6 +77,6 @@ connect()
       }
     );
   })
-  .catch(err => {
+  .catch((err) => {
     throw err;
   });

tsconfig.json

@@ -14,11 +14,11 @@
     "allowJs": true
   },
   "include": [
-      "./bin/.*.ts",
+      "./bin/*.ts",
       "./lib/**/*",
       "./ui/**/*",
       "./test/**/*",
       "./build/**/*",
-      "./tools/.*.ts"
+      "./tools/*.ts"
   ]
 }

ui/admin-menu.ts

@@ -22,7 +22,7 @@ import { ClosureComponent, Component } from "mithril";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const active = { [vnode.attrs["page"]]: "active" };
       const tabs = [];
 
@@ -97,7 +97,7 @@ const component: ClosureComponent = (): Component => {
       }
 
       return m("nav#side-menu", m("ul", tabs));
-    }
+    },
   };
 };
 

ui/app.ts

@@ -37,13 +37,14 @@ import * as usersPage from "./users-page";
 import Authorizer from "../lib//common/authorizer";
 import * as notifications from "./notifications";
 import { contextifyComponent } from "./components";
+import { PermissionSet, UiConfig } from "../lib/types";
 
 declare global {
   interface Window {
     authorizer: Authorizer;
-    permissionSets: any;
+    permissionSets: PermissionSet[];
     username: string;
-    clientConfig: any;
+    clientConfig: { ui: UiConfig };
     configSnapshot: string;
     genieacsVersion: string;
   }
@@ -58,7 +59,7 @@ const adminPages = [
   "files",
   "config",
   "users",
-  "permissions"
+  "permissions",
 ];
 
 let state;
@@ -70,7 +71,7 @@ let fulfillTimeout = null;
 function fulfill(): void {
   clearTimeout(fulfillTimeout);
   fulfillTimeout = setTimeout(() => {
-    store.fulfill(lastRenderTimestamp, pageVisitTimestamp).then(updated => {
+    store.fulfill(lastRenderTimestamp, pageVisitTimestamp, (updated) => {
       if (updated) m.redraw();
     });
   }, 100);
@@ -88,7 +89,7 @@ function pagify(pageName, page): RouteResolver {
       attrs["page"] = pageName;
       return m(layout, attrs, p);
     },
-    onmatch: null
+    onmatch: null,
   };
 
   component.onmatch = (args, requestedPath) => {
@@ -99,16 +100,16 @@ function pagify(pageName, page): RouteResolver {
       return null;
     }
 
-    return new Promise(resolve => {
+    return new Promise((resolve) => {
       page
         .init(args)
-        .then(st => {
+        .then((st) => {
           if (!st) return void m.route.set("/");
           state = st;
           resolve();
           fulfill();
         })
-        .catch(err => {
+        .catch((err) => {
           if (!window.username && err.message.indexOf("authorized") >= 0) {
             notifications.push("error", err.message);
             m.route.set("/login", { continue: requestedPath });
@@ -132,7 +133,7 @@ function redirectAdminPage(): RouteResolver {
         }
       }
       return null;
-    }
+    },
   };
   return component;
 }
@@ -154,5 +155,5 @@ m.route(document.body, "/overview", {
   "/admin/files": pagify("files", filesPage),
   "/admin/config": pagify("config", configPage),
   "/admin/users": pagify("users", usersPage),
-  "/admin/permissions": pagify("permissions", permissionsPage)
+  "/admin/permissions": pagify("permissions", permissionsPage),
 });

ui/autocomplete-compnent.ts

@@ -17,11 +17,13 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+type AutocompleteCallback = (
+  value: string,
+  callback: (suggestions: { value: string; tip?: string }[]) => void
+) => void;
+
 export default class Autocomplete {
-  private callback: (
-    value: string,
-    callback: (suggestions: { value: string; tip?: string }[]) => void
-  ) => void;
+  private callback: AutocompleteCallback;
   private element: HTMLInputElement;
   private hideTimeout: NodeJS.Timeout;
   private visible: boolean;
@@ -29,7 +31,7 @@ export default class Autocomplete {
   private selection: number;
   private container: HTMLElement;
 
-  public constructor(className, callback) {
+  public constructor(className: string, callback: AutocompleteCallback) {
     this.callback = callback;
     this.element = null;
     this.hideTimeout = null;
@@ -44,7 +46,7 @@ export default class Autocomplete {
     this.container.className = className;
   }
 
-  public attach(el): void {
+  public attach(el: HTMLInputElement): void {
     el.setAttribute("autocomplete", "off");
 
     el.addEventListener("focus", () => {
@@ -62,7 +64,7 @@ export default class Autocomplete {
       this.hide();
     });
 
-    el.addEventListener("keydown", e => {
+    el.addEventListener("keydown", (e) => {
       if (this.element !== el) return;
       if (e.key === "Escape") {
         if (this.visible) this.hide();
@@ -108,7 +110,7 @@ export default class Autocomplete {
   private update(): void {
     const el = this.element;
 
-    this.callback(el.value, suggestions => {
+    this.callback(el.value, (suggestions) => {
       if (this.element !== el) return;
       this.default = null;
 
@@ -154,7 +156,7 @@ export default class Autocomplete {
 
         const t = document.createTextNode(suggestion.value);
         e.appendChild(t);
-        e.addEventListener("mousedown", ev => {
+        e.addEventListener("mousedown", (ev) => {
           ev.preventDefault();
           el.value = suggestion.value;
           if (this.element === el) this.update();

ui/change-password-component.ts

@@ -24,7 +24,7 @@ import { m } from "./components";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const onPasswordChange = vnode.attrs["onPasswordChange"];
       const enforceAuth = !vnode.attrs["noAuth"];
       const username = vnode.attrs["username"];
@@ -40,19 +40,19 @@ const component: ClosureComponent = (): Component => {
             type: "text",
             value: vnode.state["username"],
             disabled: !!username,
-            oninput: e => {
+            oninput: (e) => {
               vnode.state["username"] = e.target.value;
             },
-            oncreate: _vnode => {
+            oncreate: (_vnode) => {
               (_vnode.dom as HTMLSelectElement).focus();
-            }
+            },
           })
-        )
+        ),
       ];
 
       let fields = {
         newPassword: "New password",
-        confirmPassword: "Confirm password"
+        confirmPassword: "Confirm password",
       };
       if (enforceAuth)
         fields = Object.assign({ authPassword: "Your password" }, fields);
@@ -66,9 +66,9 @@ const component: ClosureComponent = (): Component => {
               name: f,
               type: "password",
               value: vnode.state[f],
-              oninput: e => {
+              oninput: (e) => {
                 vnode.state[f] = e.target.value;
-              }
+              },
             })
           )
         );
@@ -77,7 +77,7 @@ const component: ClosureComponent = (): Component => {
       const submit = m(
         "button.primary",
         {
-          type: "submit"
+          type: "submit",
         },
         "Change password"
       ) as VnodeDOM;
@@ -89,7 +89,7 @@ const component: ClosureComponent = (): Component => {
         m(
           "form",
           {
-            onsubmit: e => {
+            onsubmit: (e) => {
               e.redraw = false;
               e.preventDefault();
               if (
@@ -120,19 +120,19 @@ const component: ClosureComponent = (): Component => {
                     if (onPasswordChange) onPasswordChange();
                     (submit.dom as HTMLFormElement).disabled = false;
                   })
-                  .catch(err => {
+                  .catch((err) => {
                     notifications.push("error", err.message);
                     (submit.dom as HTMLFormElement).disabled = false;
                   });
               }
-            }
+            },
           },
           form
-        )
+        ),
       ];
 
       return m("div.put-form", children);
-    }
+    },
   };
 };
 

ui/code-editor-component.ts

@@ -23,11 +23,11 @@ import { codeMirror } from "./dynamic-loader";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       return m("textarea", {
         name: vnode.attrs["id"],
         value: vnode.attrs["value"],
-        oncreate: _vnode => {
+        oncreate: (_vnode) => {
           const editor = codeMirror.fromTextArea(_vnode.dom, {
             mode: vnode.attrs["mode"],
             lineNumbers: true,
@@ -40,21 +40,21 @@ const component: ClosureComponent = (): Component => {
               "Cmd-Enter": () => {
                 if (vnode.attrs["onSubmit"])
                   vnode.attrs["onSubmit"](_vnode.dom);
-              }
-            }
+              },
+            },
           });
 
           if (vnode.attrs["onChange"]) {
-            editor.on("change", e => {
+            editor.on("change", (e) => {
               vnode.attrs["onChange"](e.getValue());
             });
           }
 
           if (vnode.attrs["focus"]) editor.focus();
           if (vnode.attrs["onReady"]) vnode.attrs["onReady"](editor);
-        }
+        },
       });
-    }
+    },
   };
 };
 

ui/components.ts

@@ -24,7 +24,7 @@ import m, {
   ComponentTypes,
   Lifecycle,
   ClosureComponent,
-  Vnode
+  Vnode,
 } from "mithril";
 import parameter from "./components/parameter";
 import parameterList from "./components/parameter-list";
@@ -39,6 +39,7 @@ import tags from "./components/tags";
 import ping from "./components/ping";
 import deviceLink from "./components/device-link";
 import longTextComponent from "./long-text-component";
+import deviceUploads from "./components/device-uploads";
 
 const comps = {
   parameter,
@@ -53,7 +54,8 @@ const comps = {
   tags,
   ping,
   "device-link": deviceLink,
-  "long-text": longTextComponent
+  "long-text": longTextComponent,
+  "device-uploads": deviceUploads,
 };
 
 const contextifiedComponents = new WeakMap<ComponentTypes, ComponentTypes>();
@@ -96,7 +98,7 @@ const M = new Proxy(m, {
   get: (target, prop) => {
     if (prop === "context") return contextFn;
     else return Reflect.get(target, prop);
-  }
+  },
 }) as MC;
 
 function contextFn(context, ...argumentsList): Vnode {
@@ -125,17 +127,17 @@ export function contextifyComponent(component: ComponentTypes): ComponentTypes {
     if (typeof component !== "function") {
       c = Object.assign({}, component);
       const view = component.view;
-      c.view = function(vnode) {
+      c.view = function (vnode) {
         const context = vnodeContext.get(vnode) || {};
         const res = Reflect.apply(view, this, [vnode]);
         applyContext(res, context);
         return res;
       };
     } else if (!component.prototype || !component.prototype.view) {
-      c = initialNode => {
+      c = (initialNode) => {
         const state = (component as ClosureComponent)(initialNode);
         const view = state.view;
-        state.view = function(vnode) {
+        state.view = function (vnode) {
           const context = vnodeContext.get(vnode) || {};
           const res = Reflect.apply(view, this, [vnode]);
           applyContext(res, context);

ui/components/all-parameters.ts

@@ -30,128 +30,168 @@ function escapeRegExp(str): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
 
+const keysByDepth: WeakMap<Record<string, unknown>, string[][]> = new WeakMap();
+
+function orderKeysByDepth(device: Record<string, unknown>): string[][] {
+  if (keysByDepth.has(device)) return keysByDepth.get(device);
+  const res: string[][] = [];
+  for (const key of Object.keys(device)) {
+    let count = 0;
+    for (
+      let i = key.lastIndexOf(".", key.length - 2);
+      i >= 0;
+      i = key.lastIndexOf(".", i - 1)
+    )
+      ++count;
+    while (res.length <= count) res.push([]);
+    res[count].push(key);
+  }
+  keysByDepth.set(device, res);
+  return res;
+}
+
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
 
       const search = m("input", {
         type: "text",
         placeholder: "Search parameters",
-        oninput: e => {
+        oninput: (e) => {
           vnode.state["searchString"] = e.target.value;
           e.redraw = false;
           clearTimeout(vnode.state["timeout"]);
           vnode.state["timeout"] = setTimeout(m.redraw, 500);
-        }
+        },
       });
 
       const instanceRegex = /\.[0-9]+$/;
       let re;
       if (vnode.state["searchString"]) {
-        const keywords = vnode.state["searchString"].split(" ").filter(s => s);
+        const keywords = vnode.state["searchString"]
+          .split(" ")
+          .filter((s) => s);
         if (keywords.length)
-          re = new RegExp(keywords.map(s => escapeRegExp(s)).join(".*"), "i");
+          re = new RegExp(keywords.map((s) => escapeRegExp(s)).join(".*"), "i");
       }
 
-      const rows = Object.keys(device)
-        .sort()
-        .map(k => {
+      const filteredKeys: string[] = [];
+      const allKeys = orderKeysByDepth(device);
+      let count = 0;
+      for (const keys of allKeys) {
+        let c = 0;
+        for (const k of keys) {
           const p = device[k];
-          const val = [];
-          const attrs = {};
-          if (re) {
-            const str = p.value && p.value[0] ? `${k} ${p.value[0]}` : k;
-            if (!re.test(str)) attrs["style"] = "display: none;";
-          }
+          const str = p.value && p.value[0] ? `${k} ${p.value[0]}` : k;
+          if (re && !re.test(str)) continue;
+          ++c;
+          if (count < 100) filteredKeys.push(k);
+        }
+        count += c;
+      }
 
-          if (p.object === false) {
-            val.push(
-              m(
-                "parameter",
-                Object.assign({ device: device, parameter: memoizedParse(k) })
-              )
-            );
-          } else if (p.object && p.writable) {
-            if (instanceRegex.test(k)) {
-              val.push(
-                m(
-                  "button",
-                  {
-                    title: "Delete this instance",
-                    onclick: () => {
-                      taskQueue.queueTask({
-                        name: "deleteObject",
-                        device: device["DeviceID.ID"].value[0],
-                        objectName: k
-                      });
-                    }
-                  },
-                  getIcon("delete-instance")
-                )
-              );
-            } else {
-              val.push(
-                m(
-                  "button",
-                  {
-                    title: "Create a new instance",
-                    onclick: () => {
-                      taskQueue.queueTask({
-                        name: "addObject",
-                        device: device["DeviceID.ID"].value[0],
-                        objectName: k
-                      });
-                    }
-                  },
-                  getIcon("add-instance")
-                )
-              );
-            }
-          }
+      filteredKeys.sort();
 
+      const rows = filteredKeys.map((k) => {
+        const p = device[k];
+        const val = [];
+        const attrs = { key: k };
+
+        if (p.object === false) {
           val.push(
             m(
-              "button",
-              {
-                title: "Refresh tree",
-                onclick: () => {
-                  taskQueue.queueTask({
-                    name: "getParameterValues",
-                    device: device["DeviceID.ID"].value[0],
-                    parameterNames: [k]
-                  });
-                }
-              },
-              getIcon("refresh")
+              "parameter",
+              Object.assign({ device: device, parameter: memoizedParse(k) })
             )
           );
+        } else if (p.object && p.writable) {
+          if (instanceRegex.test(k)) {
+            val.push(
+              m(
+                "button",
+                {
+                  title: "Delete this instance",
+                  onclick: () => {
+                    taskQueue.queueTask({
+                      name: "deleteObject",
+                      device: device["DeviceID.ID"].value[0],
+                      objectName: k,
+                    });
+                  },
+                },
+                getIcon("delete-instance")
+              )
+            );
+          } else {
+            val.push(
+              m(
+                "button",
+                {
+                  title: "Create a new instance",
+                  onclick: () => {
+                    taskQueue.queueTask({
+                      name: "addObject",
+                      device: device["DeviceID.ID"].value[0],
+                      objectName: k,
+                    });
+                  },
+                },
+                getIcon("add-instance")
+              )
+            );
+          }
+        }
 
-          return m(
-            "tr",
-            attrs,
-            m("td.left", m("long-text", { text: k })),
-            m("td.right", val)
-          );
-        });
+        val.push(
+          m(
+            "button",
+            {
+              title: "Refresh tree",
+              onclick: () => {
+                taskQueue.queueTask({
+                  name: "getParameterValues",
+                  device: device["DeviceID.ID"].value[0],
+                  parameterNames: [k],
+                });
+              },
+            },
+            getIcon("refresh")
+          )
+        );
+
+        return m(
+          "tr",
+          attrs,
+          m("td.left", m("long-text", { text: k })),
+          m("td.right", val)
+        );
+      });
 
       return m(
         ".all-parameters",
         m(
           "a.download-csv",
           {
-            href: `/api/devices/${encodeURIComponent(
+            href: `api/devices/${encodeURIComponent(
               device["DeviceID.ID"].value[0]
             )}.csv`,
             download: "",
-            style: "float: right;"
+            style: "float: right;",
           },
           "Download"
         ),
         search,
-        m(".parameter-list", m("table", m("tbody", rows)))
+        m(
+          ".parameter-list",
+          m("table", m("tbody", rows)),
+          m(
+            "m",
+            `Displaying ${filteredKeys.length} out of ${count} parameters.`
+          )
+        )
       );
-    }
+    },
   };
 };
 

ui/components/container.ts

@@ -23,7 +23,7 @@ import * as store from "../store";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       if (vnode.attrs["filter"]) {
         if (
           !store.evaluateExpression(
@@ -34,13 +34,13 @@ const component: ClosureComponent = (): Component => {
           return null;
       }
 
-      const children = Object.values(vnode.attrs["components"]).map(c => {
+      const children = Object.values(vnode.attrs["components"]).map((c) => {
         if (typeof c !== "object") return `${c}`;
         return m(c["type"], c);
       });
       if (vnode.attrs["element"]) return m(vnode.attrs["element"], children);
       else return children;
-    }
+    },
   };
 };
 

ui/components/device-actions.ts

@@ -22,10 +22,11 @@ import { m } from "../components";
 import * as taskQueue from "../task-queue";
 import * as notifications from "../notifications";
 import * as store from "../store";
+import * as overlay from "../overlay";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
 
       const buttons = [];
@@ -38,9 +39,9 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.queueTask({
                 name: "reboot",
-                device: device["DeviceID.ID"].value[0]
+                device: device["DeviceID.ID"].value[0],
               });
-            }
+            },
           },
           "Reboot"
         )
@@ -54,9 +55,9 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.queueTask({
                 name: "factoryReset",
-                device: device["DeviceID.ID"].value[0]
+                device: device["DeviceID.ID"].value[0],
               });
-            }
+            },
           },
           "Reset"
         )
@@ -70,14 +71,38 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageDownload({
                 name: "download",
-                device: device["DeviceID.ID"].value[0]
+                devices: [device["DeviceID.ID"].value[0]],
               });
-            }
+            },
           },
           "Push file"
         )
       );
 
+      buttons.push(
+        m(
+          "button.primary",
+          {
+            title: "Fetch files from device",
+            onclick: () => {
+              const deviceId = device["DeviceID.ID"].value[0];
+              overlay.open(() => {
+                const dev = store.fetch("devices", [
+                  "=",
+                  ["PARAM", "DeviceID.ID"],
+                  deviceId,
+                ]);
+                return [
+                  m("h3", `Uploads from ${deviceId}`),
+                  m("device-uploads", { device: dev.value[0] }),
+                ];
+              });
+            },
+          },
+          "Uploads"
+        )
+      );
+
       buttons.push(
         m(
           "button.primary",
@@ -93,17 +118,17 @@ const component: ClosureComponent = (): Component => {
                   notifications.push("success", `${deviceId}: Device deleted`);
                   m.route.set("/devices");
                 })
-                .catch(err => {
+                .catch((err) => {
                   notifications.push("error", err.message);
                 });
-            }
+            },
           },
           "Delete"
         )
       );
 
       return m(".actions-bar", buttons);
-    }
+    },
   };
 };
 

ui/components/device-faults.ts

@@ -25,13 +25,13 @@ import { getIcon } from "../icons";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
       const deviceId = device["DeviceID.ID"].value[0];
       const faults = store.fetch("faults", [
         "AND",
         [">", ["PARAM", "_id"], `${deviceId}:`],
-        ["<", ["PARAM", "_id"], `${deviceId}:zzzz`]
+        ["<", ["PARAM", "_id"], `${deviceId}:zzzz`],
       ]);
 
       const headers = [
@@ -39,8 +39,8 @@ const component: ClosureComponent = (): Component => {
         "Code",
         "Message",
         "Retries",
-        "Timestamp"
-      ].map(l => m("th", l));
+        "Timestamp",
+      ].map((l) => m("th", l));
       const thead = m("thead", m("tr", headers));
 
       const rows = [];
@@ -57,7 +57,7 @@ const component: ClosureComponent = (): Component => {
               "button",
               {
                 title: "Delete fault",
-                onclick: e => {
+                onclick: (e) => {
                   e.redraw = false;
                   store
                     .deleteResource("faults", f["_id"])
@@ -66,21 +66,24 @@ const component: ClosureComponent = (): Component => {
                       store.fulfill(Date.now(), Date.now());
                       m.redraw();
                     })
-                    .catch(err => {
+                    .catch((err) => {
                       notifications.push("error", err.message);
                       store.fulfill(Date.now(), Date.now());
                     });
-                }
+                },
               },
               getIcon("remove")
             )
-          )
+          ),
         ]);
       }
 
       let tbody;
       if (rows.length) {
-        tbody = m("tbody", rows.map(r => m("tr", r)));
+        tbody = m(
+          "tbody",
+          rows.map((r) => m("tr", r))
+        );
       } else {
         tbody = m(
           "tbody",
@@ -89,7 +92,7 @@ const component: ClosureComponent = (): Component => {
       }
 
       return m("table.table", thead, tbody);
-    }
+    },
   };
 };
 

ui/components/device-link.ts

@@ -22,12 +22,12 @@ import { m } from "../components";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       let deviceId;
       if (vnode.attrs["device"])
         deviceId = vnode.attrs["device"]["DeviceID.ID"].value[0];
 
-      const children = Object.values(vnode.attrs["components"]).map(c => {
+      const children = Object.values(vnode.attrs["components"]).map((c) => {
         if (typeof c !== "object") return `${c}`;
         const attrs = Object.assign({}, vnode.attrs, c);
         return m(attrs["type"], attrs);
@@ -41,7 +41,7 @@ const component: ClosureComponent = (): Component => {
       } else {
         return children;
       }
-    }
+    },
   };
 };
 

ui/components/device-uploads.ts

@@ -0,0 +1,155 @@
+/**
+ * Copyright 2013-2020  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { ClosureComponent, Component } from "mithril";
+import { m } from "../components";
+import * as store from "../store";
+import * as notifications from "../notifications";
+import { getIcon } from "../icons";
+import * as taskQueue from "../task-queue";
+
+const component: ClosureComponent = (): Component => {
+  return {
+    view: (vnode) => {
+      const device = vnode.attrs["device"];
+      const deviceId = device["DeviceID.ID"].value[0];
+      const files = store.fetch("uploads", [
+        "AND",
+        [">", ["PARAM", "_id"], `${deviceId}/`],
+        ["<", ["PARAM", "_id"], `${deviceId}/zzzz`],
+      ]);
+
+      const uploads: Record<string, any> = {};
+
+      for (const [k, v] of Object.entries(device)) {
+        if (k.startsWith("Uploads.")) {
+          const parts = k.split(".");
+          if (parts.length !== 3) continue;
+          uploads[parts[1]] = uploads[parts[1]] || {};
+          uploads[parts[1]][parts[2]] = v["value"][0];
+        }
+      }
+
+      const headers = ["Filename", "Type", "Timestamp", "Status"].map((l) =>
+        m("th", l)
+      );
+      const thead = m("thead", m("tr", headers));
+
+      let minTime = 999999;
+      const timestamp = store.getTimestamp();
+      const rows = [];
+      const uploadsSorted = Object.values(uploads).sort(
+        (a, b) => a["Upload"] - b["Upload"]
+      );
+
+      for (const u of Object.values(uploadsSorted)) {
+        if (!u["Upload"]) continue;
+        const ready = u["LastUpload"] >= u["Upload"];
+        if (!ready) {
+          minTime = Math.min(minTime, u["Upload"] - timestamp);
+          rows.push([
+            m("td", u["FileName"]),
+            m("td", u["FileType"]),
+            m("td", new Date(u["Upload"]).toLocaleString()),
+            m("td", "Waiting for upload"),
+          ]);
+        } else {
+          const filePath = `${deviceId}/${u["FileName"]}`;
+          if (!Array.isArray(files.value)) continue;
+          if (!files.value.some((f) => f["_id"] === filePath)) continue;
+
+          const deleteButton = m(
+            "button",
+            {
+              title: "Delete file",
+              onclick: (e) => {
+                e.redraw = false;
+                store
+                  .deleteResource("uploads", filePath)
+                  .then(() => {
+                    notifications.push("success", "File deleted");
+                    store.fulfill(Date.now(), Date.now());
+                    m.redraw();
+                  })
+                  .catch((err) => {
+                    notifications.push("error", err.message);
+                    store.fulfill(Date.now(), Date.now());
+                  });
+              },
+            },
+            getIcon("delete-instance")
+          );
+
+          rows.push([
+            m("td", u["FileName"]),
+            m("td", u["FileType"]),
+            m("td", new Date(u["LastUpload"]).toLocaleString()),
+            m(
+              "td",
+              m(
+                "a",
+                {
+                  href: `api/uploads/blob/${encodeURIComponent(filePath)}`,
+                },
+                "Ready"
+              )
+            ),
+            m("td", deleteButton),
+          ]);
+        }
+      }
+
+      if (minTime < 60000) {
+        setTimeout(() => {
+          store.fulfill(0, timestamp + 5000);
+        }, 10000);
+      }
+
+      const addButton = m(
+        "button",
+        {
+          title: "Fetch a new file",
+          onclick: () => {
+            taskQueue.stageUpload({
+              name: "upload",
+              devices: [deviceId],
+            });
+          },
+        },
+        getIcon("add-instance")
+      );
+
+      const empty = !rows.length;
+
+      rows.push([m("td", { colspan: headers.length }), m("td", addButton)]);
+
+      const tbody = m(
+        "tbody",
+        rows.map((r) => m("tr", r)),
+        empty
+          ? m("tr.empty", m("td", { colspan: headers.length }, "No uploads"))
+          : null
+      );
+
+      return m("table.table", thead, tbody);
+    },
+  };
+};
+
+export default component;

ui/components/overview-dot.ts

@@ -26,9 +26,9 @@ const CHARTS = config.ui.overview.charts;
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
-      const chart = CHARTS[vnode.attrs["chart"]];
+      const chart = CHARTS[vnode.attrs["chart"]] as Record<string, unknown>;
       if (!chart) return null;
       for (const slice of Object.values(chart.slices)) {
         const filter = slice["filter"];
@@ -39,20 +39,20 @@ const component: ClosureComponent = (): Component => {
               width: "1em",
               height: "1em",
               xmlns: "http://www.w3.org/2000/svg",
-              "xmlns:xlink": "http://www.w3.org/1999/xlink"
+              "xmlns:xlink": "http://www.w3.org/1999/xlink",
             },
             m("circle", {
               cx: "0.5em",
               cy: "0.5em",
               r: "0.4em",
-              fill: slice["color"]
+              fill: slice["color"],
             })
           );
           return m("span.overview-dot", dot, `${slice["label"]}`);
         }
       }
       return null;
-    }
+    },
   };
 };
 

ui/components/parameter-list.ts

@@ -22,14 +22,14 @@ import { m } from "../components";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
 
-      const rows = Object.values(vnode.attrs["parameters"]).map(parameter => {
+      const rows = Object.values(vnode.attrs["parameters"]).map((parameter) => {
         const p = m.context(
           {
             device: device,
-            parameter: parameter["parameter"]
+            parameter: parameter["parameter"],
           },
           parameter["type"] || "parameter",
           parameter
@@ -38,11 +38,11 @@ const component: ClosureComponent = (): Component => {
         return m(
           "tr",
           {
-            onupdate: vn => {
+            onupdate: (vn) => {
               (vn.dom as HTMLElement).style.display = (p as VnodeDOM).dom
                 ? ""
                 : "none";
-            }
+            },
           },
           m("th", parameter["label"]),
           m("td", p)
@@ -50,7 +50,7 @@ const component: ClosureComponent = (): Component => {
       });
 
       return m("table.parameter-list", rows);
-    }
+    },
   };
 };
 

ui/components/parameter-table.ts

@@ -26,21 +26,21 @@ import { getIcon } from "../icons";
 
 const component: ClosureComponent = (): Component => {
   return {
-    oninit: vnode => {
+    oninit: (vnode) => {
       const obj = vnode.attrs["parameter"];
       if (!Array.isArray(obj) || obj[0] !== "PARAM")
         throw new Error("Object must be a parameter path");
       vnode.state["object"] = obj[1];
       vnode.state["parameters"] = Object.values(vnode.attrs["childParameters"]);
     },
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
       const object = store.evaluateExpression(vnode.state["object"], device);
       const parameters = vnode.state["parameters"];
 
-      if (!device[object]) return null;
+      if (typeof object !== "string" || !device[object]) return null;
 
-      const instances = new Set();
+      const instances: Set<string> = new Set();
       const prefix = `${object}.`;
       for (const p in device) {
         if (p.startsWith(prefix)) {
@@ -50,7 +50,7 @@ const component: ClosureComponent = (): Component => {
         }
       }
 
-      const headers = Object.values(parameters).map(p => m("th", p["label"]));
+      const headers = Object.values(parameters).map((p) => m("th", p["label"]));
 
       const thead = m("thead", m("tr", headers));
 
@@ -59,7 +59,7 @@ const component: ClosureComponent = (): Component => {
         let filter =
           vnode.attrs["filter"] != null ? vnode.attrs["filter"] : true;
 
-        filter = expressionParser.map(filter, e => {
+        filter = expressionParser.map(filter, (e) => {
           if (Array.isArray(e) && e[0] === "PARAM")
             return ["PARAM", ["||", i, ".", e[1]]];
           return e;
@@ -67,8 +67,8 @@ const component: ClosureComponent = (): Component => {
 
         if (!store.evaluateExpression(filter, device)) continue;
 
-        const row = parameters.map(p => {
-          const param = expressionParser.map(p.parameter, e => {
+        const row = parameters.map((p) => {
+          const param = expressionParser.map(p.parameter, (e) => {
             if (Array.isArray(e) && e[0] === "PARAM")
               return ["PARAM", ["||", i, ".", e[1]]];
             return e;
@@ -78,13 +78,13 @@ const component: ClosureComponent = (): Component => {
             m.context(
               {
                 device: device,
-                parameter: param
+                parameter: param,
               },
               p.type || "parameter",
               Object.assign({}, p, {
                 device: device,
                 parameter: param,
-                label: null
+                label: null,
               })
             )
           );
@@ -102,9 +102,9 @@ const component: ClosureComponent = (): Component => {
                     taskQueue.queueTask({
                       name: "deleteObject",
                       device: device["DeviceID.ID"].value[0],
-                      objectName: i
+                      objectName: i,
                     });
-                  }
+                  },
                 },
                 getIcon("delete-instance")
               )
@@ -135,9 +135,9 @@ const component: ClosureComponent = (): Component => {
                     taskQueue.queueTask({
                       name: "addObject",
                       device: device["DeviceID.ID"].value[0],
-                      objectName: object
+                      objectName: object,
                     });
-                  }
+                  },
                 },
                 getIcon("add-instance")
               )
@@ -150,7 +150,7 @@ const component: ClosureComponent = (): Component => {
       if (vnode.attrs["label"]) label = m("h2", vnode.attrs["label"]);
 
       return [label, m("table.table", thead, m("tbody", rows))];
-    }
+    },
   };
 };
 

ui/components/parameter.ts

@@ -28,31 +28,49 @@ import { getIcon } from "../icons";
 
 const evaluateParam = memoize((exp, obj, now: number) => {
   let timestamp = now;
-  const params = new Set();
-  const value = expression.evaluate(exp, obj, now, e => {
-    if (Array.isArray(e)) {
-      if (e[0] === "PARAM") {
-        params.add(e[1]);
-        if (!Array.isArray(e[1])) {
-          const p = obj[e[1]];
-          if (p && p.valueTimestamp)
-            timestamp = Math.min(timestamp, p.valueTimestamp);
+  exp = expression.evaluate(exp, null, now, (e) => {
+    if (!Array.isArray(e)) return e;
+    for (let i = 1; i < e.length; ++i) {
+      if (
+        Array.isArray(e[i]) &&
+        e[i][0] === "PARAM" &&
+        !Array.isArray(e[i][1])
+      ) {
+        let v = null;
+        const p = obj[e[i][1]];
+        if (p && p.value) {
+          v = p.value[0];
+          timestamp = Math.min(timestamp, p.valueTimestamp);
         }
+        e = e.slice();
+        e[i] = v;
       }
-      if (e[0] === "FUNC" && e[1] === "DATE_STRING" && !Array.isArray(e[2]))
-        return new Date(e[2]).toLocaleString();
     }
+    if (e[0] === "FUNC" && e[1] === "DATE_STRING" && !Array.isArray(e[2]))
+      return new Date(e[2]).toLocaleString();
     return e;
   });
 
-  let parameter = params.size === 1 ? params.values().next().value : null;
-  if (Array.isArray(parameter)) parameter = null;
+  let parameter = null;
+  let value = null;
+
+  if (!Array.isArray(exp)) {
+    value = exp;
+  } else if (exp[0] === "PARAM") {
+    const p = obj[exp[1]];
+    if (p && p.value) {
+      timestamp = p.valueTimestamp;
+      value = p.value[0];
+      parameter = exp[1];
+    }
+  }
+
   return { value, timestamp, parameter };
 });
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
 
       const { value, timestamp, parameter } = evaluateParam(
@@ -72,16 +90,16 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageSpv({
                 name: "setParameterValues",
-                device: device["DeviceID.ID"].value[0],
+                devices: [device["DeviceID.ID"].value[0]],
                 parameterValues: [
                   [
                     parameter,
                     device[parameter].value[0],
-                    device[parameter].value[1]
-                  ]
-                ]
+                    device[parameter].value[1],
+                  ],
+                ],
               });
-            }
+            },
           },
           getIcon("edit")
         );
@@ -93,7 +111,7 @@ const component: ClosureComponent = (): Component => {
         "span",
         {
           class: "parameter-value",
-          onmouseover: e => {
+          onmouseover: (e) => {
             e.redraw = false;
             // Don't update any child element
             if (e.target === (el as VnodeDOM).dom) {
@@ -101,12 +119,12 @@ const component: ClosureComponent = (): Component => {
               const localeString = new Date(timestamp).toLocaleString();
               e.target.title = `${localeString} (${timeAgo(now - timestamp)})`;
             }
-          }
+          },
         },
         el,
         edit
       );
-    }
+    },
   };
 };
 

ui/components/ping.ts

@@ -27,30 +27,35 @@ const component: ClosureComponent = (vn): Component => {
   let interval: ReturnType<typeof setInterval>;
   let host: string;
 
-  const refresh = async (): Promise<void> => {
-    let status = "";
-    if (host) {
-      try {
-        const res = await store.ping(host);
-        if (res["avg"] != null) status = `${Math.trunc(res["avg"])} ms`;
-        else status = "Unreachable";
-      } catch (err) {
-        setTimeout(() => {
-          throw err;
-        }, 0);
-      }
+  const refresh = (): void => {
+    if (!host) {
+      const dom = (vn as VnodeDOM).dom;
+      if (dom) dom.innerHTML = "";
+      return;
     }
 
-    const dom = (vn as VnodeDOM).dom;
-
-    if (dom) dom.innerHTML = `Pinging ${host}: ${status}`;
+    let status = "";
+    store
+      .ping(host)
+      .then((res) => {
+        if (res["avg"] != null) status = `${Math.trunc(res["avg"])} ms`;
+        else status = "Unreachable";
+      })
+      .catch(() => {
+        status = "Error!";
+        clearInterval(interval);
+      })
+      .finally(() => {
+        const dom = (vn as VnodeDOM).dom;
+        if (dom) dom.innerHTML = `Pinging ${host}: ${status}`;
+      });
   };
 
   return {
     onremove: () => {
       clearInterval(interval);
     },
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
       let param =
         device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"];
@@ -73,7 +78,7 @@ const component: ClosureComponent = (vn): Component => {
       }
 
       return m("div", `Pinging ${host}:`);
-    }
+    },
   };
 };
 

ui/components/summon-button.ts

@@ -25,26 +25,26 @@ import * as notifications from "../notifications";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
 
       return m(
         "button.primary",
         {
           title: "Initiate session and refresh basic parameters",
-          onclick: e => {
+          onclick: (e) => {
             e.target.disabled = true;
             const params = Object.values(vnode.attrs["parameters"])
-              .map(exp => {
+              .map((exp) => {
                 if (!Array.isArray(exp) || exp[0] !== "PARAM") return null;
-                return store.evaluateExpression(exp[1], device);
+                return store.evaluateExpression(exp[1], device) as string;
               })
-              .filter(exp => !!exp);
+              .filter((exp) => !!exp);
 
             const task = {
               name: "getParameterValues",
               parameterNames: params,
-              device: device["DeviceID.ID"].value[0]
+              device: device["DeviceID.ID"].value[0],
             };
 
             taskQueue
@@ -79,12 +79,16 @@ const component: ClosureComponent = (): Component => {
               .then(() => {
                 e.target.disabled = false;
                 store.fulfill(0, Date.now());
+              })
+              .catch((err) => {
+                e.target.disabled = false;
+                notifications.push("error", err.message);
               });
-          }
+          },
         },
         "Summon"
       );
-    }
+    },
   };
 };
 

ui/components/tags.ts

@@ -22,32 +22,38 @@ import { m } from "../components";
 import * as notifications from "../notifications";
 import * as store from "../store";
 import { getIcon } from "../icons";
+import { decodeTag } from "../../lib/common";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
       let writable = true;
       if (vnode.attrs["writable"] != null) writable = vnode.attrs["writable"];
 
       const tags = [];
       for (const p of Object.keys(device))
-        if (p.startsWith("Tags.")) tags.push(p.slice(5));
+        if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
 
       tags.sort();
 
-      if (!writable) return m(".tags", tags.map(t => m("span.tag", t)));
+      if (!writable) {
+        return m(
+          ".tags",
+          tags.map((t) => m("span.tag", t))
+        );
+      }
 
       return m(
         ".tags",
-        tags.map(tag =>
+        tags.map((tag) =>
           m(
             "span.tag",
             tag,
             m(
               "button",
               {
-                onclick: e => {
+                onclick: (e) => {
                   e.target.disabled = true;
                   const deviceId = device["DeviceID.ID"].value[0];
                   store
@@ -60,14 +66,14 @@ const component: ClosureComponent = (): Component => {
                       );
                       store.fulfill(0, Date.now());
                     })
-                    .catch(err => {
+                    .catch((err) => {
                       e.target.disabled = false;
                       notifications.push(
                         "error",
                         `${deviceId}: ${err.message}`
                       );
                     });
-                }
+                },
               },
               getIcon("remove")
             )
@@ -79,7 +85,7 @@ const component: ClosureComponent = (): Component => {
           m(
             "button",
             {
-              onclick: e => {
+              onclick: (e) => {
                 e.target.disabled = true;
                 const deviceId = device["DeviceID.ID"].value[0];
                 const tag = prompt(`Enter tag to assign to device:`);
@@ -95,17 +101,17 @@ const component: ClosureComponent = (): Component => {
                     notifications.push("success", `${deviceId}: Tags updated`);
                     store.fulfill(0, Date.now());
                   })
-                  .catch(err => {
+                  .catch((err) => {
                     e.target.disabled = false;
                     notifications.push("error", `${deviceId}: ${err.message}`);
                   });
-              }
+              },
             },
             getIcon("add")
           )
         )
       );
-    }
+    },
   };
 };
 

ui/config-functions.ts

@@ -27,7 +27,7 @@ interface Diff {
   remove: string[];
 }
 
-export function flattenConfig(config): any {
+export function flattenConfig(config: Record<string, unknown>): any {
   const flatten = {};
   const recuresive = (obj: any, root: string): void => {
     for (const [k, v] of Object.entries(obj)) {
@@ -42,6 +42,35 @@ export function flattenConfig(config): any {
   return flatten;
 }
 
+// Order keys such that nested objects come last
+function orderKeys(config: any): number {
+  let res = 1;
+  if (typeof config !== "object") return res;
+  if (Array.isArray(config)) {
+    for (const c of config) res += orderKeys(c);
+    return res;
+  }
+
+  const weights: [string, number][] = Object.entries(config).map(([k, v]) => [
+    k,
+    orderKeys(v),
+  ]);
+
+  weights.sort((a, b) => {
+    if (a[1] !== b[1]) return a[1] - b[1];
+    if (b[0] > a[0]) return -1;
+    else return 1;
+  });
+
+  for (const [k, w] of weights) {
+    res += w;
+    const v = config[k];
+    delete config[k];
+    config[k] = v;
+  }
+  return res;
+}
+
 export function structureConfig(config: Config[]): any {
   config.sort((a, b) => (a._id > b._id ? 1 : a._id < b._id ? -1 : 0));
   const _config = {};
@@ -56,7 +85,7 @@ export function structureConfig(config: Config[]): any {
     ref[keys[0]] = c.value;
   }
 
-  const toArray = function(object): any {
+  const toArray = function (object): any {
     const MAX_BITS = 30;
     const MAX_ARRAY_SIZE = MAX_BITS * 10;
 
@@ -93,13 +122,18 @@ export function structureConfig(config: Config[]): any {
     return object;
   };
 
-  return toArray(_config);
+  const res = toArray(_config);
+  orderKeys(res);
+  return res;
 }
 
-export function diffConfig(current, target): Diff {
+export function diffConfig(
+  current: Record<string, unknown>,
+  target: Record<string, unknown>
+): Diff {
   const diff = {
     add: [],
-    remove: []
+    remove: [],
   };
 
   for (const [k, v] of Object.entries(target))

ui/config-page.ts

@@ -30,7 +30,7 @@ import { getIcon } from "./icons";
 
 const attributes = [
   { id: "_id", label: "Key" },
-  { id: "value", label: "Value", type: "textarea" }
+  { id: "value", label: "Value", type: "textarea" },
 ];
 
 interface ValidationErrors {
@@ -52,13 +52,13 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
         object.value = stringify(parse(object.value || ""));
       } catch (err) {
         return void resolve({
-          value: "Config value must be valid expression"
+          value: "Config value must be valid expression",
         });
       }
 
       store
         .resourceExists("config", id)
-        .then(exists => {
+        .then((exists) => {
           if (exists && isNew) {
             store.fulfill(0, Date.now());
             return void resolve({ _id: "Config already exists" });
@@ -90,7 +90,7 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
           store.fulfill(0, Date.now());
           resolve();
         })
-        .catch(err => {
+        .catch((err) => {
           store.fulfill(0, Date.now());
           reject(err);
         });
@@ -102,14 +102,14 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
 
 const formData = {
   resource: "config",
-  attributes: attributes
+  attributes: attributes,
 };
 
 function escapeRegExp(str): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
 
-export function init(): Promise<{}> {
+export function init(): Promise<Record<string, unknown>> {
   if (!window.authorizer.hasAccess("config", 2)) {
     return Promise.reject(
       new Error("You are not authorized to view this page")
@@ -131,9 +131,9 @@ function renderTable(confsResponse, searchString): Children {
 
   let regex;
   if (searchString) {
-    const keywords = searchString.split(" ").filter(s => s);
+    const keywords = searchString.split(" ").filter((s) => s);
     if (keywords.length)
-      regex = new RegExp(keywords.map(s => escapeRegExp(s)).join(".*"), "i");
+      regex = new RegExp(keywords.map((s) => escapeRegExp(s)).join(".*"), "i");
   }
 
   const rows = [];
@@ -147,38 +147,43 @@ function renderTable(confsResponse, searchString): Children {
       {
         title: "Edit config value",
         onclick: () => {
-          const cb = (): Children => {
-            return m(
-              putFormComponent,
-              Object.assign(
-                {
-                  base: conf,
-                  actionHandler: (action, object) => {
-                    return new Promise(resolve => {
-                      putActionHandler(action, object, false)
-                        .then(errors => {
-                          const ErrorList = errors ? Object.values(errors) : [];
-                          if (ErrorList.length) {
-                            for (const err of ErrorList)
-                              notifications.push("error", err);
-                          } else {
-                            overlay.close(cb);
-                          }
-                          resolve();
-                        })
-                        .catch(err => {
-                          notifications.push("error", err.message);
-                          resolve();
-                        });
-                    });
-                  }
+          let cb: () => Children = null;
+          const comp = m(
+            putFormComponent,
+            Object.assign(
+              {
+                base: conf,
+                actionHandler: (action, object) => {
+                  return new Promise((resolve) => {
+                    putActionHandler(action, object, false)
+                      .then((errors) => {
+                        const ErrorList = errors ? Object.values(errors) : [];
+                        if (ErrorList.length) {
+                          for (const err of ErrorList)
+                            notifications.push("error", err);
+                        } else {
+                          overlay.close(cb);
+                        }
+                        resolve();
+                      })
+                      .catch((err) => {
+                        notifications.push("error", err.message);
+                        resolve();
+                      });
+                  });
                 },
-                formData
-              )
-            );
-          };
-          overlay.open(cb);
-        }
+              },
+              formData
+            )
+          );
+          cb = () => comp;
+          overlay.open(
+            cb,
+            () =>
+              !comp.state["current"]["modified"] ||
+              confirm("You have unsaved changes. Close anyway?")
+          );
+        },
       },
       getIcon("edit")
     );
@@ -190,10 +195,10 @@ function renderTable(confsResponse, searchString): Children {
         onclick: () => {
           if (!confirm(`Deleting ${conf._id} config. Are you sure?`)) return;
 
-          putActionHandler("delete", conf).catch(err => {
+          putActionHandler("delete", conf).catch((err) => {
             throw err;
           });
-        }
+        },
       },
       getIcon("remove")
     );
@@ -219,18 +224,18 @@ function renderTable(confsResponse, searchString): Children {
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       document.title = "Config - GenieACS";
 
       const search = m("input", {
         type: "text",
         placeholder: "Search config",
-        oninput: e => {
+        oninput: (e) => {
           vnode.state["searchString"] = e.target.value;
           e.redraw = false;
           clearTimeout(vnode.state["timeout"]);
           vnode.state["timeout"] = setTimeout(m.redraw, 250);
-        }
+        },
       });
 
       const confs = store.fetch("config", true);
@@ -244,39 +249,44 @@ export const component: ClosureComponent = (): Component => {
           {
             title: "Create new config",
             onclick: () => {
-              const cb = (): Children => {
-                return m(
-                  putFormComponent,
-                  Object.assign(
-                    {
-                      actionHandler: (action, object) => {
-                        return new Promise(resolve => {
-                          putActionHandler(action, object, true)
-                            .then(errors => {
-                              const errorList = errors
-                                ? Object.values(errors)
-                                : [];
-                              if (errorList.length) {
-                                for (const err of errorList)
-                                  notifications.push("error", err);
-                              } else {
-                                overlay.close(cb);
-                              }
-                              resolve();
-                            })
-                            .catch(err => {
-                              notifications.push("error", err.message);
-                              resolve();
-                            });
-                        });
-                      }
+              let cb: () => Children = null;
+              const comp = m(
+                putFormComponent,
+                Object.assign(
+                  {
+                    actionHandler: (action, object) => {
+                      return new Promise((resolve) => {
+                        putActionHandler(action, object, true)
+                          .then((errors) => {
+                            const errorList = errors
+                              ? Object.values(errors)
+                              : [];
+                            if (errorList.length) {
+                              for (const err of errorList)
+                                notifications.push("error", err);
+                            } else {
+                              overlay.close(cb);
+                            }
+                            resolve();
+                          })
+                          .catch((err) => {
+                            notifications.push("error", err.message);
+                            resolve();
+                          });
+                      });
                     },
-                    formData
-                  )
-                );
-              };
-              overlay.open(cb);
-            }
+                  },
+                  formData
+                )
+              );
+              cb = () => comp;
+              overlay.open(
+                cb,
+                () =>
+                  !comp.state["current"]["modified"] ||
+                  confirm("You have unsaved changes. Close anyway?")
+              );
+            },
           },
           "New config"
         );
@@ -286,7 +296,7 @@ export const component: ClosureComponent = (): Component => {
           { name: "charts", prefix: "ui.overview.charts.", data: [] },
           { name: "filters", prefix: "ui.filters.", data: [] },
           { name: "index page", prefix: "ui.index.", data: [] },
-          { name: "device page", prefix: "ui.device.", data: [] }
+          { name: "device page", prefix: "ui.device.", data: [] },
         ];
 
         if (confs.fulfilled) {
@@ -307,40 +317,45 @@ export const component: ClosureComponent = (): Component => {
               "button",
               {
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
-                      uiConfigComponent,
-                      Object.assign(
-                        {
-                          onUpdate: errs => {
-                            const errors = errs ? Object.values(errs) : [];
-                            if (errors.length) {
-                              for (const err of errors)
-                                notifications.push("error", err);
-                            } else {
-                              notifications.push(
-                                "success",
-                                `${sub.name.replace(
-                                  /^[a-z]/,
-                                  sub.name[0].toUpperCase()
-                                )} config updated`
-                              );
-                              overlay.close(cb);
-                            }
-                            store.fulfill(0, Date.now());
-                          },
-                          onError: err => {
-                            notifications.push("error", err.message);
-                            store.fulfill(0, Date.now());
+                  let cb: () => Children = null;
+                  const comp = m(
+                    uiConfigComponent,
+                    Object.assign(
+                      {
+                        onUpdate: (errs: string[]) => {
+                          const errors = errs ? Object.values(errs) : [];
+                          if (errors.length) {
+                            for (const err of errors)
+                              notifications.push("error", err);
+                          } else {
+                            notifications.push(
+                              "success",
+                              `${sub.name.replace(
+                                /^[a-z]/,
+                                sub.name[0].toUpperCase()
+                              )} config updated`
+                            );
                             overlay.close(cb);
                           }
+                          store.fulfill(0, Date.now());
                         },
-                        attrs
-                      )
-                    );
-                  };
-                  overlay.open(cb);
-                }
+                        onError: (err) => {
+                          notifications.push("error", err.message);
+                          store.fulfill(0, Date.now());
+                          overlay.close(cb);
+                        },
+                      },
+                      attrs
+                    )
+                  );
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
+                },
               },
               `Edit ${sub.name}`
             )
@@ -359,8 +374,8 @@ export const component: ClosureComponent = (): Component => {
             renderTable(confs, vnode.state["searchString"])
           ),
           m(".actions-bar", [newConfig].concat(subs))
-        )
+        ),
       ];
-    }
+    },
   };
 };

ui/css/app.css

@@ -106,7 +106,7 @@ body {
 
   & > .drawer-wrapper {
     position: fixed;
-    z-index: 10;
+    z-index: 101;
     width: 720px;
     margin-left: -360px;
     top: 0;
@@ -178,6 +178,8 @@ body {
       }
 
       & > .queue {
+        max-height: 400px;
+        overflow-y: auto;
         & > .pending {
           animation: 0.5s linear 0s infinite alternate pending-blink;
           border-radius: 0.4em;
@@ -460,7 +462,7 @@ input {
   margin-bottom: 0.5em;
   & > input {
     font-size: 90%;
-    width: 30em;
+    width: 35em;
     margin: 2px;
     padding: 2px;
   }
@@ -562,7 +564,7 @@ span.na {
     font-family: monospace;
     margin-bottom: 2px;
     padding: 4px;
-    width: 600px;
+    width: 700px;
   }
 
   & > .parameter-list {

ui/device-page.ts

@@ -22,7 +22,9 @@ import { m } from "./components";
 import config from "./config";
 import * as store from "./store";
 
-export function init(args): Promise<{}> {
+export function init(
+  args: Record<string, unknown>
+): Promise<Record<string, unknown>> {
   if (!window.authorizer.hasAccess("devices", 2)) {
     return Promise.reject(
       new Error("You are not authorized to view this page")
@@ -31,13 +33,13 @@ export function init(args): Promise<{}> {
 
   return Promise.resolve({
     deviceId: args.id,
-    deviceFilter: ["=", ["PARAM", "DeviceID.ID"], args.id]
+    deviceFilter: ["=", ["PARAM", "DeviceID.ID"], args.id],
   });
 }
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       document.title = `${vnode.attrs["deviceId"]} - Devices - GenieACS`;
 
       const dev = store.fetch("devices", vnode.attrs["deviceFilter"]).value;
@@ -49,6 +51,6 @@ export const component: ClosureComponent = (): Component => {
         cmps.push(m.context({ device: dev[0] }, c["type"], c));
 
       return [m("h1", vnode.attrs["deviceId"]), cmps];
-    }
+    },
   };
 };

ui/devices-page.ts

@@ -23,7 +23,7 @@ import config from "./config";
 import indexTableComponent from "./index-table-component";
 import filterComponent from "./filter-component";
 import * as store from "./store";
-import * as taskQueue from "./task-queue";
+import { queueTask, stageDownload } from "./task-queue";
 import * as notifications from "./notifications";
 import { parse, stringify } from "../lib/common/expression-parser";
 import { evaluate, extractParams } from "../lib/common/expression";
@@ -35,7 +35,7 @@ const PAGE_SIZE = config.ui.pageSize || 10;
 
 const memoizedParse = memoize(parse);
 const memoizedJsonParse = memoize(JSON.parse);
-const memoizedGetSortable = memoize(p => {
+const memoizedGetSortable = memoize((p) => {
   const expressionParams = extractParams(p);
   if (expressionParams.length === 1) {
     const param = evaluate(expressionParams[0]);
@@ -47,21 +47,23 @@ const memoizedGetSortable = memoize(p => {
 const getDownloadUrl = memoize((filter, indexParameters) => {
   const columns = {};
   for (const p of indexParameters) columns[p.label] = stringify(p.parameter);
-  return `/api/devices.csv?${m.buildQueryString({
+  return `api/devices.csv?${m.buildQueryString({
     filter: stringify(filter),
-    columns: JSON.stringify(columns)
+    columns: JSON.stringify(columns),
   })}`;
 });
 
-const unpackSmartQuery = memoize(query => {
-  return expressionParser.map(query, e => {
+const unpackSmartQuery = memoize((query) => {
+  return expressionParser.map(query, (e) => {
     if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "Q")
       return smartQuery.unpack("devices", e[2], e[3]);
     return e;
   });
 });
 
-export function init(args): Promise<{}> {
+export function init(
+  args: Record<string, unknown>
+): Promise<Record<string, unknown>> {
   return new Promise((resolve, reject) => {
     if (!window.authorizer.hasAccess("devices", 2))
       return void reject(new Error("You are not authorized to view this page"));
@@ -72,14 +74,14 @@ export function init(args): Promise<{}> {
     if (!indexParameters.length) {
       indexParameters.push({
         label: "ID",
-        parameter: ["PARAM", "DeviceID.ID"]
+        parameter: ["PARAM", "DeviceID.ID"],
       });
     }
     resolve({ filter, indexParameters, sort });
   });
 }
 
-function renderActions(selected): Children {
+function renderActions(selected: Set<string>): Children {
   const buttons = [];
 
   buttons.push(
@@ -89,13 +91,12 @@ function renderActions(selected): Children {
         title: "Reboot selected devices",
         disabled: !selected.size,
         onclick: () => {
-          for (const d of selected) {
-            taskQueue.queueTask({
-              name: "reboot",
-              device: d
-            });
-          }
-        }
+          const tasks = [...selected].map((s) => ({
+            name: "reboot",
+            device: s,
+          }));
+          queueTask(...tasks);
+        },
       },
       "Reboot"
     )
@@ -108,18 +109,34 @@ function renderActions(selected): Children {
         title: "Factory reset selected devices",
         disabled: !selected.size,
         onclick: () => {
-          for (const d of selected) {
-            taskQueue.queueTask({
-              name: "factoryReset",
-              device: d
-            });
-          }
-        }
+          const tasks = [...selected].map((s) => ({
+            name: "factoryReset",
+            device: s,
+          }));
+          queueTask(...tasks);
+        },
       },
       "Reset"
     )
   );
 
+  buttons.push(
+    m(
+      "button.critical",
+      {
+        title: "Push a firmware or a config file",
+        disabled: !selected.size,
+        onclick: () => {
+          stageDownload({
+            name: "download",
+            devices: [...selected],
+          });
+        },
+      },
+      "Push file"
+    )
+  );
+
   buttons.push(
     m(
       "button.primary",
@@ -139,13 +156,13 @@ function renderActions(selected): Children {
                 notifications.push("success", `${id}: Deleted`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               })
-              .catch(err => {
+              .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
           if (--counter === 0) store.fulfill(0, Date.now());
-        }
+        },
       },
       "Delete"
     )
@@ -171,13 +188,13 @@ function renderActions(selected): Children {
                 notifications.push("success", `${id}: Tags updated`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               })
-              .catch(err => {
+              .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
           if (--counter === 0) store.fulfill(0, Date.now());
-        }
+        },
       },
       "Tag"
     )
@@ -205,13 +222,13 @@ function renderActions(selected): Children {
                 notifications.push("success", `${id}: Tags updated`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               })
-              .catch(err => {
+              .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
           if (--counter === 0) store.fulfill(0, Date.now());
-        }
+        },
       },
       "Untag"
     )
@@ -222,7 +239,7 @@ function renderActions(selected): Children {
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       document.title = "Devices - GenieACS";
       const attributes = vnode.attrs["indexParameters"];
 
@@ -273,7 +290,7 @@ export const component: ClosureComponent = (): Component => {
 
       const devs = store.fetch("devices", filter, {
         limit: vnode.state["showCount"] || PAGE_SIZE,
-        sort: sort
+        sort: sort,
       });
       const count = store.count("devices", filter);
 
@@ -302,7 +319,7 @@ export const component: ClosureComponent = (): Component => {
           {
             href: `#!/devices/${encodeURIComponent(
               device["DeviceID.ID"].value[0]
-            )}`
+            )}`,
           },
           "Show"
         );
@@ -319,8 +336,8 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing devices"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs)
+        m(indexTableComponent, attrs),
       ];
-    }
+    },
   };
 };

ui/drawer-component.ts

@@ -22,12 +22,23 @@ import m, {
   Child,
   ClosureComponent,
   Component,
-  VnodeDOM
+  VnodeDOM,
 } from "mithril";
-import * as taskQueue from "./task-queue";
 import * as store from "./store";
 import * as notifications from "./notifications";
 import { getIcon } from "./icons";
+import {
+  clear,
+  commit,
+  deleteTask,
+  getQueue,
+  getStaging,
+  QueueTask,
+  queueTask,
+  StageTask,
+} from "./task-queue";
+
+const invalid: WeakSet<StageTask> = new WeakSet();
 
 function mparam(param): Children {
   return m("span.parameter", { title: param }, `${param}`);
@@ -37,11 +48,11 @@ function mval(val): Children {
   return m("span.value", { title: val }, `${val}`);
 }
 
-function renderStagingSpv(task, queueFunc, cancelFunc): Children {
-  function keydown(e): void {
-    if (e.keyCode === 13) queueFunc();
-    else if (e.keyCode === 27) cancelFunc();
-    else e.redraw = false;
+function renderStagingSpv(task: StageTask, queueFunc, cancelFunc): Children {
+  function keydown(e: KeyboardEvent): void {
+    if (e.key === "Enter") queueFunc();
+    else if (e.key === "Escape") cancelFunc();
+    else e["redraw"] = false;
   }
 
   let input;
@@ -50,18 +61,18 @@ function renderStagingSpv(task, queueFunc, cancelFunc): Children {
       "select",
       {
         value: task.parameterValues[0][1].toString(),
-        onchange: e => {
+        onchange: (e) => {
           e.redraw = false;
           task.parameterValues[0][1] = input.dom.value;
         },
         onkeydown: keydown,
-        oncreate: vnode => {
+        oncreate: (vnode) => {
           (vnode.dom as HTMLSelectElement).focus();
-        }
+        },
       },
       [
         m("option", { value: "true" }, "true"),
-        m("option", { value: "false" }, "false")
+        m("option", { value: "false" }, "false"),
       ]
     );
   } else {
@@ -72,31 +83,42 @@ function renderStagingSpv(task, queueFunc, cancelFunc): Children {
         ? "number"
         : "text",
       value: task.parameterValues[0][1],
-      oninput: e => {
+      oninput: (e) => {
         e.redraw = false;
         task.parameterValues[0][1] = input.dom.value;
       },
       onkeydown: keydown,
-      oncreate: vnode => {
+      oncreate: (vnode) => {
         (vnode.dom as HTMLInputElement).focus();
         (vnode.dom as HTMLInputElement).select();
         // Need to prevent scrolling on focus because
         // we're animating height and using overflow: hidden
         (vnode.dom.parentNode.parentNode as Element).scrollTop = 0;
-      }
+      },
     });
   }
 
   return [m("span", "Editing ", mparam(task.parameterValues[0][0])), input];
 }
 
-function renderStagingDownload(task): Children {
-  task.invalid = !task.fileName || !task.fileType;
+function renderStagingDownload(task: StageTask): Children {
+  if (!task.fileName || !task.fileType) invalid.add(task);
+  else invalid.delete(task);
   const files = store.fetch("files", true);
-  const idParts = task.device.split("-");
-  const oui = decodeURIComponent(idParts[0]);
-  const productClass =
-    idParts.length === 3 ? decodeURIComponent(idParts[1]) : "";
+  let oui = "";
+  let productClass = "";
+  for (const d of task.devices) {
+    const parts = d.split("-");
+    if (oui === "") oui = parts[0];
+    else if (oui !== parts[0]) oui = null;
+    if (parts.length === 3) {
+      if (productClass === "") productClass = parts[1];
+      else if (productClass !== parts[1]) productClass = null;
+    }
+  }
+
+  if (oui) oui = decodeURIComponent(oui);
+  if (productClass) productClass = decodeURIComponent(productClass);
 
   const typesList = [
     "",
@@ -104,8 +126,8 @@ function renderStagingDownload(task): Children {
     "2 Web Content",
     "3 Vendor Configuration File",
     "4 Tone File",
-    "5 Ringer File"
-  ].map(t =>
+    "5 Ringer File",
+  ].map((t) =>
     m(
       "option",
       { disabled: !t, value: t, selected: (task.fileType || "") === t },
@@ -117,14 +139,14 @@ function renderStagingDownload(task): Children {
     .concat(
       files.value
         .filter(
-          f =>
+          (f) =>
             (!f["metadata.oui"] || f["metadata.oui"] === oui) &&
             (!f["metadata.productClass"] ||
               f["metadata.productClass"] === productClass)
         )
-        .map(f => f._id)
+        .map((f) => f._id)
     )
-    .map(f =>
+    .map((f) =>
       m(
         "option",
         { disabled: !f, value: f, selected: (task.fileName || "") === f },
@@ -137,7 +159,7 @@ function renderStagingDownload(task): Children {
     m(
       "select",
       {
-        onchange: e => {
+        onchange: (e) => {
           const f = e.target.value;
           task.fileName = f;
           task.fileType = "";
@@ -145,7 +167,7 @@ function renderStagingDownload(task): Children {
             if (file._id === f) task.fileType = file["metadata.fileType"];
         },
         disabled: files.fulfilling,
-        style: "width: 350px"
+        style: "width: 350px",
       },
       filesList
     ),
@@ -153,22 +175,142 @@ function renderStagingDownload(task): Children {
     m(
       "select",
       {
-        onchange: e => {
+        onchange: (e) => {
           task.fileType = e.target.value;
-        }
+        },
       },
       typesList
-    )
+    ),
   ];
 }
 
-function renderStaging(staging): Child[] {
+const uploadTypesForDevice: WeakMap<
+  Record<string, unknown>,
+  Record<string, string>
+> = new WeakMap();
+function getUploadTypesForDevice(
+  device: Record<string, unknown>
+): Record<string, string> {
+  if (uploadTypesForDevice.has(device)) return uploadTypesForDevice.get(device);
+  const instances: Set<string> = new Set();
+  const filenames: Record<string, string> = {};
+  const versions: Record<string, string> = {};
+
+  if (device) {
+    for (const [k, v] of Object.entries(device)) {
+      if (/DeviceInfo\.VendorConfigFile\.[0-9]+(\.Name|\.Version)?$/.test(k)) {
+        const p = k.split(".");
+        if (p[p.length - 1] === "Name") {
+          const n = `3 Vendor Configuration File ${p[p.length - 2]}`;
+          instances.add(n);
+          if (v["value"]) filenames[n] = v["value"][0];
+        } else if (p[p.length - 1] === "Version") {
+          const n = `3 Vendor Configuration File ${p[p.length - 2]}`;
+          instances.add(n);
+          if (v["value"]) versions[n] = v["value"][0];
+        } else {
+          const n = `3 Vendor Configuration File ${p[p.length - 1]}`;
+          instances.add(n);
+        }
+      } else if (/DeviceInfo\.VendorLogFile\.[0-9]+(\.Name)?$/.test(k)) {
+        const p = k.split(".");
+        if (p[p.length - 1] === "Name") {
+          const n = `4 Vendor Log File ${p[p.length - 2]}`;
+          instances.add(n);
+          if (v["value"]) filenames[n] = v["value"][0];
+        } else {
+          const n = `4 Vendor Log File ${p[p.length - 1]}`;
+          instances.add(n);
+        }
+      }
+    }
+  }
+
+  if (!instances.size) {
+    instances.add("1 Vendor Configuration File");
+    instances.add("2 Vendor Log File");
+  }
+
+  const res: Record<string, string> = {};
+
+  for (const i of instances) {
+    let n = filenames[i];
+    const v = versions[i];
+    if (!n) n = i.replace(/\s+/g, "_") + ".txt";
+    if (v) n = `${n}-${v}`;
+    res[i] = n;
+  }
+
+  uploadTypesForDevice.set(device, res);
+  return res;
+}
+
+function renderStagingUpload(task: StageTask): Children {
+  if (!task.fileName || !task.fileType) invalid.add(task);
+  else invalid.delete(task);
+
+  const commonTypes: Record<string, number> = {};
+
+  const filenames: Record<string, string> = {};
+
+  for (const d of task.devices) {
+    const dev = store.fetch("devices", ["=", ["PARAM", "DeviceID.ID"], d])
+      .value[0];
+    if (dev) {
+      const u = getUploadTypesForDevice(dev);
+      Object.assign(filenames, u);
+      for (const k of Object.keys(u))
+        commonTypes[k] = (commonTypes[k] || 0) + 1;
+    }
+  }
+
+  const typesList = [""];
+  typesList.push(
+    ...Object.entries(commonTypes)
+      .filter(([, c]) => c === task.devices.length)
+      .map(([k]) => k)
+      .sort()
+  );
+
+  return [
+    "Fetch ",
+    m(
+      "select",
+      {
+        onchange: (e) => {
+          task.fileType = e.target.value;
+          task.fileName = filenames[task.fileType];
+        },
+      },
+      typesList.map((t) =>
+        m(
+          "option",
+          { disabled: !t, value: t, selected: (task.fileType || "") === t },
+          t
+        )
+      )
+    ),
+    " as ",
+    m("input", {
+      onchange: (e) => {
+        task.fileName = e.target.value;
+      },
+      value: task.fileName || "",
+    }),
+  ];
+}
+
+function renderStaging(staging: Set<StageTask>): Child[] {
   const elements: Child[] = [];
 
   for (const s of staging) {
     const queueFunc = (): void => {
       staging.delete(s);
-      taskQueue.queueTask(s);
+      for (const d of s.devices) {
+        const t = Object.assign({ device: d }, s);
+        delete t.devices;
+        queueTask(t);
+      }
     };
     const cancelFunc = (): void => {
       staging.delete(s);
@@ -178,10 +320,11 @@ function renderStaging(staging): Child[] {
     if (s.name === "setParameterValues")
       elms = renderStagingSpv(s, queueFunc, cancelFunc);
     else if (s.name === "download") elms = renderStagingDownload(s);
+    else if (s.name === "upload") elms = renderStagingUpload(s);
 
     const queue = m(
       "button.primary",
-      { title: "Queue task", onclick: queueFunc, disabled: s.invalid },
+      { title: "Queue task", onclick: queueFunc, disabled: invalid.has(s) },
       "Queue"
     );
     const cancel = m(
@@ -195,7 +338,7 @@ function renderStaging(staging): Child[] {
   return elements;
 }
 
-function renderQueue(queue): Child[] {
+function renderQueue(queue: Set<QueueTask>): Child[] {
   const details: Child[] = [];
   const devices: { [deviceId: string]: any[] } = {};
   for (const t of queue) {
@@ -215,8 +358,8 @@ function renderQueue(queue): Child[] {
             {
               title: "Retry this task",
               onclick: () => {
-                taskQueue.queueTask(t);
-              }
+                queueTask(t);
+              },
             },
             getIcon("retry")
           )
@@ -229,8 +372,8 @@ function renderQueue(queue): Child[] {
           {
             title: "Remove this task",
             onclick: () => {
-              taskQueue.deleteTask(t);
-            }
+              deleteTask(t);
+            },
           },
           getIcon("remove")
         )
@@ -297,6 +440,14 @@ function renderQueue(queue): Child[] {
             m(".actions", actions)
           )
         );
+      } else if (t.name === "upload") {
+        details.push(
+          m(
+            `div.${t.status}`,
+            `Fetch file ${t.fileName} (${t.fileType})`,
+            m(".actions", actions)
+          )
+        );
       } else {
         details.push(m(`div.${t.status}`, t.name, m(".actions", actions)));
       }
@@ -324,18 +475,18 @@ function renderNotifications(notifs): Child[] {
         {
           class: n.type,
           style: "position: absolute;opacity: 0",
-          oncreate: vnode => {
+          oncreate: (vnode) => {
             (vnode.dom as HTMLDivElement).style.opacity = "1";
           },
-          onbeforeremove: vnode => {
+          onbeforeremove: (vnode) => {
             (vnode.dom as HTMLDivElement).style.opacity = "0";
-            return new Promise(resolve => {
+            return new Promise((resolve) => {
               setTimeout(() => {
                 resolve();
               }, 500);
             });
           },
-          key: n.timestamp
+          key: n.timestamp,
         },
         m("div", buttons, n.message)
       )
@@ -346,9 +497,9 @@ function renderNotifications(notifs): Child[] {
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
-      const queue = taskQueue.getQueue();
-      const staging = taskQueue.getStaging();
+    view: (vnode) => {
+      const queue = getQueue();
+      const staging = getStaging();
       const notifs = notifications.getNotifications();
 
       let drawerElement, statusElement;
@@ -394,55 +545,57 @@ const component: ClosureComponent = (): Component => {
               title: "Commit queued tasks",
               disabled: !statusCount.queued,
               onclick: () => {
-                const tasks = Array.from(taskQueue.getQueue()).filter(
-                  t => t["status"] === "queued"
+                const tasks = Array.from(getQueue()).filter(
+                  (t) => t["status"] === "queued"
                 );
-                taskQueue
-                  .commit(
-                    tasks,
-                    (deviceId, err, connectionRequestStatus, tasks2) => {
-                      if (err) {
-                        notifications.push(
-                          "error",
-                          `${deviceId}: ${err.message}`
-                        );
-                        return;
-                      }
-
-                      if (connectionRequestStatus !== "OK") {
-                        notifications.push(
-                          "error",
-                          `${deviceId}: ${connectionRequestStatus}`
-                        );
-                        return;
-                      }
-
-                      for (const t of tasks2) {
-                        if (t.status === "stale") {
-                          notifications.push(
-                            "error",
-                            `${deviceId}: No contact from device`
-                          );
-                          return;
-                        } else if (t.status === "fault") {
-                          notifications.push(
-                            "error",
-                            `${deviceId}: Task(s) faulted`
-                          );
-                          return;
-                        }
-                      }
-
+                commit(
+                  tasks,
+                  (deviceId, err, connectionRequestStatus, tasks2) => {
+                    if (err) {
                       notifications.push(
-                        "success",
-                        `${deviceId}: Task(s) committed`
+                        "error",
+                        `${deviceId}: ${err.message}`
                       );
+                      return;
                     }
-                  )
+
+                    if (connectionRequestStatus !== "OK") {
+                      notifications.push(
+                        "error",
+                        `${deviceId}: ${connectionRequestStatus}`
+                      );
+                      return;
+                    }
+
+                    for (const t of tasks2) {
+                      if (t.status === "stale") {
+                        notifications.push(
+                          "error",
+                          `${deviceId}: No contact from device`
+                        );
+                        return;
+                      } else if (t.status === "fault") {
+                        notifications.push(
+                          "error",
+                          `${deviceId}: Task(s) faulted`
+                        );
+                        return;
+                      }
+                    }
+
+                    notifications.push(
+                      "success",
+                      `${deviceId}: Task(s) committed`
+                    );
+                  }
+                )
                   .then(() => {
                     store.fulfill(0, Date.now());
+                  })
+                  .catch((err) => {
+                    notifications.push("error", err.message);
                   });
-              }
+              },
             },
             "Commit"
           ),
@@ -450,8 +603,8 @@ const component: ClosureComponent = (): Component => {
             "button",
             {
               title: "Clear tasks",
-              onclick: taskQueue.clear,
-              disabled: !queueElements.length
+              onclick: clear,
+              disabled: !queueElements.length,
             },
             "Clear"
           )
@@ -487,31 +640,31 @@ const component: ClosureComponent = (): Component => {
           {
             key: "drawer",
             style: "opacity: 0;height: 0;",
-            oncreate: vnode2 => {
+            oncreate: (vnode2) => {
               vnode.state["mouseIn"] = false;
               (vnode2.dom as HTMLDivElement).style.opacity = "1";
               resizeDrawer();
             },
-            onmouseover: e => {
+            onmouseover: (e) => {
               vnode.state["mouseIn"] = true;
               resizeDrawer();
               e.redraw = false;
             },
-            onmouseleave: e => {
+            onmouseleave: (e) => {
               vnode.state["mouseIn"] = false;
               resizeDrawer();
               e.redraw = false;
             },
             onupdate: resizeDrawer,
-            onbeforeremove: vnode2 => {
+            onbeforeremove: (vnode2) => {
               (vnode2.dom as HTMLDivElement).onmouseover = null;
               (vnode2.dom as HTMLDivElement).onmouseleave = null;
               (vnode2.dom as HTMLDivElement).style.opacity = "0";
               (vnode2.dom as HTMLDivElement).style.height = "0";
-              return new Promise(resolve => {
+              return new Promise((resolve) => {
                 setTimeout(resolve, 500);
               });
-            }
+            },
           },
           statusElement,
           stagingElements.length ? stagingElements : m(".queue", queueElements)
@@ -527,12 +680,12 @@ const component: ClosureComponent = (): Component => {
             key: "notifications",
             style: "position: relative;",
             onupdate: repositionNotifications,
-            oncreate: repositionNotifications
+            oncreate: repositionNotifications,
           },
           notificationElements
         )
       );
-    }
+    },
   };
 };
 

ui/dynamic-loader.ts

@@ -32,7 +32,7 @@ function onError(): void {
       {
         Reload: () => {
           window.location.reload();
-        }
+        },
       }
     );
   }
@@ -47,14 +47,14 @@ export function loadCodeMirror(): Promise<void> {
       import(
         /* webpackChunkName: "codemirror" */ "codemirror/mode/javascript/javascript"
       ),
-      import(/* webpackChunkName: "codemirror" */ "codemirror/mode/yaml/yaml")
+      import(/* webpackChunkName: "codemirror" */ "codemirror/mode/yaml/yaml"),
     ];
     Promise.all(promises)
-      .then(modules => {
-        codeMirror = modules[0];
+      .then((modules) => {
+        codeMirror = modules[0].default;
         resolve();
       })
-      .catch(err => {
+      .catch((err) => {
         onError();
         reject(err);
       });
@@ -66,11 +66,11 @@ export function loadYaml(): Promise<void> {
 
   return new Promise((resolve, reject) => {
     import(/* webpackChunkName: "yaml" */ "yaml")
-      .then(module => {
+      .then((module) => {
         yaml = module;
         resolve();
       })
-      .catch(err => {
+      .catch((err) => {
         onError();
         reject(err);
       });

ui/error-page.ts

@@ -22,9 +22,9 @@ import { m } from "./components";
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: function(vnode) {
+    view: function (vnode) {
       document.title = "Error! - GenieACS";
       return m("p.error", vnode.attrs["error"]);
-    }
+    },
   };
 };

ui/faults-page.ts

@@ -41,31 +41,33 @@ const attributes = [
   { id: "message", label: "Message" },
   { id: "detail", label: "Detail" },
   { id: "retries", label: "Retries" },
-  { id: "timestamp", label: "Timestamp" }
+  { id: "timestamp", label: "Timestamp" },
 ];
 
-const getDownloadUrl = memoize(filter => {
+const getDownloadUrl = memoize((filter) => {
   const cols = {};
   for (const attr of attributes) {
     cols[attr.label] =
       attr.id === "timestamp" ? `DATE_STRING(${attr.id})` : attr.id;
   }
 
-  return `/api/faults.csv?${m.buildQueryString({
+  return `api/faults.csv?${m.buildQueryString({
     filter: stringify(filter),
-    columns: JSON.stringify(cols)
+    columns: JSON.stringify(cols),
   })}`;
 });
 
-const unpackSmartQuery = memoize(query => {
-  return map(query, e => {
+const unpackSmartQuery = memoize((query) => {
+  return map(query, (e) => {
     if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "Q")
       return smartQuery.unpack("faults", e[2], e[3]);
     return e;
   });
 });
 
-export function init(args): Promise<{}> {
+export function init(
+  args: Record<string, unknown>
+): Promise<Record<string, unknown>> {
   if (!window.authorizer.hasAccess("faults", 2)) {
     return Promise.reject(
       new Error("You are not authorized to view this page")
@@ -85,7 +87,7 @@ export function init(args): Promise<{}> {
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       document.title = "Faults - GenieACS";
 
       function showMore(): void {
@@ -130,7 +132,7 @@ export const component: ClosureComponent = (): Component => {
 
       const faults = store.fetch("faults", filter, {
         limit: vnode.state["showCount"] || PAGE_SIZE,
-        sort: sort
+        sort: sort,
       });
       const count = store.count("faults", filter);
 
@@ -165,13 +167,13 @@ export const component: ClosureComponent = (): Component => {
       attrs["downloadUrl"] = downloadUrl;
 
       if (window.authorizer.hasAccess("faults", 3)) {
-        attrs["actionsCallback"] = (selected): Children => {
+        attrs["actionsCallback"] = (selected: Set<string>): Children => {
           return m(
             "button.primary",
             {
               disabled: selected.size === 0,
               title: "Delete selected faults",
-              onclick: e => {
+              onclick: (e) => {
                 e.redraw = false;
                 e.target.disabled = true;
 
@@ -179,22 +181,22 @@ export const component: ClosureComponent = (): Component => {
                   return;
 
                 Promise.all(
-                  Array.from(selected).map(id =>
+                  Array.from(selected).map((id) =>
                     store.deleteResource("faults", id)
                   )
                 )
-                  .then(res => {
+                  .then((res) => {
                     notifications.push(
                       "success",
                       `${res.length} faults deleted`
                     );
                     store.fulfill(0, Date.now());
                   })
-                  .catch(err => {
+                  .catch((err) => {
                     notifications.push("error", err.message);
                     store.fulfill(0, Date.now());
                   });
-              }
+              },
             },
             "Delete"
           );
@@ -209,8 +211,8 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing faults"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs)
+        m(indexTableComponent, attrs),
       ];
-    }
+    },
   };
 };

ui/files-page.ts

@@ -51,23 +51,23 @@ const attributes: {
       "2 Web Content",
       "3 Vendor Configuration File",
       "4 Tone File",
-      "5 Ringer File"
-    ]
+      "5 Ringer File",
+    ],
   },
   { id: "metadata.oui", label: "OUI" },
   { id: "metadata.productClass", label: "Product Class" },
-  { id: "metadata.version", label: "Version" }
+  { id: "metadata.version", label: "Version" },
 ];
 
 const formData = {
   resource: "files",
   attributes: attributes
     .slice(1) // remove _id from new object form
-    .concat([{ id: "file", label: "File", type: "file" }])
+    .concat([{ id: "file", label: "File", type: "file" }]),
 };
 
-const unpackSmartQuery = memoize(query => {
-  return map(query, e => {
+const unpackSmartQuery = memoize((query) => {
+  return map(query, (e) => {
     if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "Q")
       return smartQuery.unpack("files", e[2], e[3]);
     return e;
@@ -91,7 +91,7 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
 
       store
         .resourceExists("files", id)
-        .then(exists => {
+        .then((exists) => {
           if (exists) {
             store.fulfill(0, Date.now());
             return void resolve({ file: "File already exists" });
@@ -99,7 +99,7 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
           const headers = Object.assign(
             {
               "Content-Type": "application/octet-stream",
-              Accept: "application/octet-stream"
+              Accept: "application/octet-stream",
             },
             object
           );
@@ -108,9 +108,9 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
             .xhrRequest({
               method: "PUT",
               headers: headers,
-              url: `/api/files/${encodeURIComponent(id)}`,
-              serialize: body => body, // Identity function to prevent JSON.parse on blob data
-              body: file
+              url: `api/files/${encodeURIComponent(id)}`,
+              serialize: (body) => body, // Identity function to prevent JSON.parse on blob data
+              body: file,
             })
             .then(() => {
               notifications.push(
@@ -129,16 +129,18 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
   });
 }
 
-const getDownloadUrl = memoize(filter => {
+const getDownloadUrl = memoize((filter) => {
   const cols = {};
   for (const attr of attributes) cols[attr.label] = attr.id;
-  return `/api/files.csv?${m.buildQueryString({
+  return `api/files.csv?${m.buildQueryString({
     filter: stringify(filter),
-    columns: JSON.stringify(cols)
+    columns: JSON.stringify(cols),
   })}`;
 });
 
-export function init(args): Promise<{}> {
+export function init(
+  args: Record<string, unknown>
+): Promise<Record<string, unknown>> {
   if (!window.authorizer.hasAccess("files", 2)) {
     return Promise.reject(
       new Error("You are not authorized to view this page")
@@ -152,7 +154,7 @@ export function init(args): Promise<{}> {
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       document.title = "Files - GenieACS";
 
       function showMore(): void {
@@ -195,7 +197,7 @@ export const component: ClosureComponent = (): Component => {
 
       const files = store.fetch("files", filter, {
         limit: vnode.state["showCount"] || PAGE_SIZE,
-        sort: sort
+        sort: sort,
       });
 
       const count = store.count("files", filter);
@@ -212,46 +214,51 @@ export const component: ClosureComponent = (): Component => {
       attrs["downloadUrl"] = downloadUrl;
 
       if (window.authorizer.hasAccess("files", 3)) {
-        attrs["actionsCallback"] = (selected): Children => {
+        attrs["actionsCallback"] = (selected: Set<string>): Children => {
           return [
             m(
               "button.primary",
               {
                 title: "Create new file",
                 onclick: () => {
-                  const cb = (): Children => {
-                    return m(
-                      putFormComponent,
-                      Object.assign(
-                        {
-                          actionHandler: (action, object) => {
-                            return new Promise(resolve => {
-                              putActionHandler(action, object)
-                                .then(errors => {
-                                  const errorList = errors
-                                    ? Object.values(errors)
-                                    : [];
-                                  if (errorList.length) {
-                                    for (const err of errorList)
-                                      notifications.push("error", err);
-                                  } else {
-                                    overlay.close(cb);
-                                  }
-                                  resolve();
-                                })
-                                .catch(err => {
-                                  notifications.push("error", err.message);
-                                  resolve();
-                                });
-                            });
-                          }
+                  let cb: () => Children = null;
+                  const comp = m(
+                    putFormComponent,
+                    Object.assign(
+                      {
+                        actionHandler: (action, object) => {
+                          return new Promise((resolve) => {
+                            putActionHandler(action, object)
+                              .then((errors) => {
+                                const errorList = errors
+                                  ? Object.values(errors)
+                                  : [];
+                                if (errorList.length) {
+                                  for (const err of errorList)
+                                    notifications.push("error", err);
+                                } else {
+                                  overlay.close(cb);
+                                }
+                                resolve();
+                              })
+                              .catch((err) => {
+                                notifications.push("error", err.message);
+                                resolve();
+                              });
+                          });
                         },
-                        formData
-                      )
-                    );
-                  };
-                  overlay.open(cb);
-                }
+                      },
+                      formData
+                    )
+                  );
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
+                },
               },
               "New"
             ),
@@ -260,7 +267,7 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Delete selected files",
                 disabled: !selected.size,
-                onclick: e => {
+                onclick: (e) => {
                   if (
                     !confirm(`Deleting ${selected.size} files. Are you sure?`)
                   )
@@ -269,25 +276,25 @@ export const component: ClosureComponent = (): Component => {
                   e.redraw = false;
                   e.target.disabled = true;
                   Promise.all(
-                    Array.from(selected).map(id =>
+                    Array.from(selected).map((id) =>
                       store.deleteResource("files", id)
                     )
                   )
-                    .then(res => {
+                    .then((res) => {
                       notifications.push(
                         "success",
                         `${res.length} files deleted`
                       );
                       store.fulfill(0, Date.now());
                     })
-                    .catch(err => {
+                    .catch((err) => {
                       notifications.push("error", err.message);
                       store.fulfill(0, Date.now());
                     });
-                }
+                },
               },
               "Delete"
-            )
+            ),
           ];
         };
       }
@@ -300,8 +307,8 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing files"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs)
+        m(indexTableComponent, attrs),
       ];
-    }
+    },
   };
 };