1.2.3

Some CPEs incorrectly report writable parameters as non-writable. This
config option lets us bypass the check and attempt to modify the
parameter even if it reported as read-only.

.eslintrc.json

@@ -21,6 +21,7 @@
     "@typescript-eslint/no-explicit-any": 0,
     "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
     "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
+    "@typescript-eslint/no-floating-promises": "error",
     "require-atomic-updates": 0,
     "no-prototype-builtins": 0
   }

CHANGELOG.md

@@ -1,5 +1,61 @@
 # Change Log
 
+## 1.2.3 (2020-10-26)
+
+- New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
+report writable parameters as non-writable. When set to true, the scripts will
+no longer respect the 'writable' attribute of the CPE parameters and will send
+a SetParamteerValues, AddObject, or DeleteObject request anyway.
+- Tags no longer restrict what characters are allowed. Any character other than
+alphanumeric characters, hyphen, or underscore is now encoded in the data
+model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
+- Ask for a confirmation before closing a pop-up dialog with unsaved changes.
+- Better XML validation to avoid crashes caused by invalid CPE requests.
+- Fix confusing 404 error message when the user attempts to modify a resource
+when they don't have the necessary permissions.
+- Fix a rare issue where genieacs-cwmp stops accepting new connections after
+running for a few weeks.
+- Fix exception when IS NULL operator is used in certain situations.
+
+## 1.2.2 (2020-10-03)
+
+- Added button to push files to selected devices from device listing page.
+- A few minor UI improvements.
+- Fix exception that can happen and persist after a Download request.
+- Fix validation bug preventing running refreshObject task on data model root.
+- Fix invalid arguments fault in refresh preset configuration when upgrading
+from v1.1.
+
+## 1.2.1 (2020-09-08)
+
+- Fix bug causing faults to not be displayed in the UI.
+- Fix bug where deleting objects does not get reflected immediately in the UI.
+- Improve conversion between filters written in the expression format and
+MongoDB queries. There should now be fewer edge cases where the two are not
+equisatisfiable.
+
+## 1.2.0 (2020-09-01)
+
+- Support GetParameterAttributes and SetParameterAttributes TR-069 methods.
+- Support CASE statement and COALESCE function in expressions.
+- Provision arguments can now be a list of expressions that are dynamically
+evaluated.
+- Support Forwarded HTTP header to display in the logs the correct IP of CPEs
+behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
+- Config expressions can now access all available device parameters, not only
+serial number, product class, and OUI.
+- Use relative URLs throughout the UI to allow serving from a subdirectory
+using a reverse proxy.
+- Make Date.parse() and Date.UTC() available to provision scripts.
+- libxmljs has been entirely removed in favor of our bespoke XML parser.
+- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now
+used to determine the TCP connection timeout.
+- The all-parameters component now limits the number of parameters displayed
+for better performance.
+- The process genieacs-cwmp is now much less likely to throw exceptions as a
+result of invalid requests from CPE.
+- A large number of bug fixes and stability improvements.
+
 ## 1.2.0-beta.0 (2019-07-30)
 
 - A brand new UI superseding genieacs-gui.

CONTRIBUTING.md

@@ -1,22 +1,21 @@
 # Have a question?
 
-Please ask in the [Users mailing list][mailing-lists] instead of opening a
+Please ask in the [forum] instead of opening a Github Issue. The Github issue
-Github Issue. The Github issue tracker is strictly for bug reports.
+tracker is strictly for bug reports.
 
 # Issues
 
 We use GitHub issues to track bugs. Please ensure your bug description is
 clear and has sufficient instructions to be able to reproduce the issue.
 
-If you face interoperability issues with your device, more often than not it's
+If you face interoperability issues with your CPE, more often than not it's a
-a device related issue. Please consult the [Users mailing list][mailing-lists]
+device related issue. Please consult the [forum] before submitting an Issue.
-before submitting an Issue.
 
 # Pull Requests
 
 Pull requests are welcome. If you have a fix for a bug, go ahead and create a
-pull request. For other kinds of enhancements, please discuss in the [Users
+pull request. For other kinds of enhancements, please discuss in the [forum]
-mailing list][mailing-lists] first.
+first.
 
 Please follow the guidelines [here][commit-guidelines] when committing your
 changes.
@@ -36,4 +35,4 @@ source code contained therein, and that your content does not infringe upon
 the intellectual property rights of a third party.
 
 [commit-guidelines]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
-[mailing-lists]: http://lists.genieacs.com
+[forum]: https://forum.genieacs.com

README.md

@@ -8,36 +8,31 @@ millions of concurrent devices.
 
 ## Quick Start
 
-*Important: This is a pre-release branch. Use
-[v1.1](https://github.com/genieacs/genieacs/tree/v1.1) for production
-deployments.*
-
 Install [Node.js](http://nodejs.org/) and [MongoDB](http://www.mongodb.org/).
 Refer to their corresponding documentation for installation instructions. The
 supported versions are:
 
-- Node.js: 10.x through 12.x
+- Node.js: 12.3+
-- MongoDB: 2.6 through 4.1
+- MongoDB: 3.6+
 
-Clone this repo or download the source archive then *cd* into the source
+Install GenieACS from NPM:
-directory and install the required dependencies:
+
+    sudo npm install -g --unsafe-perm genieacs
+
+To build from source instead, clone this repo or download the source archive
+then *cd* into the source directory then run:
 
     npm install
-
-Now build:
-
     npm run build
 
-Finally, run the following services from the build that's generated under
+Finally, run the following services (found under `./dist/bin/` if building from
-'dist' directory:
+source):
 
 ### genieacs-cwmp
 
 This is the service that the CPEs will communicate with. It listens on port
 7547 by default. Configure the ACS URL in your devices accordingly.
 
-    ./dist/bin/genieacs-cwmp
-
 You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
 as a dummy TR-069 simulator if you don't have a CPE at hand.
 
@@ -47,40 +42,36 @@ This is the northbound interface module. It exposes a REST API on port 7557 by
 default. This one is only required if you have an external system integrating
 with GenieACS using this API.
 
-    ./dist/bin/genieacs-nbi
-
 ### genieacs-fs
 
 This is the file server from which the CPEs will download firmware images and
-such.
+such. It listens on port 7567 by default.
-
-    ./dist/bin/genieacs-fs
 
 ### genieacs-ui
 
 This serves the web based user interface. It listens on port 3000 by default.
+You must pass *--ui-jwt-secret* argument to supply the secret key used for
+signing browser cookies:
 
-    ./dist/bin/genieacs-ui --ui-jwt-secret secret
+    genieacs-ui --ui-jwt-secret secret
-
-The argument *--ui-jwt-secret* supplies the key used for signing browser
-cookies.
 
 The UI has plenty of configuration options. When you open GenieACS's UI in a
 browser you'll be greeted with a database initialization wizard to help you
 populate some initial configuration.
 
+Visit [docs.genieacs.com](https://docs.genieacs.com) for more documentation and
+a complete installation guide for production deployments.
+
 ## Support
 
 The [forum](https://forum.genieacs.com) is a good place to get guidance and
-help from the community. Head on over and join the conversation! In addition,
+help from the community. Head on over and join the conversation!
-the [wiki](https://github.com/genieacs/genieacs/wiki) provides useful
-documentation and tips from GenieACS users.
 
-For commercial support options and professional services, please visit
+For commercial support options, please visit
 [genieacs.com](https://genieacs.com/support/).
 
 ## License
 
-Copyright 2013-2019 GenieACS Inc. GenieACS is released under the [AGPLv3
+Copyright 2013-2020 GenieACS Inc. GenieACS is released under the [AGPLv3
 license
 terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

ava.config.js

@@ -1,5 +1,4 @@
 export default {
-  "compileEnhancements": false,
   "extensions": ["ts"],
   "require": ["esm", "ts-node/register/transpile-only"]
 };

bin/genieacs-cwmp.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -31,29 +29,32 @@ import { version as VERSION } from "../package.json";
 
 logger.init("cwmp", VERSION);
 
-const SERVICE_ADDRESS = config.get("CWMP_INTERFACE");
+const SERVICE_ADDRESS = config.get("CWMP_INTERFACE") as string;
-const SERVICE_PORT = config.get("CWMP_PORT");
+const SERVICE_PORT = config.get("CWMP_PORT") as number;
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
-  cache.disconnect();
+  Promise.all([
-  extensions.killAll();
+    db.disconnect(),
-  cluster.worker.disconnect();
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES");
+  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES") as number;
 
   logger.info({
     message: `genieacs-cwmp starting`,
     pid: process.pid,
-    version: VERSION
+    version: VERSION,
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -61,7 +62,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
@@ -70,60 +71,53 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
   });
 } else {
-  const ssl = {
+  const key = config.get("CWMP_SSL_KEY") as string;
-    key: config.get("CWMP_SSL_KEY"),
+  const cert = config.get("CWMP_SSL_CERT") as string;
-    cert: config.get("CWMP_SSL_CERT")
+
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    onConnection: cwmp.onConnection,
+    timeout: 30000,
+    keepAliveTimeout: 0,
   };
 
-  const keepAliveTimeout = config.get("CWMP_KEEP_ALIVE_TIMEOUT") as number;
+  process.on("uncaughtException", (err) => {
-
-  process.on("uncaughtException", err => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid
+      pid: process.pid,
     });
-    server
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
   });
 
-  process.on("SIGINT", () => {
+  const initPromise = Promise.all([db.connect(), cache.connect()])
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
     .then(() => {
-      server.start(
+      server.start(options, cwmp.listener);
-        SERVICE_PORT,
-        SERVICE_ADDRESS,
-        ssl,
-        cwmp.listener,
-        cwmp.onConnection,
-        keepAliveTimeout
-      );
     })
-    .catch(err => {
+    .catch((err) => {
       setTimeout(() => {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
 }

bin/genieacs-ext.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -35,8 +33,8 @@ function errorToFault(err: Error): Fault {
     message: err.message,
     detail: {
       name: err.name,
-      message: err.message
+      message: err.message,
-    }
+    },
   };
 
   if (err.stack) {
@@ -56,16 +54,16 @@ function errorToFault(err: Error): Fault {
   return fault;
 }
 
-process.on("uncaughtException", err => {
+process.on("uncaughtException", (err) => {
   const fault = errorToFault(err);
-  jobs.forEach(jobId => {
+  jobs.forEach((jobId) => {
     process.send([jobId, fault, null]);
   });
   jobs.clear();
   process.disconnect();
 });
 
-process.on("message", message => {
+process.on("message", (message) => {
   jobs.add(message[0]);
 
   if (!script) {
@@ -79,7 +77,7 @@ process.on("message", message => {
   if (!script[funcName]) {
     const fault = {
       code: "ext",
-      message: `No such function '${funcName}' in extension '${fileName}'`
+      message: `No such function '${funcName}' in extension '${fileName}'`,
     };
     process.send([message[0], fault, null]);
     return;
@@ -93,4 +91,6 @@ process.on("message", message => {
 });
 
 // Ignore SIGINT
-process.on("SIGINT", () => {});
+process.on("SIGINT", () => {
+  // Ignore
+});

bin/genieacs-fs.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -30,13 +28,16 @@ import { version as VERSION } from "../package.json";
 
 logger.init("fs", VERSION);
 
-const SERVICE_ADDRESS = config.get("FS_INTERFACE");
+const SERVICE_ADDRESS = config.get("FS_INTERFACE") as string;
-const SERVICE_PORT = config.get("FS_PORT");
+const SERVICE_PORT = config.get("FS_PORT") as number;
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
-  cache.disconnect();
+  Promise.all([
-  cluster.worker.disconnect();
+    db.disconnect(),
+    cache.disconnect(),
+    cluster.worker.disconnect(),
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
@@ -44,12 +45,12 @@ function exitWorkerUngracefully(): void {
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES");
+  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES") as number;
 
   logger.info({
     message: `genieacs-fs starting`,
     pid: process.pid,
-    version: VERSION
+    version: VERSION,
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -57,7 +58,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
@@ -66,51 +67,60 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
   });
 } else {
-  const ssl = {
+  const key = config.get("FS_SSL_KEY") as string;
-    key: config.get("FS_SSL_KEY"),
+  const cert = config.get("FS_SSL_CERT") as string;
-    cert: config.get("FS_SSL_CERT")
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
-  process.on("uncaughtException", err => {
+  let stopping = false;
+
+  process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid
+      pid: process.pid,
     });
-    server
+    stopping = true;
-      .stop()
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
   });
 
-  process.on("SIGINT", () => {
+  const _listener = (req, res): void => {
-    server
+    if (stopping) res.setHeader("Connection", "close");
-      .stop()
+    listener(req, res);
-      .then(exitWorkerGracefully)
+  };
-      .catch(exitWorkerUngracefully);
-  });
 
-  process.on("SIGTERM", () => {
+  const initPromise = Promise.all([db.connect(), cache.connect()])
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
+      server.start(options, _listener);
     })
-    .catch(err => {
+    .catch((err) => {
       setTimeout(() => {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
 }

bin/genieacs-nbi.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -31,29 +29,32 @@ import { version as VERSION } from "../package.json";
 
 logger.init("nbi", VERSION);
 
-const SERVICE_ADDRESS = config.get("NBI_INTERFACE");
+const SERVICE_ADDRESS = config.get("NBI_INTERFACE") as string;
-const SERVICE_PORT = config.get("NBI_PORT");
+const SERVICE_PORT = config.get("NBI_PORT") as number;
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
-  cache.disconnect();
+  Promise.all([
-  extensions.killAll();
+    db.disconnect(),
-  cluster.worker.disconnect();
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES");
+  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES") as number;
 
   logger.info({
     message: `genieacs-nbi starting`,
     pid: process.pid,
-    version: VERSION
+    version: VERSION,
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -61,7 +62,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
@@ -70,51 +71,60 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
   });
 } else {
-  const ssl = {
+  const key = config.get("NBI_SSL_KEY") as string;
-    key: config.get("NBI_SSL_KEY"),
+  const cert = config.get("NBI_SSL_CERT") as string;
-    cert: config.get("NBI_SSL_CERT")
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
-  process.on("uncaughtException", err => {
+  let stopping = false;
+
+  process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid
+      pid: process.pid,
     });
-    server
+    stopping = true;
-      .stop()
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
   });
 
-  process.on("SIGINT", () => {
+  const _listener = (req, res): void => {
-    server
+    if (stopping) res.setHeader("Connection", "close");
-      .stop()
+    listener(req, res);
-      .then(exitWorkerGracefully)
+  };
-      .catch(exitWorkerUngracefully);
-  });
 
-  process.on("SIGTERM", () => {
+  const initPromise = Promise.all([db.connect(), cache.connect()])
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
+      server.start(options, _listener);
     })
-    .catch(err => {
+    .catch((err) => {
       setTimeout(() => {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
 }

bin/genieacs-ui.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -32,30 +30,33 @@ import { version as VERSION } from "../package.json";
 
 logger.init("ui", VERSION);
 
-const SERVICE_ADDRESS = config.get("UI_INTERFACE");
+const SERVICE_ADDRESS = config.get("UI_INTERFACE") as string;
-const SERVICE_PORT = config.get("UI_PORT");
+const SERVICE_PORT = config.get("UI_PORT") as number;
 
 function exitWorkerGracefully(): void {
-  db.disconnect();
+  setTimeout(exitWorkerUngracefully, 5000).unref();
-  db2.disconnect();
+  Promise.all([
-  cache.disconnect();
+    db.disconnect(),
-  extensions.killAll();
+    db2.disconnect(),
-  cluster.worker.disconnect();
+    cache.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES");
+  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES") as number;
 
   logger.info({
     message: `genieacs-ui starting`,
     pid: process.pid,
-    version: VERSION
+    version: VERSION,
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -63,7 +64,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
@@ -72,47 +73,32 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
 
     cluster.stop();
   });
 } else {
-  const ssl = {
+  const key = config.get("UI_SSL_KEY") as string;
-    key: config.get("UI_SSL_KEY"),
+  const cert = config.get("UI_SSL_CERT") as string;
-    cert: config.get("UI_SSL_CERT")
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
   let stopping = false;
 
-  process.on("uncaughtException", err => {
+  process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid
+      pid: process.pid,
     });
     stopping = true;
-    server
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGINT", () => {
-    stopping = true;
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    stopping = true;
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
   });
 
   const _listener = (req, res): void => {
@@ -120,13 +106,27 @@ if (!cluster.worker) {
     listener(req, res);
   };
 
-  Promise.all([db2.connect(), cache.connect()])
+  const initPromise = Promise.all([db2.connect(), cache.connect()])
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+      server.start(options, _listener);
     })
-    .catch(err => {
+    .catch((err) => {
       setTimeout(() => {
         throw err;
       });
     });
+
+  process.on("SIGINT", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
 }

build/build.ts

@@ -19,13 +19,13 @@
 
 import * as path from "path";
 import * as fs from "fs";
-import { promisify } from "util";
 import { rollup, WarningHandler } from "rollup";
-import rollupReplace from "rollup-plugin-replace";
+import rollupJson from "@rollup/plugin-json";
-import rollupJson from "rollup-plugin-json";
+import typescript from "@rollup/plugin-typescript";
-import typescript from "rollup-plugin-typescript";
 import { terser } from "rollup-plugin-terser";
-import webpack from "webpack";
+import nodeResolve from "@rollup/plugin-node-resolve";
+import commonjs from "@rollup/plugin-commonjs";
+import webpack, { Configuration } from "webpack";
 import postcss from "postcss";
 import postcssImport from "postcss-import";
 import postcssPresetEnv from "postcss-preset-env";
@@ -55,7 +55,6 @@ const externals = [
   "crypto",
   "util",
   "mongodb",
-  "libxmljs",
   "vm",
   "later",
   "parsimmon",
@@ -64,6 +63,7 @@ const externals = [
   "child_process",
   "dgram",
   "url",
+  "iconv-lite",
   "koa",
   "koa-router",
   "koa-compress",
@@ -77,7 +77,10 @@ const externals = [
   "yaml",
   "codemirror",
   "codemirror/mode/javascript/javascript",
-  "codemirror/mode/yaml/yaml"
+  "codemirror/mode/yaml/yaml",
+  "ipaddr.js",
+  "jsbi",
+  "espresso-iisojs",
 ];
 
 function rmDirSync(dirPath): void {
@@ -122,9 +125,7 @@ function generateSymbol(id: string, svgStr: string): string {
     }
   }
   const symbolBody = xml.children[0].children
-    .map(c => {
+    .map((c) => xmlTostring(c))
-      return xmlTostring(c);
-    })
     .join("");
   return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
 }
@@ -146,7 +147,7 @@ async function init(): Promise<void> {
   delete packageJson["devDependencies"];
   packageJson["scripts"] = {
     install: packageJson["scripts"].install,
-    configure: packageJson["scripts"].configure
+    configure: packageJson["scripts"].configure,
   };
   packageJson["version"] = `${packageJson["version"]}+${BUILD_METADATA}`;
   fs.writeFileSync(
@@ -172,7 +173,7 @@ async function copyStatic(): Promise<void> {
     "README.md",
     "CHANGELOG.md",
     "public/logo.svg",
-    "public/favicon.png"
+    "public/favicon.png",
   ];
 
   for (const file of files) {
@@ -193,35 +194,31 @@ async function generateCss(): Promise<void> {
       stage: 3,
       features: {
         "nesting-rules": true,
-        "color-mod-function": true
+        "color-mod-function": true,
-      }
+      },
     }),
-    cssnano
+    cssnano,
   ]).process(cssIn, { from: cssInPath, to: cssOutPath });
   fs.writeFileSync(cssOutPath, cssOut.css);
 }
 
 async function generateToolsJs(): Promise<void> {
   for (const bin of ["dump-data-model"]) {
-    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}`);
+    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}.ts`);
     const outputFile = path.resolve(OUTPUT_DIR, `tools/${bin}`);
     const bundle = await rollup({
       input: inputFile,
       external: externals,
       acorn: {
-        allowHashBang: true
+        allowHashBang: true,
       },
       plugins: [
-        rollupReplace({
-          delimiters: ["", ""],
-          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
-        }),
         typescript({
           tsconfig: "./tsconfig.json",
-          include: [`tools/${bin}`, "lib/**/*.ts"]
+          include: [`tools/${bin}.ts`, "lib/**/*.ts"],
         }),
-        MODE === "production" ? terser() : null
+        MODE === "production" ? terser() : null,
-      ]
+      ],
     });
 
     await bundle.write({
@@ -230,7 +227,7 @@ async function generateToolsJs(): Promise<void> {
       sourcemap: "inline",
       sourcemapExcludeSources: true,
       banner: "#!/usr/bin/env node",
-      file: outputFile
+      file: outputFile,
     });
 
     // Mark as executable
@@ -245,27 +242,24 @@ async function generateBackendJs(): Promise<void> {
     "genieacs-ext",
     "genieacs-nbi",
     "genieacs-fs",
-    "genieacs-ui"
+    "genieacs-ui",
   ]) {
-    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}`);
+    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}.ts`);
     const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
     const bundle = await rollup({
       input: inputFile,
       external: externals,
       acorn: {
-        allowHashBang: true
+        allowHashBang: true,
       },
       treeshake: {
         propertyReadSideEffects: false,
-        pureExternalModules: true
+        moduleSideEffects: false,
       },
       plugins: [
-        rollupReplace({
-          delimiters: ["", ""],
-          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
-        }),
         rollupJson({ preferConst: true }),
         {
+          name: "",
           resolveId: (importee, importer) => {
             if (importee.endsWith("/package.json")) {
               const p = path.resolve(path.dirname(importer), importee);
@@ -273,14 +267,14 @@ async function generateBackendJs(): Promise<void> {
                 return path.resolve(OUTPUT_DIR, "package.json");
             }
             return null;
-          }
+          },
         },
         typescript({
           tsconfig: "./tsconfig.json",
-          include: [`bin/${bin}`, "lib/**/*.ts"]
+          include: [`bin/${bin}.ts`, "lib/**/*.ts"],
         }),
-        MODE === "production" ? terser() : null
+        MODE === "production" ? terser() : null,
-      ]
+      ],
     });
 
     await bundle.write({
@@ -289,7 +283,7 @@ async function generateBackendJs(): Promise<void> {
       sourcemap: "inline",
       sourcemapExcludeSources: true,
       banner: "#!/usr/bin/env node",
-      file: outputFile
+      file: outputFile,
     });
 
     // Mark as executable
@@ -302,37 +296,54 @@ async function generateFrontendJs(): Promise<void> {
   const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
   const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
 
+  const inlineDeps = ["mithril", "parsimmon", "jsbi", "espresso-iisojs"];
   const bundle = await rollup({
     input: inputFile,
-    external: externals,
+    external: externals.filter((e) => !inlineDeps.includes(e)),
     plugins: [
       rollupJson({ preferConst: true }),
-      typescript({ tsconfig: "./tsconfig.json" })
+      {
+        name: "",
+        resolveId: function (importee, importer) {
+          if (importee.endsWith("/bigint")) {
+            return this.resolve(importee + "-jsbi", importer);
+          } else if (importee === "espresso-iisojs") {
+            return this.resolve(
+              "espresso-iisojs/dist/espresso-iisojs-jsbi.mjs",
+              importer
+            );
+          }
+          return null;
+        },
+      },
+      typescript({ tsconfig: "./tsconfig.json" }),
+      nodeResolve(),
+      commonjs(),
     ],
     inlineDynamicImports: true,
     treeshake: {
       propertyReadSideEffects: false,
-      pureExternalModules: true
+      moduleSideEffects: false,
     },
     onwarn: ((warning, warn) => {
       // Ignore circular dependency warnings
       if (warning.code !== "CIRCULAR_DEPENDENCY") warn(warning);
-    }) as WarningHandler
+    }) as WarningHandler,
   });
 
   await bundle.write({
     preferConst: true,
-    format: "esm",
+    format: "es",
     sourcemap: "inline",
     sourcemapExcludeSources: true,
-    file: outputFile
+    file: outputFile,
   });
 
-  const webpackConf = {
+  const webpackConf: Configuration = {
-    mode: MODE,
+    mode: MODE as "development" | "production",
     entry: outputFile,
     resolve: {
-      aliasFields: ["module"]
+      aliasFields: ["module"],
     },
     devtool: "nosources-source-map",
     module: {
@@ -340,18 +351,23 @@ async function generateFrontendJs(): Promise<void> {
         {
           test: /\.js$/,
           use: ["source-map-loader"],
-          enforce: "pre"
+          enforce: "pre",
-        }
+        },
-      ]
+      ],
     },
     output: {
       path: path.resolve(OUTPUT_DIR, "public"),
-      filename: "app.js"
+      filename: "app.js",
-    }
+    },
   };
 
-  const stats = await promisify(webpack)(webpackConf);
+  return new Promise((resolve, reject) => {
+    webpack(webpackConf, (err, stats) => {
+      if (err) return reject(err);
       process.stdout.write(stats.toString({ colors: true }) + "\n");
+      resolve();
+    });
+  });
 }
 
 async function generateIconsSprite(): Promise<void> {
@@ -378,13 +394,15 @@ init()
       generateIconsSprite(),
       generateToolsJs(),
       generateBackendJs(),
-      generateFrontendJs()
+      generateFrontendJs(),
     ])
-      .then(() => {})
+      .then(() => {
-      .catch(err => {
+        // Ignore
+      })
+      .catch((err) => {
         process.stderr.write(err.stack + "\n");
       });
   })
-  .catch(err => {
+  .catch((err) => {
     process.stderr.write(err.stack + "\n");
   });

docs/administration-faq.rst

@@ -0,0 +1,64 @@
+.. _administration-faq:
+
+Administration FAQ
+==================
+
+.. _administration-faq-duplicate-log-entries:
+
+Duplicate log entries when using :func:`log()` function
+-------------------------------------------------------
+
+Because GenieACS uses a full fledged scripting language for device
+configuration, the only way to guarantee that it has satisfied the 'desired
+state' is by repeatedly executing the script until there's no more
+discrepancies with the current device state. Though it may seem like this will
+cause duplicate requests going to the device, this isn't actually the case
+because device configuration are stated declaratively and that the scripts
+themselves are pure functions in the context of a session (e.g. Date.now()
+always returns the same value within the session).
+
+To illustrate with an example, consider the following script:
+
+.. code:: javascript
+
+  log("Executing script");
+  declare("Device.param", null, {value: 1});
+  commit();
+  declare("Device.param", null, {value: 2});
+
+This will set the value of the 'Device.param' to 1, then to 2. Then as the
+script is run again the value is set back to 1 and so on. A stable state will
+never be reached so GenieACS will execute the script a few times until it gives
+up and throws a fault. This is an edge case that should be avoided. A more
+typical case is where the script is run once or twice. Essentially if an
+execution doesn't result in any request to the CPE or a change in the data
+model then a stable state is deemed to have been reached.
+
+Configurations not pushed to device after factory reset
+---------------------------------------------------------
+
+After a device is reset to its factory default state, the cached data model in
+GenieACS's database needs to be invalidated to force rediscovery. Ensure the
+following lines are called on ``0 BOOTSTRAP`` event:
+
+.. code:: javascript
+
+  const now = Date.now();
+
+  // Clear cached data model to force a refresh
+  clear("Device", now);
+  clear("InternetGatewayDevice", now);
+
+
+Most device parameters are missing
+----------------------------------
+
+For performance reasons (server, client, and network), GenieACS by default only
+fetches parts of the data model that are necessary to satisfy the declarations
+in your provision scripts. Create declarations for any parameters you need
+fetched by default.
+
+If you're unsure and want to explore the available parameters exposed by the
+device, refresh the root parameter (e.g. ``InternetGatewayDevice``) from
+GenieACS's UI. You typically only need to do that one time for a given CPE
+model.

docs/api-reference.rst

@@ -415,6 +415,15 @@ parameterValues key. For example:
   -X POST \
   --data '{"name":"addObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection"}'
 
+``deleteObject``
+~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name":"deleteObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection.1"}'
+
 ``reboot``
 ~~~~~~~~~~
 

docs/environment-variables.rst

@@ -3,4 +3,215 @@
 Environment Variables
 =====================
 
-TODO
+Configuring GenieACS services can be done through the following environment
+variables:
+
+.. attention::
+
+  All GenieACS environment variables must be prefixed with ``GENIEACS_``.
+
+MONGODB_CONNECTION_URL
+  MongoDB connection string.
+
+  Default: ``mongodb://127.0.0.1/genieacs``
+
+EXT_DIR
+  The directory from which to look up extension scripts.
+
+  Default: ``<installation dir>/config/ext``
+
+EXT_TIMEOUT
+  Timeout (in milliseconds) to allow for calls to extensions to return a
+  response.
+
+  Default: ``3000``
+
+DEBUG_FILE
+  File to dump CPE debug log.
+
+  Default: unset
+
+DEBUG_FORMAT
+  Debug log format. Valid values are 'yaml' and 'json'.
+
+  Default: ``yaml``
+
+LOG_FORMAT
+  The format used for the log entries in ``CWMP_LOG_FILE``, ``NBI_LOG_FILE``,
+  ``FS_LOG_FILE``, and ``UI_LOG_FILE``. Possible values are ``simple`` and
+  ``json``.
+
+  Default: ``simple``
+
+ACCESS_LOG_FORMAT
+  The format used for the log entries in ``CWMP_ACCESS_LOG_FILE``,
+  ``NBI_ACCESS_LOG_FILE``, ``FS_ACCESS_LOG_FILE``, and ``UI_ACCESS_LOG_FILE``.
+  Possible values are ``simple`` and ``json``.
+
+  Default: ``simple``
+
+CWMP_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-cwmp. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+CWMP_PORT
+  The TCP port that genieacs-cwmp listens on.
+
+  Default: ``7547``
+
+CWMP_INTERFACE
+  The network interface that genieacs-cwmp binds to.
+
+  Default: ``0.0.0.0``
+
+CWMP_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_LOG_FILE
+  File to log process related events for genieacs-cwmp. If omitted, logs will
+  go to stderr.
+
+  Default: unset
+
+CWMP_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-cwmp. If omitted, logs will go to
+  stdout.
+
+  Default: unest
+
+NBI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-nbi. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+NBI_PORT
+  The TCP port that genieacs-nbi listens on.
+
+  Default: ``7557``
+
+NBI_INTERFACE
+  The network interface that genieacs-nbi binds to.
+
+  Default: ``0.0.0.0``
+
+NBI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unest
+
+NBI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+NBI_LOG_FILE
+  File to log process related events for genieacs-nbi. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+NBI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-nbi. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-fs. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+FS_PORT
+  The TCP port that genieacs-fs listens on.
+
+  Default: ``7567``
+
+FS_INTERFACE
+  The network interface that genieacs-fs binds to.
+
+  Default: ``0.0.0.0``
+
+FS_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_LOG_FILE
+  File to log process related events for genieacs-fs. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+FS_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-fs. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_URL_PREFIX
+  The URL prefix (e.g. 'https://example.com:7657/') to use when generating the
+  file URL for TR-069 Download requests. Set this if genieacs-fs and
+  genieacs-cwmp are behind a proxy or running on different servers.
+
+  Default: auto generated based on the hostname from the ACS URL, FS_PORT
+  config, and whether or not SSL is enabled for genieacs-fs.
+
+UI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-ui. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+UI_PORT
+  The TCP port that genieacs-ui listens on.
+
+  Default: ``3000``
+
+UI_INTERFACE
+  The network interface that genieacs-ui binds to.
+
+  Default: ``0.0.0.0``
+
+UI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_LOG_FILE
+  File to log process related events for genieacs-ui. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+UI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-ui. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+UI_JWT_SECRET
+  The key used for signing JWT tokens that are stored in browser cookies. The
+  string can be up to 64 characters in length.
+
+  Default: unset

docs/index.rst

@@ -30,6 +30,7 @@ Welcome to GenieACS's documentation!
 
   provisions
   virtual-parameters
+  administration-faq
 
 .. raw:: latex
 

docs/installation-guide.rst

@@ -34,7 +34,7 @@ Install GenieACS
 
   .. parsed-literal::
 
-    sudo npm install -g genieacs@\ |release|
+    sudo npm install -g --unsafe-perm genieacs@\ |release|
 
 .. topic:: Installing from source
 
@@ -68,11 +68,12 @@ Configure systemd
 
   .. code:: bash
 
-    GENIEACS_CWMP_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-cmwp-access.log
+    GENIEACS_CWMP_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-cwmp-access.log
     GENIEACS_NBI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-nbi-access.log
     GENIEACS_FS_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-fs-access.log
     GENIEACS_UI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-ui-access.log
     GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
+    GENIEACS_EXT_DIR=/opt/genieacs/ext
     GENIEACS_UI_JWT_SECRET=secret
   
   Set file ownership and permissions:
@@ -100,11 +101,16 @@ Configure systemd
   :file:`/var/log/genieacs/` while process logs go to *journald*. Use
   ``journalctl`` command to view process logs.
 
+  .. attention::
+
+    If the command :command:`systemctl edit --force --full` fails, you can
+    create the unit file manually.
+
   1. Run the following command to create ``genieacs-cwmp`` service:
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-cwmp
+      sudo systemctl edit --force --full genieacs-cwmp
     
     Then paste the following in the editor and save:
 
@@ -117,8 +123,7 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-cwmp
+      ExecStart=/usr/bin/genieacs-cwmp
-      KillMode=process
 
       [Install]
       WantedBy=default.target
@@ -127,7 +132,7 @@ Configure systemd
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-nbi
+      sudo systemctl edit --force --full genieacs-nbi
     
     Then paste the following in the editor and save:
 
@@ -140,8 +145,7 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-nbi
+      ExecStart=/usr/bin/genieacs-nbi
-      KillMode=process
 
       [Install]
       WantedBy=default.target
@@ -150,7 +154,7 @@ Configure systemd
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-fs
+      sudo systemctl edit --force --full genieacs-fs
     
     Then paste the following in the editor and save:
 
@@ -163,8 +167,7 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-fs
+      ExecStart=/usr/bin/genieacs-fs
-      KillMode=process
 
       [Install]
       WantedBy=default.target
@@ -173,7 +176,7 @@ Configure systemd
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-ui
+      sudo systemctl edit --force --full genieacs-ui
     
     Then paste the following in the editor and save:
 
@@ -186,19 +189,18 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-ui
+      ExecStart=/usr/bin/genieacs-ui
-      KillMode=process
 
       [Install]
       WantedBy=default.target
 
 .. topic:: Configure log file rotation using logrotate
 
-  Save the following under :file:`/etc/logrotate.d/genieacs`
+  Save the following as :file:`/etc/logrotate.d/genieacs`
 
   .. code::
   
-    /var/log/genieacs/*.{log,yaml} {
+    /var/log/genieacs/*.log /var/log/genieacs/*.yaml {
         daily
         rotate 30
         compress

docs/provisions.rst

@@ -128,6 +128,14 @@ script filename while second argument is the function name within that script.
 Any remaining arguments will be passed to that function. See :ref:`extensions`
 for more details.
 
+``log(message)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Prints out a string in genieacs-cwmp's access log. It's meant to be used for
+debugging. Note that you may see multiple log entries as the script can be
+executed multiple times in a session. See :ref:`this FAQ
+<administration-faq-duplicate-log-entries>`.
+
 .. _path-format:
 
 Path format

docs/virtual-parameters.rst

@@ -18,8 +18,8 @@ differences between the scripts of provisions and virtual parameters are:
 
 .. code:: javascript
 
-  // [<current attr timestamps>, <current attr values>, <declared attr timestamps, declared attr values>]
+  // [<declared attr timestamps, declared attr values>, <current attr timestamps>, <current attr values>]
-  [{path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}, {path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}]
+  [{path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}, {path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}]
 
 - Virtual parameter scripts must return an object containing the attributes of
   this parameter.
@@ -69,9 +69,9 @@ Expose an external value as a virtual parameter
 
   // Example: Expose an external value as a virtual parameter
   let serial = declare("DeviceID.SerialNumber", {value: 1});
-  if (values.value) {
+  if (args[1].value) {
-    ext("example-ext", "set", serial.value[0], values.value[0]);
+    ext("example-ext", "set", serial.value[0], args[1].value[0]);
-    return {writable: true, value: [values.value[0], "xsd:string"]};
+    return {writable: true, value: [args[1].value[0], "xsd:string"]};
   }
   else {
     let v = ext("example-ext", "get", serial.value[0]);
@@ -85,8 +85,8 @@ Create an editable virtual parameter for WPA passphrase
 
   // Example: Create an editable virtual parameter for WPA passphrase
   let m = "";
-  if (VALUES && VALUES.value) {
+  if (args[1].value) {
-    m = VALUES.value[0];
+    m = args[1].value[0];
     declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", null, {value: m});
     declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", null, {value: m});
   }

lib/api-functions.ts

@@ -24,85 +24,108 @@ import * as cache from "./cache";
 import {
   getCurrentSnapshot,
   getConfig,
-  getConfigExpression
+  getConfigExpression,
 } from "./local-cache";
 import {
   httpConnectionRequest,
-  udpConnectionRequest
+  udpConnectionRequest,
 } from "./connection-request";
 import { Expression, Task } from "./types";
+import { flattenDevice } from "./mongodb-functions";
+import { evaluate } from "./common/expression";
 
-export async function connectionRequest(deviceId): Promise<void> {
+export async function connectionRequest(
-  const options = {
+  deviceId: string,
-    projection: {
+  device?: Record<string, { value?: [boolean | number | string, string] }>
-      _deviceId: 1,
+): Promise<void> {
-      "Device.ManagementServer.ConnectionRequestURL._value": 1,
+  if (!device) {
-      "Device.ManagementServer.UDPConnectionRequestAddress._value": 1,
+    const res = await db.devicesCollection.findOne({ _id: deviceId });
-      "Device.ManagementServer.ConnectionRequestUsername._value": 1,
+    if (!res) throw new Error("No such device");
-      "Device.ManagementServer.ConnectionRequestPassword._value": 1,
+    device = flattenDevice(res);
-      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value": 1,
-      "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress._value": 1,
-      "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername._value": 1,
-      "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword._value": 1
   }
-  };
 
-  const device = await db.devicesCollection.findOne({ _id: deviceId }, options);
+  let connectionRequestUrl, udpConnectionRequestAddress, username, password;
-  if (!device) throw new Error("No such device");
 
-  let managementServer,
+  if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
-    connectionRequestUrl,
+    connectionRequestUrl = (device[
-    udpConnectionRequestAddress,
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL"
-    username,
+    ].value || [""])[0];
-    password;
+    udpConnectionRequestAddress = ((
-  if (device.Device)
+      device[
-    // TR-181 data model
+        "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
-    managementServer = device.Device.ManagementServer;
+      ] || {}
-  // TR-098 data model
+    ).value || [""])[0];
-  else managementServer = device.InternetGatewayDevice.ManagementServer;
+    username = ((
-
+      device[
-  if (managementServer.ConnectionRequestURL)
+        "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
-    connectionRequestUrl = managementServer.ConnectionRequestURL._value;
+      ] || {}
-  if (managementServer.UDPConnectionRequestAddress) {
+    ).value || [""])[0];
-    udpConnectionRequestAddress =
+    password = ((
-      managementServer.UDPConnectionRequestAddress._value;
+      device[
+        "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword"
+      ] || {}
+    ).value || [""])[0];
+  } else {
+    connectionRequestUrl = (device[
+      "Device.ManagementServer.ConnectionRequestURL"
+    ].value || [""])[0];
+    udpConnectionRequestAddress = ((
+      device["Device.ManagementServer.UDPConnectionRequestAddress"] || {}
+    ).value || [""])[0];
+    username = ((
+      device["Device.ManagementServer.ConnectionRequestUsername"] || {}
+    ).value || [""])[0];
+    password = ((
+      device["Device.ManagementServer.ConnectionRequestPassword"] || {}
+    ).value || [""])[0];
   }
-  if (managementServer.ConnectionRequestUsername)
-    username = managementServer.ConnectionRequestUsername._value;
-  if (managementServer.ConnectionRequestPassword)
-    password = managementServer.ConnectionRequestPassword._value;
 
-  const context = {
+  const remoteAddress = parse(connectionRequestUrl).host;
-    id: device["_id"],
+
-    serialNumber: device["_deviceId"]["SerialNumber"],
+  const evalCallback = (exp): Expression => {
-    productClass: device["_deviceId"]["ProductClass"],
+    if (!Array.isArray(exp)) return exp;
-    oui: device["_deviceId"]["OUI"],
+    if (exp[0] === "PARAM" && typeof exp[1] === "string") {
-    remoteAddress: connectionRequestUrl
+      let name = exp[1];
-      ? parse(connectionRequestUrl).host
+      if (name === "id") name = "DeviceID.ID";
-      : null,
+      else if (name === "serialNumber") name = "DeviceID.SerialNumber";
-    username: username || "",
+      else if (name === "productClass") name = "DeviceID.ProductClass";
-    password: password || ""
+      else if (name === "oui") name = "DeviceID.OUI";
+      else if (name === "remoteAddress") return remoteAddress;
+      else if (name === "username") return username;
+      else if (name === "password") return password;
+
+      const p = device[name];
+      if (p && p.value) return p.value[0];
+    } else if (exp[0] === "FUNC") {
+      if (exp[1] === "REMOTE_ADDRESS") return remoteAddress;
+      else if (exp[1] === "USERNAME") return username;
+      else if (exp[1] === "PASSWORD") return password;
+    }
+    return exp;
   };
 
   const snapshot = await getCurrentSnapshot();
   const now = Date.now();
   const UDP_CONNECTION_REQUEST_PORT = +getConfig(
     snapshot,
-    "cwmp.udpConnectionRequestAuth",
+    "cwmp.udpConnectionRequestPort",
-    context,
+    {},
-    now
+    now,
+    evalCallback
   );
   const CONNECTION_REQUEST_TIMEOUT = +getConfig(
     snapshot,
     "cwmp.connectionRequestTimeout",
-    context,
+    {},
-    now
+    now,
+    evalCallback
   );
   const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = !!getConfig(
     snapshot,
     "cwmp.connectionRequestAllowBasicAuth",
-    context,
+    {},
-    now
+    now,
+    evalCallback
   );
   let authExp: Expression = getConfigExpression(
     snapshot,
@@ -114,18 +137,19 @@ export async function connectionRequest(deviceId): Promise<void> {
       "FUNC",
       "AUTH",
       ["PARAM", "username"],
-      ["PARAM", "password"]
+      ["PARAM", "password"],
     ] as Expression;
   }
 
-  const debug = !!getConfig(snapshot, "cwmp.debug", context, now);
+  authExp = evaluate(authExp, {}, now, evalCallback);
+
+  const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
 
   let udpProm;
   if (udpConnectionRequestAddress) {
     udpProm = udpConnectionRequest(
       udpConnectionRequestAddress,
       authExp,
-      context,
       UDP_CONNECTION_REQUEST_PORT,
       debug,
       deviceId
@@ -136,7 +160,6 @@ export async function connectionRequest(deviceId): Promise<void> {
     await httpConnectionRequest(
       connectionRequestUrl,
       authExp,
-      context,
       CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
       CONNECTION_REQUEST_TIMEOUT,
       debug,
@@ -148,8 +171,12 @@ export async function connectionRequest(deviceId): Promise<void> {
   }
 }
 
-export async function watchTask(deviceId, taskId, timeout): Promise<string> {
+export async function watchTask(
-  await new Promise(resolve => setTimeout(resolve, 500));
+  deviceId: string,
+  taskId: string,
+  timeout: number
+): Promise<string> {
+  await new Promise((resolve) => setTimeout(resolve, 500));
 
   const task = await db.tasksCollection.findOne(
     { _id: taskId },
@@ -159,7 +186,7 @@ export async function watchTask(deviceId, taskId, timeout): Promise<string> {
 
   const q = { _id: `${deviceId}:task_${taskId}` };
   const fault = await db.faultsCollection.findOne(q, {
-    projection: { _id: 1 }
+    projection: { _id: 1 },
   });
   if (fault) return "fault";
 
@@ -171,7 +198,7 @@ export async function watchTask(deviceId, taskId, timeout): Promise<string> {
 function sanitizeTask(task): void {
   task.timestamp = new Date(task.timestamp || Date.now());
   if (task.expiry) {
-    if (common.typeOf(task.expiry) === common.DATE_TYPE || isNaN(task.expiry))
+    if (task.expiry instanceof Date || isNaN(task.expiry))
       task.expiry = new Date(task.expiry);
     else task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000);
   }
@@ -209,6 +236,10 @@ function sanitizeTask(task): void {
       break;
 
     case "refreshObject":
+      if (typeof task.objectName !== "string")
+        throw new Error("Missing 'objectName' property");
+      break;
+
     case "deleteObject":
       if (typeof task.objectName !== "string" || !task.objectName.length)
         throw new Error("Missing 'objectName' property");
@@ -226,11 +257,14 @@ function sanitizeTask(task): void {
       break;
 
     case "download":
+      // genieacs-gui sends file ID instead of fileName and fileType
+      if (!task.file) {
         if (typeof task.fileType !== "string" || !task.fileType.length)
           throw new Error("Missing 'fileType' property");
 
         if (typeof task.fileName !== "string" || !task.fileName.length)
           throw new Error("Missing 'fileName' property");
+      }
 
       if (
         task.targetFileName != null &&
@@ -238,13 +272,26 @@ function sanitizeTask(task): void {
       )
         throw new Error("Invalid 'targetFileName' property");
       break;
+
+    case "provisions":
+      if (
+        !Array.isArray(task.provisions) ||
+        !task.provisions.every((arr) =>
+          arr.every(
+            (s) =>
+              s == null || ["boolean", "number", "string"].includes(typeof s)
+          )
+        )
+      )
+        throw new Error("Invalid 'provisions' property");
+      break;
   }
 
   return task;
 }
 
-export async function insertTasks(tasks): Promise<Task[]> {
+export async function insertTasks(tasks: any[]): Promise<Task[]> {
-  if (tasks && common.typeOf(tasks) !== common.ARRAY_TYPE) tasks = [tasks];
+  if (tasks && !Array.isArray(tasks)) tasks = [tasks];
   else if (!tasks || tasks.length === 0) return tasks || [];
 
   for (const task of tasks) {
@@ -252,7 +299,7 @@ export async function insertTasks(tasks): Promise<Task[]> {
     if (task.uniqueKey) {
       await db.tasksCollection.deleteOne({
         device: task.device,
-        uniqueKey: task.uniqueKey
+        uniqueKey: task.uniqueKey,
       });
     }
   }
@@ -260,20 +307,20 @@ export async function insertTasks(tasks): Promise<Task[]> {
   return tasks;
 }
 
-export async function deleteDevice(deviceId): Promise<void> {
+export async function deleteDevice(deviceId: string): Promise<void> {
   await Promise.all([
     db.tasksCollection.deleteMany({ device: deviceId }),
     db.devicesCollection.deleteOne({ _id: deviceId }),
     db.faultsCollection.deleteMany({
       _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
-      }
+      },
     }),
     db.operationsCollection.deleteMany({
       _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
-      }
+      },
     }),
-    cache.del(`${deviceId}_tasks_faults_operations`)
+    cache.del(`${deviceId}_tasks_faults_operations`),
   ]);
 }

lib/auth.ts

@@ -19,15 +19,17 @@
 
 import { createHash, randomBytes, pbkdf2 } from "crypto";
 
-function parseHeaderFeilds(str: string): {} {
+function parseHeaderFeilds(str: string): Record<string, string> {
   const res = {};
   const parts = str.split(",");
 
-  let part;
+  let part: string;
   while ((part = parts.shift()) != null) {
     const name = part.split("=", 1)[0];
-    if (name.length === part.length)
+    if (name.length === part.length) {
+      if (!part.trim()) continue;
       throw new Error("Unable to parse auth header");
+    }
 
     let value = part.slice(name.length + 1);
     if (!/^\s*"/.test(value)) {
@@ -50,7 +52,9 @@ function parseHeaderFeilds(str: string): {} {
   return res;
 }
 
-export function parseAuthorizationHeader(authHeader): { method: string } {
+export function parseAuthorizationHeader(
+  authHeader: string
+): { method: string } {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };
@@ -72,7 +76,9 @@ export function parseAuthorizationHeader(authHeader): { method: string } {
   return res;
 }
 
-export function parseWwwAuthenticateHeader(authHeader): {} {
+export function parseWwwAuthenticateHeader(
+  authHeader: string
+): Record<string, string> {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };
@@ -80,7 +86,7 @@ export function parseWwwAuthenticateHeader(authHeader): {} {
   return res;
 }
 
-export function basic(username, password): string {
+export function basic(username: string, password: string): string {
   return "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
 }
 
@@ -97,20 +103,12 @@ export function digest(
   nc?: string | Buffer
 ): string {
   const ha1 = createHash("md5");
-  ha1
+  ha1.update(username).update(":").update(realm).update(":").update(password);
-    .update(username)
-    .update(":")
-    .update(realm)
-    .update(":")
-    .update(password);
   // TODO support "MD5-sess" algorithm directive
   const ha1d = ha1.digest("hex");
 
   const ha2 = createHash("md5");
-  ha2
+  ha2.update(httpMethod).update(":").update(uri);
-    .update(httpMethod)
-    .update(":")
-    .update(uri);
 
   if (qop === "auth-int") {
     const bodyHash = createHash("md5")
@@ -122,10 +120,7 @@ export function digest(
   const ha2d = ha2.digest("hex");
 
   const hash = createHash("md5");
-  hash
+  hash.update(ha1d).update(":").update(nonce);
-    .update(ha1d)
-    .update(":")
-    .update(nonce);
   if (qop) {
     hash
       .update(":")
@@ -146,7 +141,7 @@ export function solveDigest(
   uri: string | Buffer,
   httpMethod: string | Buffer,
   body: string | Buffer,
-  authHeader
+  authHeader: Record<string, string>
 ): string {
   const cnonce = randomBytes(8).toString("hex");
   const nc = "00000001";
@@ -183,7 +178,7 @@ export function solveDigest(
   return authString;
 }
 
-export function generateSalt(length): Promise<string> {
+export function generateSalt(length: number): Promise<string> {
   return new Promise((resolve, reject) => {
     randomBytes(length, (err, rand) => {
       if (err) return void reject(err);
@@ -192,7 +187,7 @@ export function generateSalt(length): Promise<string> {
   });
 }
 
-export function hashPassword(pass, salt): Promise<string> {
+export function hashPassword(pass: string, salt: string): Promise<string> {
   return new Promise((resolve, reject) => {
     pbkdf2(pass, salt, 10000, 128, "sha512", (err, hash) => {
       if (err) return void reject(err);

lib/cache.ts

@@ -22,16 +22,17 @@ import * as config from "./config";
 
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let mongoClient: MongoClient;
+let clientPromise: Promise<MongoClient>;
 let mongoCollection: Collection;
 let mongoTimeOffset = 0;
 
 export async function connect(): Promise<void> {
   const MONGODB_CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
-  mongoClient = await MongoClient.connect(MONGODB_CONNECTION_URL, {
+  clientPromise = MongoClient.connect(MONGODB_CONNECTION_URL, {
-    useNewUrlParser: true
+    useNewUrlParser: true,
+    useUnifiedTopology: true,
   });
-  const db = mongoClient.db();
+  const db = (await clientPromise).db();
   mongoCollection = db.collection("cache");
   await mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
   const now = Date.now();
@@ -40,10 +41,10 @@ export async function connect(): Promise<void> {
 }
 
 export async function disconnect(): Promise<void> {
-  if (mongoClient) await mongoClient.close();
+  if (clientPromise) await (await clientPromise).close();
 }
 
-export async function get(key): Promise<any> {
+export async function get(key: string | string[]): Promise<any> {
   const expire = new Date(Date.now() - mongoTimeOffset);
   if (Array.isArray(key)) {
     const res = await mongoCollection.find({ _id: { $in: key } }).toArray();
@@ -54,7 +55,7 @@ export async function get(key): Promise<any> {
     });
 
     const values = [];
-    res.forEach(r => {
+    res.forEach((r) => {
       if (r["expire"] > expire) values[indices[r["_id"]]] = r["value"];
     });
 
@@ -62,11 +63,11 @@ export async function get(key): Promise<any> {
   } else {
     const res = await mongoCollection.findOne({ _id: { $in: [key] } });
     if (res && res["expire"] > expire) return res["value"];
-    return res;
+    return null;
   }
 }
 
-export async function del(key): Promise<void> {
+export async function del(key: string | string[]): Promise<void> {
   if (Array.isArray(key))
     await mongoCollection.deleteMany({ _id: { $in: key } });
   else await mongoCollection.deleteOne({ _id: key });
@@ -85,7 +86,7 @@ export async function set(
   );
 }
 
-export async function pop(key): Promise<any> {
+export async function pop(key: string): Promise<any> {
   const res = await mongoCollection.findOneAndDelete({ _id: key });
 
   if (
@@ -98,16 +99,17 @@ export async function pop(key): Promise<any> {
   return null;
 }
 
-export async function lock(lockName, ttl): Promise<Function> {
+export async function lock(
-  const token = Math.random()
+  lockName: string,
-    .toString(36)
+  ttl: number
-    .slice(2);
+): Promise<(extendTtl?: number) => Promise<void>> {
+  const token = Math.random().toString(36).slice(2);
 
-  async function unlockOrExtend(extendTtl): Promise<void> {
+  async function unlockOrExtend(extendTtl?: number): Promise<void> {
     if (!extendTtl) {
       const res = await mongoCollection.deleteOne({
         _id: lockName,
-        value: token
+        value: token,
       });
       if (res["result"]["n"] !== 1) throw new Error("Lock expired");
     } else {
@@ -131,7 +133,7 @@ export async function lock(lockName, ttl): Promise<Function> {
     );
   } catch (err) {
     if (err && err.code === 11000) {
-      await new Promise(resolve => setTimeout(resolve, 200));
+      await new Promise((resolve) => setTimeout(resolve, 200));
       return lock(lockName, ttl);
     }
   }

lib/cluster.ts

@@ -22,14 +22,26 @@ import { cpus } from "os";
 import * as logger from "./logger";
 
 let respawnTimestamp = 0;
-let tooManyCrashesTimestamp = 0;
+let crashes: number[] = [];
+
+function fork(): cluster.Worker {
+  const w = cluster.fork();
+  w.on("error", (err: NodeJS.ErrnoException) => {
+    // Avoid exception when attempting to kill the process just as it's exiting
+    if (err.code !== "EPIPE") throw err;
+    setTimeout(() => {
+      if (!w.isDead()) throw err;
+    }, 50);
+  });
+  return w;
+}
 
 function restartWorker(worker, code, signal): void {
   const msg = {
     message: "Worker died",
     pid: worker.process.pid,
     exitCode: null,
-    signal: null
+    signal: null,
   };
 
   if (code != null) msg.exitCode = code;
@@ -39,32 +51,49 @@ function restartWorker(worker, code, signal): void {
   logger.error(msg);
 
   const now = Date.now();
-  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  crashes.push(now);
-  if (respawnTimestamp === now) {
-    tooManyCrashesTimestamp = now;
-    cluster.fork();
-    return;
-  }
 
-  if (now - tooManyCrashesTimestamp > 60000) {
+  let min1 = 0,
+    min2 = 0,
+    min3 = 0;
+
+  crashes = crashes.filter((n) => {
+    if (n > now - 60000) ++min1;
+    else if (n > now - 120000) ++min2;
+    else if (n > now - 180000) ++min3;
+    else return false;
+    return true;
+  });
+
+  if (min1 > 5 && min2 > 5 && min3 > 5) {
     process.exitCode = 1;
     cluster.removeListener("exit", restartWorker);
     for (const pid in cluster.workers) cluster.workers[pid].kill();
 
     logger.error({
       message: "Too many crashes, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
     return;
   }
 
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    fork();
+    return;
+  }
+
   setTimeout(() => {
     if (process.exitCode) return;
-    cluster.fork();
+    fork();
   }, respawnTimestamp - now);
 }
 
-export function start(workerCount, servicePort, serviceAddress): void {
+export function start(
+  workerCount: number,
+  servicePort: number,
+  serviceAddress: string
+): void {
   cluster.on("listening", (worker, address) => {
     if (
       (address.addressType === 4 || address.addressType === 6) &&
@@ -75,7 +104,7 @@ export function start(workerCount, servicePort, serviceAddress): void {
         message: "Worker listening",
         pid: worker.process.pid,
         address: address.address,
-        port: address.port
+        port: address.port,
       });
     }
   });
@@ -84,7 +113,7 @@ export function start(workerCount, servicePort, serviceAddress): void {
 
   if (!workerCount) workerCount = Math.max(2, cpus().length);
 
-  for (let i = 0; i < workerCount; ++i) cluster.fork();
+  for (let i = 0; i < workerCount; ++i) fork();
 }
 
 export function stop(): void {

lib/common.ts

@@ -17,22 +17,12 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-export const UNDEFINED_TYPE = "[object Undefined]";
+export function generateDeviceId(
-export const NULL_TYPE = "[object Null]";
+  deviceIdStruct: Record<string, string>
-export const BOOLEAN_TYPE = "[object Boolean]";
+): string {
-export const NUMBER_TYPE = "[object Number]";
-export const STRING_TYPE = "[object String]";
-export const ARRAY_TYPE = "[object Array]";
-export const OBJECT_TYPE = "[object Object]";
-export const REGEXP_TYPE = "[object RegExp]";
-export const DATE_TYPE = "[object Date]";
-
-export const typeOf = (obj): string => Object.prototype.toString.call(obj);
-
-export function generateDeviceId(deviceIdStruct): string {
   // Escapes everything except alphanumerics and underscore
   function esc(str): string {
-    return str.replace(/[^A-Za-z0-9_]/g, chr => {
+    return str.replace(/[^A-Za-z0-9_]/g, (chr) => {
       const buf = Buffer.from(chr, "utf8");
       let rep = "";
       for (const b of buf) rep += "%" + b.toString(16).toUpperCase();
@@ -54,6 +44,20 @@ export function generateDeviceId(deviceIdStruct): string {
 }
 
 // Source: http://stackoverflow.com/a/6969486
-export function escapeRegExp(str): string {
+export function escapeRegExp(str: string): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
+
+export function encodeTag(tag: string): string {
+  return encodeURIComponent(tag)
+    .replace(
+      /[!~*'()]/g,
+      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
+    )
+    .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")
+    .replace(/%/g, "0x");
+}
+
+export function decodeTag(tag: string): string {
+  return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%"));
+}

lib/common/authorizer.ts

@@ -18,23 +18,25 @@
  */
 
 import { PermissionSet, Expression } from "../types";
-import { evaluate } from "./expression";
+import { evaluate, or } from "./expression";
 
 export default class Authorizer {
   private permissionSets: PermissionSet[];
   private validatorCache: WeakMap<
-    object,
+    any,
     (mutationType, mutation, any) => boolean
   >;
   private hasAccessCache: Map<string, boolean>;
+  private getFilterCache: Map<string, Expression>;
 
-  public constructor(permissionSets) {
+  public constructor(permissionSets: PermissionSet[]) {
     this.permissionSets = permissionSets;
     this.validatorCache = new WeakMap();
     this.hasAccessCache = new Map();
+    this.getFilterCache = new Map();
   }
 
-  public hasAccess(resourceType, access): boolean {
+  public hasAccess(resourceType: string, access: number): boolean {
     const cacheKey = `${resourceType}-${access}`;
     if (this.hasAccessCache.has(cacheKey))
       return this.hasAccessCache.get(cacheKey);
@@ -55,10 +57,29 @@ export default class Authorizer {
     return has;
   }
 
+  public getFilter(resourceType: string, access: number): Expression {
+    const cacheKey = `${resourceType}-${access}`;
+    if (this.getFilterCache.has(cacheKey))
+      return this.getFilterCache.get(cacheKey);
+
+    let filter: Expression = null;
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (perm[resourceType]) {
+          if (perm[resourceType].access >= access)
+            filter = or(filter, perm[resourceType].filter);
+        }
+      }
+    }
+
+    this.getFilterCache.set(cacheKey, filter);
+    return filter;
+  }
+
   public getValidator(
-    resourceType,
+    resourceType: string,
-    resource
+    resource: unknown
-  ): (mutationType: string, mutation: any, args: any) => boolean {
+  ): (mutationType: string, mutation?: any, args?: any) => boolean {
     if (this.validatorCache.has(resource))
       return this.validatorCache.get(resource);
 
@@ -87,7 +108,7 @@ export default class Authorizer {
         mutation,
         resourceType,
         object: resource,
-        options: any
+        options: any,
       };
 
       const valueFunction = (paramName): any => {
@@ -103,7 +124,8 @@ export default class Authorizer {
             if (value == null) break;
           }
         } else if (object[entry]) {
-          value = object[entry][paramName];
+          if (paramName) value = object[entry][paramName];
+          else value = object[entry];
         }
 
         return value;
@@ -120,4 +142,8 @@ export default class Authorizer {
     this.validatorCache.set(resource, validator);
     return validator;
   }
+
+  public getPermissionSets(): PermissionSet[] {
+    return this.permissionSets;
+  }
 }

lib/common/bigint-jsbi.ts

@@ -0,0 +1,108 @@
+/**
+ * Copyright 2013-2020  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import JSBI from "jsbi";
+
+export function and(a: JSBI, b: JSBI): JSBI {
+  return JSBI.bitwiseAnd(a, b);
+}
+
+export function or(a: JSBI, b: JSBI): JSBI {
+  return JSBI.bitwiseOr(a, b);
+}
+
+export function xor(a: JSBI, b: JSBI): JSBI {
+  return JSBI.bitwiseXor(a, b);
+}
+
+export function not(a: JSBI): JSBI {
+  return JSBI.bitwiseNot(a);
+}
+
+export function lshift(a: JSBI, b: JSBI): JSBI {
+  return JSBI.leftShift(a, b);
+}
+
+export function rshift(a: JSBI, b: JSBI): JSBI {
+  return JSBI.signedRightShift(a, b);
+}
+
+export function add(a: JSBI, b: JSBI): JSBI {
+  return JSBI.add(a, b);
+}
+
+export function sub(a: JSBI, b: JSBI): JSBI {
+  return JSBI.subtract(a, b);
+}
+
+export function mul(a: JSBI, b: JSBI): JSBI {
+  return JSBI.multiply(a, b);
+}
+
+export function div(a: JSBI, b: JSBI): JSBI {
+  return JSBI.divide(a, b);
+}
+
+export function exp(a: JSBI, b: JSBI): JSBI {
+  return JSBI.exponentiate(a, b);
+}
+
+export function rem(a: JSBI, b: JSBI): JSBI {
+  return JSBI.remainder(a, b);
+}
+
+export function toNumber(a: JSBI): number {
+  return JSBI.toNumber(a);
+}
+
+export function eq(a: JSBI, b: JSBI): boolean {
+  return JSBI.equal(a, b);
+}
+
+export function ne(a: JSBI, b: JSBI): boolean {
+  return JSBI.notEqual(a, b);
+}
+
+export function lt(a: JSBI, b: JSBI): boolean {
+  return JSBI.lessThan(a, b);
+}
+
+export function lte(a: JSBI, b: JSBI): boolean {
+  return JSBI.lessThanOrEqual(a, b);
+}
+
+export function gt(a: JSBI, b: JSBI): boolean {
+  return JSBI.greaterThan(a, b);
+}
+
+export function gte(a: JSBI, b: JSBI): boolean {
+  return JSBI.greaterThanOrEqual(a, b);
+}
+
+export function asUintN(a: number, b: JSBI): JSBI {
+  return JSBI.asUintN(a, b);
+}
+
+export function asIntN(a: number, b: JSBI): JSBI {
+  return JSBI.asIntN(a, b);
+}
+
+const _BigInt = JSBI.BigInt;
+type _bigint = JSBI;
+export { _BigInt as BigInt, _bigint as bigint };

lib/common/bigint.ts

@@ -0,0 +1,106 @@
+/**
+ * Copyright 2013-2020  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+export function and(a: bigint, b: bigint): bigint {
+  return a & b;
+}
+
+export function or(a: bigint, b: bigint): bigint {
+  return a | b;
+}
+
+export function xor(a: bigint, b: bigint): bigint {
+  return a ^ b;
+}
+
+export function not(a: bigint): bigint {
+  return ~a;
+}
+
+export function lshift(a: bigint, b: bigint): bigint {
+  return a << b;
+}
+
+export function rshift(a: bigint, b: bigint): bigint {
+  return a >> b;
+}
+
+export function add(a: bigint, b: bigint): bigint {
+  return a + b;
+}
+
+export function sub(a: bigint, b: bigint): bigint {
+  return a - b;
+}
+
+export function mul(a: bigint, b: bigint): bigint {
+  return a * b;
+}
+
+export function div(a: bigint, b: bigint): bigint {
+  return a / b;
+}
+
+export function exp(a: bigint, b: bigint): bigint {
+  return a ** b;
+}
+
+export function rem(a: bigint, b: bigint): bigint {
+  return a % b;
+}
+
+export function toNumber(a: bigint): number {
+  return Number(a);
+}
+
+export function eq(a: bigint, b: bigint): boolean {
+  return a === b;
+}
+
+export function ne(a: bigint, b: bigint): boolean {
+  return a !== b;
+}
+
+export function lt(a: bigint, b: bigint): boolean {
+  return a < b;
+}
+
+export function lte(a: bigint, b: bigint): boolean {
+  return a <= b;
+}
+
+export function gt(a: bigint, b: bigint): boolean {
+  return a > b;
+}
+
+export function gte(a: bigint, b: bigint): boolean {
+  return a >= b;
+}
+
+export function asUintN(a: number, b: bigint): bigint {
+  return BigInt.asUintN(a, b);
+}
+
+export function asIntN(a: number, b: bigint): bigint {
+  return BigInt.asIntN(a, b);
+}
+
+const _BigInt = BigInt;
+type _bigint = bigint;
+export { _BigInt as BigInt, _bigint as bigint };

lib/common/expression-cnf.ts

@@ -1,285 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { map, parseLikePattern } from "./expression-parser";
-import { Expression } from "../types";
-
-type CNF = number[][];
-
-function or(cnf1: CNF, cnf2: CNF): CNF {
-  if (cnf1.length === 0) return cnf2;
-  else if (cnf2.length === 0) return cnf1;
-
-  if (cnf1.length > cnf2.length) {
-    const c = cnf1;
-    cnf1 = cnf2;
-    cnf2 = c;
-  }
-
-  const res = [];
-  for (let i = 0; i < cnf1.length; ++i)
-    for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j]));
-
-  return res;
-}
-
-function likePatternIncludes(pat1, pat2, idx1 = 0, idx2 = 0): boolean {
-  while (idx1 < pat1.length && idx2 < pat2.length) {
-    if (pat1[idx1] === "\\%") {
-      for (let i = idx2; i <= pat2.length; ++i)
-        if (likePatternIncludes(pat1, pat2, idx1 + 1, i)) return true;
-      return false;
-    } else if (pat2[idx2] === "\\%") {
-      if (pat1[idx1] !== "\\%") return false;
-    } else if (pat1[idx1] === "\\_") {
-      // Ignore
-    } else if (pat2[idx2] === "\\_") {
-      return false;
-    } else if (pat2[idx2] !== pat1[idx1]) {
-      return false;
-    }
-    ++idx1;
-    ++idx2;
-  }
-
-  if (idx1 === pat1.length && idx2 === pat2.length) return true;
-  return false;
-}
-
-export function booleanCnf(exp: Expression): { vars: number; clauses: CNF } {
-  const ranges = new Map();
-  const likePatterns = new Map();
-  map(exp, e => {
-    if (!Array.isArray(e)) return e;
-    if (e[0] === "LIKE" || e[0] === "NOT LIKE") {
-      if (
-        typeof e[2] === "string" &&
-        (e.length >= 4 || (e[3] != null && !Array.isArray(e[3])))
-      ) {
-        const key = JSON.stringify(e[1]);
-        let s = likePatterns.get(key);
-        if (!s) likePatterns.set(key, (s = new Map()));
-        s.set(`${e[3] || ""}:${e[2]}`, parseLikePattern(e[2], e[3]));
-      }
-    } else if ([">", ">=", "<", "<=", "=", "<>"].includes(e[0])) {
-      if (!Array.isArray(e[2])) {
-        const key = `${JSON.stringify(e[1])}:${typeof e[2]}`;
-        let s = ranges.get(key);
-        if (!s) ranges.set(key, (s = new Set()));
-        s.add(e[2]);
-      } else if (!Array.isArray(e[1])) {
-        const key = `${JSON.stringify(e[2])}:${typeof e[1]}`;
-        let s = ranges.get(key);
-        if (!s) ranges.set(key, (s = new Set()));
-        s.add(e[2]);
-      }
-    }
-    return e;
-  });
-
-  for (const [k, v] of ranges) {
-    if (k.endsWith(":number")) {
-      ranges.set(
-        k,
-        Array.from(v as number[]).sort((a: number, b: number) => a - b)
-      );
-    } else {
-      ranges.set(k, Array.from(v).sort());
-    }
-  }
-
-  const expressions: Map<string, number> = new Map();
-  const mutuallyExclusive: Map<string, Set<number>> = new Map();
-
-  function getVariable(lhs, op, rhs = ""): number {
-    const key = `${lhs}:${op}:${rhs}`;
-    let f = expressions.get(key);
-    if (!f) {
-      expressions.set(key, (f = expressions.size + 1));
-      let m = mutuallyExclusive.get(lhs);
-      if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
-      m.add(f);
-      if (rhs || op !== "null") m.add(getVariable(lhs, "null"));
-    }
-    return f;
-  }
-
-  function getComparisonVariables(
-    lhs,
-    from,
-    fromInclusive,
-    to,
-    toInclusive
-  ): number[] {
-    const type = from != null ? typeof from : typeof to;
-    const range = ranges.get(`${lhs}:${type}`);
-    const vars = [];
-
-    if (from == null) vars.push(`${lhs}:--${range[0]}`);
-
-    for (let i = 0; i < range.length; ++i) {
-      if (
-        i > 0 &&
-        (from == null || range[i] > from) &&
-        (to == null || range[i] <= to)
-      )
-        vars.push(`${lhs}:${range[i - 1]}--${range[i]}`);
-
-      if (
-        (range[i] > from && range[i] < to) ||
-        (range[i] === from && fromInclusive) ||
-        (range[i] === to && toInclusive)
-      )
-        vars.push(`${lhs}:${range[i]}`);
-    }
-
-    if (to == null) vars.push(`${lhs}:${range[range.length - 1]}--`);
-
-    return vars.map(key => {
-      let f = expressions.get(key);
-      if (!f) {
-        expressions.set(key, (f = expressions.size + 1));
-        let m = mutuallyExclusive.get(lhs);
-        if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
-        m.add(f);
-      }
-      return f;
-    });
-  }
-
-  function getLikeVariables(lhs, pat, esc): number[] {
-    const vars = [];
-    const pats = likePatterns.get(lhs);
-    pat = pats.get(`${esc || ""}:${pat}`);
-    for (const p of pats.values()) {
-      if (pat === p || likePatternIncludes(pat, p))
-        vars.push(`${lhs}:like:${JSON.stringify(p)}`);
-    }
-
-    return vars.map(key => {
-      let f = expressions.get(key);
-      if (!f) expressions.set(key, (f = expressions.size + 1));
-      return f;
-    });
-  }
-
-  interface ComparisonOperators {
-    [name: string]: (v: string) => [string, boolean, string, boolean];
-  }
-
-  const comparisonOperators: ComparisonOperators = {
-    "=": v => [v, true, v, true],
-    ">": v => [v, false, null, null],
-    ">=": v => [v, true, null, null],
-    "<": v => [null, null, v, false],
-    "<=": v => [null, null, v, true]
-  };
-
-  function recursive(clause, negate): CNF {
-    let op = clause[0];
-
-    if ((op === "AND" && !negate) || (op === "OR" && negate)) {
-      let res = [];
-      for (let i = 1; i < clause.length; ++i)
-        res = res.concat(recursive(clause[i], negate));
-      return res;
-    } else if ((op === "OR" && !negate) || (op === "AND" && negate)) {
-      let res = [];
-      for (let i = 1; i < clause.length; ++i)
-        res = or(res, recursive(clause[i], negate));
-      return res;
-    } else if (op === "NOT") {
-      return recursive(clause[1], !negate);
-    } else if (op === "<>") {
-      const vars = getComparisonVariables(
-        JSON.stringify(clause[1]),
-        ...comparisonOperators["="](clause[2])
-      );
-      if (!negate) return vars.map(x => [0 - x]);
-      else return [vars];
-    } else if (op === "IS NULL") {
-      const v = getVariable(JSON.stringify(clause[1]), "null");
-      if (negate) return [[0 - v]];
-      else return [[v]];
-    } else if (op === "IS NOT NULL") {
-      const v = getVariable(JSON.stringify(clause[1]), "null");
-      if (!negate) return [[0 - v]];
-      else return [[v]];
-    } else if (op in comparisonOperators) {
-      if (!Array.isArray(clause[2])) {
-        const vars = getComparisonVariables(
-          JSON.stringify(clause[1]),
-          ...comparisonOperators[op](clause[2])
-        );
-        if (negate) return vars.map(x => [0 - x]);
-        else return [vars];
-      } else if (!Array.isArray(clause[1])) {
-        op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
-        const vars = getComparisonVariables(
-          JSON.stringify(clause[2]),
-          ...comparisonOperators[op](clause[1])
-        );
-        if (negate) return vars.map(x => [0 - x]);
-        else return [vars];
-      } else {
-        let lhs = JSON.stringify(clause[1]);
-        let rhs = JSON.stringify(clause[2]);
-        // For consistency
-        if (rhs > lhs) {
-          const t = rhs;
-          rhs = lhs;
-          lhs = t;
-          op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
-        }
-
-        const v = getVariable(lhs, op, rhs);
-        if (negate) return [[0 - v]];
-        else return [[v]];
-      }
-    } else if (
-      (op === "LIKE" || op === "NOT LIKE") &&
-      !Array.isArray(clause[2]) &&
-      clause[2] != null &&
-      (clause.length >= 4 || clause[3] != null)
-    ) {
-      if (op === "NOT LIKE") negate = !negate;
-      const vars = getLikeVariables(
-        JSON.stringify(clause[1]),
-        clause[2],
-        clause[3]
-      );
-      if (negate) return vars.map(x => [0 - x]);
-      else return [vars];
-    } else {
-      const v = getVariable(JSON.stringify(clause), "");
-      if (negate) return [[0 - v]];
-      else return [[v]];
-    }
-  }
-
-  const cnf = recursive(exp, false);
-
-  for (const m of mutuallyExclusive.values()) {
-    const ar: number[] = Array.from(m);
-    for (let i = 0; i < ar.length; ++i)
-      for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]);
-  }
-
-  return { vars: expressions.size, clauses: cnf };
-}

lib/common/expression-parser.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import * as parsimmon from "parsimmon";
+import parsimmon from "parsimmon";
 import { Expression } from "../types";
 
 // Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
@@ -27,7 +27,7 @@ function interpretEscapes(str): string {
     f: "\f",
     n: "\n",
     r: "\r",
-    t: "\t"
+    t: "\t",
   };
   return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
     const type = escape.charAt(0);
@@ -40,7 +40,10 @@ function interpretEscapes(str): string {
   });
 }
 
-export function map(exp, callback): Expression {
+export function map(
+  exp: Expression,
+  callback: (e: Expression) => Expression
+): Expression {
   if (!Array.isArray(exp)) return callback(exp);
 
   let clone;
@@ -55,7 +58,10 @@ export function map(exp, callback): Expression {
   return callback(clone || exp);
 }
 
-export async function mapAsync(exp, callback): Promise<Expression> {
+export async function mapAsync(
+  exp: Expression,
+  callback: (e: Expression) => Promise<Expression>
+): Promise<Expression> {
   if (!Array.isArray(exp)) return callback(exp);
 
   let clone;
@@ -70,7 +76,10 @@ export async function mapAsync(exp, callback): Promise<Expression> {
   return callback(clone || exp);
 }
 
-function binaryLeft(operatorsParser, nextParser): parsimmon.Parser<{}> {
+function binaryLeft(
+  operatorsParser: parsimmon.Parser<string>,
+  nextParser: parsimmon.Parser<Expression>
+): parsimmon.Parser<Expression> {
   return parsimmon.seqMap(
     nextParser,
     parsimmon.seq(operatorsParser, nextParser).many(),
@@ -86,7 +95,7 @@ function binaryLeft(operatorsParser, nextParser): parsimmon.Parser<{}> {
 }
 
 const lang = parsimmon.createLanguage({
-  ComparisonOperator: function() {
+  ComparisonOperator: function () {
     return parsimmon
       .alt(
         parsimmon.string(">="),
@@ -98,13 +107,10 @@ const lang = parsimmon.createLanguage({
       )
       .skip(parsimmon.optWhitespace);
   },
-  LikeOperator: function() {
+  LikeOperator: function () {
     return parsimmon
       .alt(
-        parsimmon
+        parsimmon.regexp(/like/i).result("LIKE").desc("LIKE"),
-          .regexp(/like/i)
-          .result("LIKE")
-          .desc("LIKE"),
         parsimmon
           .regexp(/not\s+like/i)
           .result("NOT LIKE")
@@ -113,7 +119,7 @@ const lang = parsimmon.createLanguage({
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
   },
-  IsNullOperator: function() {
+  IsNullOperator: function () {
     return parsimmon
       .alt(
         parsimmon
@@ -128,7 +134,7 @@ const lang = parsimmon.createLanguage({
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
   },
-  NotOperator: function() {
+  NotOperator: function () {
     return parsimmon
       .regexp(/not/i)
       .result("NOT")
@@ -136,7 +142,7 @@ const lang = parsimmon.createLanguage({
       .skip(parsimmon.optWhitespace)
       .desc("NOT");
   },
-  AndOperator: function() {
+  AndOperator: function () {
     return parsimmon
       .regexp(/and/i)
       .result("AND")
@@ -144,7 +150,7 @@ const lang = parsimmon.createLanguage({
       .skip(parsimmon.optWhitespace)
       .desc("AND");
   },
-  OrOperator: function() {
+  OrOperator: function () {
     return parsimmon
       .regexp(/or/i)
       .result("OR")
@@ -152,7 +158,7 @@ const lang = parsimmon.createLanguage({
       .skip(parsimmon.optWhitespace)
       .desc("OR");
   },
-  Parameter: function(r) {
+  Parameter: function (r) {
     return parsimmon
       .alt(
         parsimmon.regexp(/[a-zA-Z0-9_.*-]+/),
@@ -162,48 +168,43 @@ const lang = parsimmon.createLanguage({
         )
       )
       .atLeast(1)
-      .map(x => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
+      .map((x) => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
       .skip(parsimmon.optWhitespace)
       .desc("parameter");
   },
-  StringValueSql: function() {
+  StringValueSql: function () {
     return parsimmon
       .regexp(/'([^']*)'/, 1)
       .atLeast(1)
       .skip(parsimmon.optWhitespace)
-      .map(s => s.join("'"))
+      .map((s) => s.join("'"))
       .desc("string");
   },
-  StringValueJs: function() {
+  StringValueJs: function () {
     return parsimmon
       .regexp(/"((?:\\.|.)*?)"/, 1)
       .skip(parsimmon.optWhitespace)
       .map(interpretEscapes)
       .desc("string");
   },
-  NumberValue: function() {
+  NumberValue: function () {
     return parsimmon
       .regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/)
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace)
       .map(Number)
       .desc("number");
   },
-  BooleanValue: function() {
+  BooleanValue: function () {
     return parsimmon
       .alt(
-        parsimmon
+        parsimmon.regexp(/true/i).result(true).desc("TRUE"),
-          .regexp(/true/i)
+        parsimmon.regexp(/false/i).result(false).desc("FALSE")
-          .result(true)
-          .desc("TRUE"),
-        parsimmon
-          .regexp(/false/i)
-          .result(false)
-          .desc("FALSE")
       )
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
   },
-  NullValue: function() {
+  NullValue: function () {
     return parsimmon
       .regexp(/null/i)
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
@@ -211,32 +212,73 @@ const lang = parsimmon.createLanguage({
       .result(null)
       .desc("NULL");
   },
-  FuncValue: function(r) {
+  FuncValue: function (r) {
     return parsimmon.seqMap(
       parsimmon
         .regexp(/([a-zA-Z0-9_]+)/, 1)
         .skip(parsimmon.optWhitespace)
         .desc("function"),
-      r.ValueExpression.sepBy(
+      r.ExpressionList.wrap(
-        parsimmon.string(",").skip(parsimmon.optWhitespace)
-      ).wrap(
         parsimmon.string("(").skip(parsimmon.optWhitespace),
         parsimmon.string(")").skip(parsimmon.optWhitespace)
       ),
       (f, args) => ["FUNC", f.toUpperCase()].concat(args)
     );
   },
-  Value: function(r) {
+  WhenPair: function (r) {
+    return parsimmon.seq(
+      parsimmon
+        .regexp(/when/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("WHEN")
+        .then(r.Expression),
+      parsimmon
+        .regexp(/then/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("THEN")
+        .then(r.Expression)
+    );
+  },
+  CaseStatement: function (r) {
+    return parsimmon.seqMap(
+      parsimmon
+        .regexp(/case/i)
+        .result("CASE")
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("CASE"),
+      r.WhenPair.many(),
+      parsimmon
+        .regexp(/else/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("ELSE")
+        .then(r.Expression)
+        .map((e) => [[true, e]])
+        .fallback(null)
+        .skip(
+          parsimmon
+            .regex(/end/i)
+            .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        )
+        .skip(parsimmon.optWhitespace),
+      (...arr) => arr.flat(2)
+    );
+  },
+  Value: function (r) {
     return parsimmon.alt(
       r.NullValue,
       r.BooleanValue,
       r.NumberValue,
       r.StringValueSql,
       r.StringValueJs,
-      r.FuncValue
+      r.FuncValue,
+      r.CaseStatement
     );
   },
-  ValueExpression: function(r) {
+  ValueExpression: function (r) {
     return binaryLeft(
       parsimmon.string("||").skip(parsimmon.optWhitespace),
       binaryLeft(
@@ -259,7 +301,7 @@ const lang = parsimmon.createLanguage({
       )
     );
   },
-  Comparison: function(r) {
+  Comparison: function (r) {
     return parsimmon.alt(
       parsimmon.seqMap(r.ValueExpression, r.IsNullOperator, (p, o) => [o, p]),
       parsimmon.seqMap(
@@ -289,8 +331,16 @@ const lang = parsimmon.createLanguage({
       )
     );
   },
-  Expression: function(r) {
+  ExpressionList: function (r) {
-    function unary(operatorsParser, nextParser): parsimmon.Parser<{}> {
+    return r.Expression.sepBy(
+      parsimmon.string(",").skip(parsimmon.optWhitespace)
+    );
+  },
+  Expression: function (r) {
+    function unary(
+      operatorsParser: parsimmon.Parser<string>,
+      nextParser: parsimmon.Parser<Expression>
+    ): parsimmon.Parser<Expression> {
       return parsimmon.seq(operatorsParser, nextParser).or(nextParser);
     }
 
@@ -301,7 +351,7 @@ const lang = parsimmon.createLanguage({
         unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
       )
     ).trim(parsimmon.optWhitespace);
-  }
+  },
 });
 
 export function parse(str: string): Expression {
@@ -309,7 +359,12 @@ export function parse(str: string): Expression {
   return lang.Expression.tryParse(str);
 }
 
-export function stringify(exp, level = 0): string {
+export function parseList(str: string): Expression[] {
+  if (!str) return [];
+  return lang.ExpressionList.tryParse(str);
+}
+
+export function stringify(exp: Expression, level = 0): string {
   if (!Array.isArray(exp)) return JSON.stringify(exp);
 
   const opLevels = {
@@ -330,7 +385,7 @@ export function stringify(exp, level = 0): string {
     "+": 31,
     "-": 31,
     "*": 32,
-    "/": 32
+    "/": 32,
   };
 
   const op = exp[0].toUpperCase();
@@ -344,7 +399,7 @@ export function stringify(exp, level = 0): string {
     return wrap(
       `${exp[1]}(${exp
         .slice(2)
-        .map(e => stringify(e))
+        .map((e) => stringify(e))
         .join(", ")})`
     );
   } else if (op === "PARAM") {
@@ -354,7 +409,7 @@ export function stringify(exp, level = 0): string {
       return wrap(
         exp[1]
           .slice(1)
-          .map(p => {
+          .map((p) => {
             if (typeof p === "string") return p;
             else return `{${stringify(p)}}`;
           })
@@ -381,6 +436,17 @@ export function stringify(exp, level = 0): string {
         )}`
       );
     }
+  } else if (op === "CASE") {
+    const parts: string[] = ["CASE"];
+    for (let i = 1; i < exp.length - 1; i += 2) {
+      if (!Array.isArray(exp[i]) && exp[i]) {
+        if (exp[i + 1] != null) parts.push("ELSE", stringify(exp[i + 1]));
+        break;
+      }
+      parts.push("WHEN", stringify(exp[i]), "THEN", stringify(exp[i + 1]));
+    }
+    parts.push("END");
+    return parts.join(" ");
   } else if (op in opLevels) {
     const parts = exp.slice(1).map((e, i) => {
       return stringify(e, opLevels[exp[0]] + Math.min(i - 1, 0));
@@ -393,7 +459,7 @@ export function stringify(exp, level = 0): string {
   }
 }
 
-export function parseLikePattern(pat, esc): string[] {
+export function parseLikePattern(pat: string, esc: string): string[] {
   const chars = pat.split("");
 
   for (let i = 0; i < chars.length; ++i) {
@@ -408,5 +474,5 @@ export function parseLikePattern(pat, esc): string[] {
       while (chars[i + 1] === "%") chars[++i] = "";
     }
   }
-  return chars.filter(c => c);
+  return chars.filter((c) => c);
 }

lib/common/expression.ts

@@ -18,13 +18,11 @@
  */
 
 import { map, mapAsync, parseLikePattern } from "./expression-parser";
-import { booleanCnf } from "./expression-cnf";
-import { naiveDpll } from "./sat-solver";
 import { Expression } from "../types";
 
 const isArray = Array.isArray;
 
-const regExpCache = new WeakMap<object, RegExp>();
+const regExpCache = new WeakMap<any, RegExp>();
 
 const REDUCE_SKIP = {};
 function reduce(exp, callback): Expression {
@@ -44,7 +42,7 @@ function reduce(exp, callback): Expression {
   return exp;
 }
 
-export function likePatternToRegExp(pat, esc = "", flags = ""): RegExp {
+export function likePatternToRegExp(pat: string, esc = "", flags = ""): RegExp {
   const convChars = {
     "-": "\\-",
     "/": "\\/",
@@ -63,159 +61,231 @@ export function likePatternToRegExp(pat, esc = "", flags = ""): RegExp {
     "{": "\\{",
     "}": "\\}",
     "\\%": ".*",
-    "\\_": "."
+    "\\_": ".",
   };
   let chars = parseLikePattern(pat, esc);
   if (!chars.length) return new RegExp("^$", flags);
-  chars = chars.map(c => convChars[c] || c);
+  chars = chars.map((c) => convChars[c] || c);
   chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
   const l = chars.length - 1;
   chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
   return new RegExp(chars.join(""), flags);
 }
 
-function evalExp(e: Expression): Expression {
+function compare(
-  if (e[0] === "AND") {
+  a: boolean | number | string,
-    return reduce(e, (a, b) => {
+  b: boolean | number | string
-      if (!isArray(a)) return a ? b : a;
+): number {
-      if (!isArray(b)) return b ? a : b;
+  if (typeof a === "boolean") a = +a;
-      return REDUCE_SKIP;
+  if (typeof b === "boolean") b = +b;
-    });
+  if (typeof a !== typeof b) return typeof a === "string" ? 1 : -1;
-  } else if (e[0] === "OR") {
+  return a > b ? 1 : a < b ? -1 : 0;
-    return reduce(e, (a, b) => {
+}
-      if (!isArray(a)) return a ? a : b;
+
-      if (!isArray(b)) return b ? b : a;
+function toNumber(a: boolean | number | string): number {
-      return REDUCE_SKIP;
+  switch (typeof a) {
-    });
+    case "number":
-  } else if (e[0] === "NOT") {
+      return a;
-    if (!isArray(e[1])) return !e[1];
+    case "boolean":
-    else if (e[1][0] === "NOT") return e[1][1];
+      return +a;
-  } else if (e[0] === "IS NULL") {
+    case "string":
-    if (isArray(e[1])) return e;
+      return parseFloat(a) || 0;
-    else if (e[1] == null) return true;
+  }
+}
+
+function toString(a: boolean | number | string): string {
+  switch (typeof a) {
+    case "string":
+      return a;
+    case "number":
+      return a.toString();
+    case "boolean":
+      return (+a).toString();
+  }
+}
+
+export function evaluateCallback(exp: Expression): Expression {
+  if (!Array.isArray(exp)) return exp;
+  if (exp[0] === "CASE") {
+    for (let i = 1; i < exp.length; i += 2) {
+      if (Array.isArray(exp[i])) return exp;
+      if (exp[i]) return exp[i + 1];
+    }
+    return null;
+  } else if (exp[0] === "FUNC") {
+    if (exp[1] === "COALESCE") {
+      const args: Expression[] = [];
+      for (let i = 2; i < exp.length; ++i) {
+        const e = exp[i];
+        if (e == null) continue;
+        args.push(e);
+        if (!Array.isArray(e)) break;
+      }
+      if (!args.length) return null;
+      if (args.length === 1) return args[0];
+      return ["FUNC", "COALESCE", ...args];
+    } else if (exp[1] === "UPPER") {
+      if (exp[2] == null) return null;
+      if (!isArray(exp[2])) return toString(exp[2]).toUpperCase();
+    } else if (exp[1] === "LOWER") {
+      if (exp[2] == null) return null;
+      if (!isArray(exp[2])) return toString(exp[2]).toLowerCase();
+    }
+  } else if (exp[0] === "PARAM") {
+    if (exp[1] == null) return null;
+  } else if (exp[0] === "AND") {
+    for (let i = 1; i < exp.length; ++i)
+      if (!Array.isArray(exp[i]) && exp[i] != null && !exp[i]) return false;
+    const args: Expression[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const ee = exp[i];
+      if (ee == null) return null;
+      if (Array.isArray(ee)) {
+        if (ee[0] === "AND") args.push(...ee.slice(1));
+        else args.push(ee);
+      }
+    }
+    if (!args.length) return true;
+    if (args.length === 1) args.push(true);
+    return ["AND", ...args];
+  } else if (exp[0] === "OR") {
+    const args: Expression[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const ee = exp[i];
+      if (Array.isArray(ee)) {
+        if (ee[0] === "OR") args.push(...ee.slice(1));
+        else args.push(ee);
+      } else if (ee) {
+        return true;
+      }
+    }
+    if (!args.length) return exp.some((ee) => ee == null) ? null : false;
+    if (args.length === 1) args.push(false);
+    return ["OR", ...args];
+  } else if (exp[0] === "NOT") {
+    if (exp[1] == null) return null;
+    if (!isArray(exp[1])) return !exp[1];
+    else if (exp[1][0] === "NOT") return exp[1][1];
+  } else if (exp[0] === "IS NULL") {
+    if (isArray(exp[1])) return exp;
+    else if (exp[1] == null) return true;
     else return null;
-  } else if (e[0] === "IS NOT NULL") {
+  } else if (exp[0] === "IS NOT NULL") {
-    if (isArray(e[1])) return e;
+    if (isArray(exp[1])) return exp;
-    else if (e[1] != null) return true;
+    else if (exp[1] != null) return true;
     else return null;
-  } else if (e[0] === "LIKE") {
+  } else if (exp[0] === "LIKE") {
-    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
+    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
     else if (
-      e[1] == null ||
+      exp[1] == null ||
-      e[2] == null ||
+      exp[2] == null ||
-      ((e as any[]).length >= 4 && e[3] == null)
+      ((exp as any[]).length >= 4 && exp[3] == null)
     )
       return null;
-    let r = regExpCache.get(e as any[]);
+    let r = regExpCache.get(exp as any[]);
     if (!r) {
-      r = likePatternToRegExp(e[2], e[3]);
+      r = likePatternToRegExp(exp[2], exp[3]);
-      regExpCache.set(e as any[], r);
+      regExpCache.set(exp as any[], r);
     }
-    return r.test(e[1]);
+    return r.test(exp[1]);
-  } else if (e[0] === "NOT LIKE") {
+  } else if (exp[0] === "NOT LIKE") {
-    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
+    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
     else if (
-      e[1] == null ||
+      exp[1] == null ||
-      e[2] == null ||
+      exp[2] == null ||
-      ((e as any[]).length >= 4 && e[3] == null)
+      ((exp as any[]).length >= 4 && exp[3] == null)
     )
       return null;
-    let r = regExpCache.get(e as any[]);
+    let r = regExpCache.get(exp as any[]);
     if (!r) {
-      r = likePatternToRegExp(e[2], e[3]);
+      r = likePatternToRegExp(exp[2], exp[3]);
-      regExpCache.set(e as any[], r);
+      regExpCache.set(exp as any[], r);
     }
-    return !r.test(e[1]);
+    return !r.test(exp[1]);
-  } else if (e[0] === "=") {
+  } else if (exp[0] === "=") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (exp[1] == null || exp[2] == null) return null;
-    if (e[1] == null || e[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return e[1] === e[2];
+    return compare(exp[1], exp[2]) === 0;
-  } else if (e[0] === "<>") {
+  } else if (exp[0] === "<>") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (exp[1] == null || exp[2] == null) return null;
-    if (e[1] == null || e[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return e[1] !== e[2];
+    return compare(exp[1], exp[2]) !== 0;
-  } else if (e[0] === ">") {
+  } else if (exp[0] === ">") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (exp[1] == null || exp[2] == null) return null;
-    if (e[1] == null || e[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return e[1] > e[2];
+    return compare(exp[1], exp[2]) > 0;
-  } else if (e[0] === ">=") {
+  } else if (exp[0] === ">=") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (exp[1] == null || exp[2] == null) return null;
-    if (e[1] == null || e[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return e[1] >= e[2];
+    return compare(exp[1], exp[2]) >= 0;
-  } else if (e[0] === "<") {
+  } else if (exp[0] === "<") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (exp[1] == null || exp[2] == null) return null;
-    if (e[1] == null || e[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return e[1] < e[2];
+    return compare(exp[1], exp[2]) < 0;
-  } else if (e[0] === "<=") {
+  } else if (exp[0] === "<=") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (exp[1] == null || exp[2] == null) return null;
-    if (e[1] == null || e[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return e[1] <= e[2];
+    return compare(exp[1], exp[2]) <= 0;
-  } else if (e[0] === "*") {
+  } else if (exp[0] === "*") {
-    return reduce(e, (a, b) => {
+    return reduce(exp, (a, b) => {
-      if (!isArray(a) && !isArray(b)) {
       if (a == null || b == null) return null;
-        return a * b;
+      if (!isArray(a) && !isArray(b)) return toNumber(a) * toNumber(b);
-      }
       return REDUCE_SKIP;
     });
-  } else if (e[0] === "/") {
+  } else if (exp[0] === "/") {
-    return reduce(e, (a, b, i) => {
+    return reduce(exp, (a, b, i) => {
-      if (!isArray(a) && !isArray(b)) {
       if (a == null || b == null) return null;
-        return i === 0 ? a / b : a * b;
+      if (!isArray(a) && !isArray(b))
-      }
+        return i === 0 ? toNumber(a) / toNumber(b) : toNumber(a) * toNumber(b);
       return REDUCE_SKIP;
     });
-  } else if (e[0] === "+") {
+  } else if (exp[0] === "+") {
-    return reduce(e, (a, b) => {
+    return reduce(exp, (a, b) => {
-      if (!isArray(a) && !isArray(b)) {
       if (a == null || b == null) return null;
-        return a + b;
+      if (!isArray(a) && !isArray(b)) return toNumber(a) + toNumber(b);
-      }
       return REDUCE_SKIP;
     });
-  } else if (e[0] === "-") {
+  } else if (exp[0] === "-") {
-    return reduce(e, (a, b, i) => {
+    return reduce(exp, (a, b, i) => {
-      if (!isArray(a) && !isArray(b)) {
       if (a == null || b == null) return null;
-        return i === 0 ? a - b : a + b;
+      if (!isArray(a) && !isArray(b))
-      }
+        return i === 0 ? toNumber(a) - toNumber(b) : toNumber(a) + toNumber(b);
       return REDUCE_SKIP;
     });
-  } else if (e[0] === "||") {
+  } else if (exp[0] === "||") {
-    return reduce(e, (a, b) => {
+    return reduce(exp, (a, b) => {
-      if (!isArray(a) && !isArray(b)) {
       if (a == null || b == null) return null;
-        return `${a}${b}`;
+      if (!isArray(a) && !isArray(b)) return toString(a) + toString(b);
-      }
       return REDUCE_SKIP;
     });
   }
-  return e;
+  return exp;
 }
 
 export function evaluate(
-  exp,
+  exp: Expression,
-  obj,
+  obj: Record<string, unknown> | ((e: string) => Expression),
   now: number,
-  cb?: Function
+  cb?: (e: Expression) => Expression
 ): string | number | boolean | null;
-export function evaluate(exp, obj?, now?: number, cb?: Function): Expression;
+export function evaluate(
-export function evaluate(exp, obj?, now?: number, cb?: Function): Expression {
+  exp: Expression,
-  return map(exp, e => {
+  obj?: Record<string, unknown> | ((e: string) => Expression),
+  now?: number,
+  cb?: (e: Expression) => Expression
+): Expression;
+export function evaluate(
+  exp: Expression,
+  obj?: Record<string, unknown> | ((e: string) => Expression),
+  now?: number,
+  cb?: (e: Expression) => Expression
+): Expression {
+  return map(exp, (e) => {
     if (cb) e = cb(e);
     if (!isArray(e)) return e;
 
-    if (e[0] === "FUNC") {
+    if (e[0] === "FUNC" && e[1] === "NOW") {
-      if (e[1] === "NOW") {
       if (now) return now;
-      } else if (e[1] === "UPPER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
-      } else if (e[1] === "LOWER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
-      }
     } else if (e[0] === "PARAM") {
       if (e[1] == null) return null;
       if (obj && !isArray(e[1])) {
@@ -228,29 +298,29 @@ export function evaluate(exp, obj?, now?: number, cb?: Function): Expression {
         return v;
       }
     }
-    return evalExp(e);
+    return evaluateCallback(e);
   });
 }
 
 export async function evaluateAsync(
-  exp,
+  exp: Expression,
-  obj,
+  obj: Record<string, unknown>,
   now: number,
-  cb?: Function
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<string | number | boolean | null>;
 export async function evaluateAsync(
-  exp,
+  exp: Expression,
-  obj?,
+  obj?: Record<string, unknown>,
   now?: number,
-  cb?: Function
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<Expression>;
 export async function evaluateAsync(
-  exp,
+  exp: Expression,
-  obj?,
+  obj?: Record<string, unknown>,
   now?: number,
-  cb?: Function
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<Expression> {
-  return mapAsync(exp, async e => {
+  return mapAsync(exp, async (e) => {
     if (cb) e = await cb(e);
     if (!isArray(e)) return e;
 
@@ -269,59 +339,59 @@ export async function evaluateAsync(
       if (obj && !isArray(e[1])) {
         let v = obj[e[1]];
         if (v == null) return null;
-        if (typeof v === "object") v = v.value ? v.value[0] : null;
+        if (typeof v === "object") v = v["value"] ? v["value"][0] : null;
-        return v;
+        return v as Expression;
       }
     }
-    return evalExp(e);
+    return evaluateCallback(e);
   });
 }
 
-export function and(exp1, exp2): Expression {
+export function and(exp1: Expression, exp2: Expression): Expression {
-  if (!isArray(exp1)) return exp1 ? exp2 : exp1;
+  if (exp1 != null && !exp1) return false;
-  if (!isArray(exp2)) return exp2 ? exp1 : exp2;
+  if (exp2 != null && !exp2) return false;
+  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
+    if (exp1 == null || exp2 == null) return null;
+    return true;
+  }
+  if (!Array.isArray(exp2) && exp1[0] === "AND") return exp1;
+  if (!Array.isArray(exp1) && exp2[0] === "AND") return exp2;
 
-  let res: Expression = ["AND"];
+  const res: Expression = ["AND"];
 
-  if (exp1[0] === "AND") res = res.concat(exp1.slice(1));
+  if (Array.isArray(exp1) && exp1[0] === "AND") res.push(...exp1.slice(1));
   else res.push(exp1);
 
-  if (exp2[0] === "AND") res = res.concat(exp2.slice(1));
+  if (Array.isArray(exp2) && exp2[0] === "AND") res.push(...exp2.slice(1));
   else res.push(exp2);
 
   return res;
 }
 
-export function or(exp1, exp2): Expression {
+export function or(exp1: Expression, exp2: Expression): Expression {
-  if (!isArray(exp1)) return exp1 ? exp1 : exp2;
+  if (!Array.isArray(exp1) && exp1) return true;
-  if (!isArray(exp2)) return exp2 ? exp2 : exp1;
+  if (!Array.isArray(exp2) && exp2) return true;
+  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
+    if (exp1 == null || exp2 == null) return null;
+    return false;
+  }
+  if (!Array.isArray(exp2) && exp1[0] === "OR") return exp1;
+  if (!Array.isArray(exp1) && exp2[0] === "OR") return exp2;
 
-  let res: Expression = ["OR"];
+  const res: Expression = ["OR"];
 
-  if (exp1[0] === "OR") res = res.concat(exp1.slice(1));
+  if (Array.isArray(exp1) && exp1[0] === "OR") res.push(...exp1.slice(1));
   else res.push(exp1);
 
-  if (exp2[0] === "OR") res = res.concat(exp2.slice(1));
+  if (Array.isArray(exp2) && exp2[0] === "OR") res.push(...exp2.slice(1));
   else res.push(exp2);
 
   return res;
 }
 
-export function not(exp): Expression {
+export function extractParams(exp: Expression): Expression[] {
-  if (isArray(exp) && exp[0] === "NOT") return exp[1];
-  return ["NOT", exp];
-}
-
-export function subset(exp1, exp2): boolean {
-  const e = evaluate(["NOT", ["OR", ["NOT", exp1], exp2]]);
-  if (!isArray(e)) return !e;
-  const { vars, clauses } = booleanCnf(e);
-  return !naiveDpll(clauses, vars);
-}
-
-export function extractParams(exp): Expression[] {
   const params = new Set<Expression>();
-  map(exp, e => {
+  map(exp, (e) => {
     if (isArray(e) && e[0] === "PARAM") params.add(e[1]);
     return e;
   });

lib/common/memoize.ts

@@ -39,7 +39,7 @@ function getKey(obj): string {
   return k;
 }
 
-export default function memoize<T extends Function>(func: T): T {
+export default function memoize<T extends (...args: any[]) => any>(func: T): T {
   const funcKey = getKey(func);
   return ((...args) => {
     const key = JSON.stringify(args.map(getKey)) + funcKey;

lib/common/path-set.ts

@@ -73,8 +73,8 @@ export default class PathSet {
 
   public find(
     path: Path,
-    superset: boolean = false,
+    superset = false,
-    subset: boolean = false,
+    subset = false,
     depth: number = path.length
   ): Path[] {
     if (path.alias) throw new Error("PathSet does not support aliased paths");
@@ -106,24 +106,24 @@ export default class PathSet {
       if (!fragmentIndexSet1) {
         if (!fragmentIndexSet2) return [];
         if (!res) res = [...fragmentIndexSet2];
-        else res = res.filter(r => fragmentIndexSet2.has(r));
+        else res = res.filter((r) => fragmentIndexSet2.has(r));
       } else if (!fragmentIndexSet2) {
         if (!res) res = [...fragmentIndexSet1];
-        else res = res.filter(r => fragmentIndexSet1.has(r));
+        else res = res.filter((r) => fragmentIndexSet1.has(r));
       } else {
         if (!res) {
           res = [...fragmentIndexSet1, ...fragmentIndexSet2];
         } else {
           res = res.filter(
-            r => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
+            (r) => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
           );
         }
       }
       if (!res.length) return res;
     }
 
-    if (!res) res = [].concat(...lengthIndex.map(a => [...a]));
+    if (!res) res = [].concat(...lengthIndex.map((a) => [...a]));
-    else res = res.filter(r => lengthIndex.some(a => a.has(r)));
+    else res = res.filter((r) => lengthIndex.some((a) => a.has(r)));
 
     return res;
   }

lib/common/path.ts

@@ -97,6 +97,7 @@ export default class Path {
           )
             ++index;
           const s = pattern.slice(j, index).trim();
+          if (!s) throw new Error("Invalid parameter path");
           segments.push(s);
         }
 
@@ -118,7 +119,7 @@ export default class Path {
       if (Array.isArray(s)) {
         alias |= 1 << i;
         const parts = s.map(
-          al => `${al[0].toString()}:${JSON.stringify(al[1])}`
+          (al) => `${al[0].toString()}:${JSON.stringify(al[1])}`
         );
         return `[${parts.join(",")}]`;
       } else if (s === "*") {
@@ -159,7 +160,7 @@ export default class Path {
     return this._string;
   }
 
-  public slice(start: number = 0, end: number = this.segments.length): Path {
+  public slice(start = 0, end: number = this.segments.length): Path {
     if (start < 0) start = Math.max(0, this.segments.length + start);
     if (end < 0) end = Math.max(0, this.segments.length + end);
 
@@ -211,7 +212,7 @@ export default class Path {
 
   public stripAlias(): Path {
     if (!this.alias) return this;
-    const segments = this.segments.map(s => (Array.isArray(s) ? "*" : s));
+    const segments = this.segments.map((s) => (Array.isArray(s) ? "*" : s));
     const str = segments.join(".");
 
     let path = cache1.get(str);

lib/common/sat-solver.ts

@@ -1,175 +0,0 @@
-/**
- * Naive DPLL solver for propositional logic adapted from
- * https://github.com/tammet/logictools
- *
- * Origianl code by Tanel Tammet released under the MIT license.
- *
- * Check whether a clause set is satisfiable or contradictory, using a naive
- * implementation of the dpll (Davis-Putnam-Logemann-Loveland) procedure
- * combining truth table search with the resolution-like unit propagation.
- *
- * The standard pure-literal rule is not implemented in this naive version,
- * since it is not really crucial for the algorithm.
- *
- * The code is very similar to the pure table solver, except we use a
- * unit-propagating, derived-varlist-returing satisfiableAt.
- */
-
-/* eslint-disable @typescript-eslint/no-use-before-define */
-
-function satisfiableAt(clauses, varVals, varnr, val): boolean {
-  if (varnr !== 0) varVals[varnr] = val;
-
-  const propRes = unitPropagate(clauses, varVals);
-  if (propRes === 1) {
-    varVals[varnr] = 0;
-    return true;
-  }
-
-  // if result false (-1) unitPropagate restores the varVals state itself
-  if (propRes === -1) {
-    varVals[varnr] = 0;
-    return false;
-  }
-
-  // find next unassigned var to split
-  let nextVar = 0;
-  for (let i = 1; i < varVals.length; i++) {
-    if (varVals[i] === 0) {
-      nextVar = i;
-      break;
-    }
-  }
-
-  if (nextVar === 0) throw new Error("Error in satisfiableAt");
-
-  if (
-    satisfiableAt(clauses, varVals, nextVar, 1) ||
-    satisfiableAt(clauses, varVals, nextVar, -1)
-  )
-    // not necessary to restore split var: search is halted anyway
-    return true;
-
-  // restore split var
-  varVals[nextVar] = 0;
-
-  // restore derived vars
-  for (let i = 0; i < propRes.length; i++) varVals[propRes[i]] = 0;
-
-  return false;
-}
-
-/**
- * Check whether clauses are satisfiable under the varVals assignment, without
- * search.
- *
- * Iterate through all clauses and check whether
- *
- *  - all literals in the clause are assigned false: undo derived units (set
- *    values in varVals to 0) and return false immediately
- *
- *  - a clause contains a literal assigned true: skip further actions on this
- *    clause
- *
- *  - a single literal in the clause is unassigned, all the rest are assigned
- *    false: derive this literal as a unit clause.
- *
- * In case at least one literal is derived during the process above, iterate
- * the whole process until no more new literals are derived.
- *
- * Returns:
- *
- *   1: all clauses were assigned true, hence the clause set is satisfiable.
- *   The whole search will terminate after that.
- *
- *   -1: clause set is contradictory. Derived units have been restored.
- *
- *   a list of derived variables:  the status of the clause set is
- *   undetermined. The values of variables on the list will be restored (set to
- *   0) during backtracking.
- */
-function unitPropagate(clauses, varVals): number[] | 1 | -1 {
-  // all vars with a value derived in the forthcoming main loop
-  const derivedVars = [];
-
-  // the main loop is run, deriving new unit clauses, until either:
-  // - the clause set is detected to be unsatisfiable
-  // - the clause set is detected to be satisfiable
-  // - there are no more derived unit clauses
-
-  for (;;) {
-    let allClausesFound = true; // set to false if we find an undetermined clause
-    const derivedQueue = []; // derived units (literals) during one iteration
-    for (let i = 0; i < clauses.length; i++) {
-      const clause = clauses[i];
-      let clauseVal = 0;
-      let unassignedCount = 0; // count unassigned vars in clause
-      let unassignedLit = 0; // 0 means none found for this clause
-
-      for (let j = 0; j < clause.length; j++) {
-        const lit = clause[j];
-        let nr, polarity;
-        if (lit < 0) {
-          nr = 0 - lit;
-          polarity = -1;
-        } else {
-          nr = lit;
-          polarity = 1;
-        }
-
-        if (varVals[nr] === polarity) {
-          clauseVal = 1;
-          break;
-        } else if (varVals[nr] === 0) {
-          unassignedCount++;
-          unassignedLit = lit;
-        }
-      }
-
-      if (clauseVal === 1) continue; // clause is subsumed
-
-      // clause is not subsumed by varVals
-      if (unassignedCount === 0) {
-        for (let j = 0; j < derivedVars.length; j++)
-          varVals[derivedVars[j]] = 0; // restore derived
-        return -1;
-      } else if (unassignedCount === 1) {
-        // unassignedLit is a derived literal
-        derivedQueue.push(unassignedLit);
-      }
-
-      if (unassignedCount !== 0) allClausesFound = false;
-    }
-
-    // if all clauses were subsumed or all the variables in a clause had a value
-    // no need to restore varVals: whole search finished
-    if (allClausesFound) return 1;
-
-    // in case there are any derived units present, store to varVals and
-    // iterate the main loop
-    if (derivedQueue.length === 0) break;
-
-    // some units were derived: iterate the main loop
-    for (let j = 0; j < derivedQueue.length; j++) {
-      const lit = derivedQueue[j];
-      let nr, polarity;
-      if (lit < 0) {
-        nr = 0 - lit;
-        polarity = -1;
-      } else {
-        nr = lit;
-        polarity = 1;
-      }
-      varVals[nr] = polarity;
-      derivedVars.push(nr);
-    }
-  }
-
-  return derivedVars;
-}
-
-export function naiveDpll(clauses, maxvarnr): boolean {
-  // variable values are 0 if not set, 1 if positive, -1 if negative
-  const varVals = new Int32Array(maxvarnr + 1);
-  return satisfiableAt(clauses, varVals, 0, 0);
-}

lib/config.ts

@@ -32,13 +32,13 @@ while (!existsSync(`${ROOT_DIR}/package.json`)) {
 }
 
 // For compatibility with v1.1
-let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl;
+let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl, fsHostname;
 
 const options = {
   EXT_DIR: { type: "path", default: resolve(ROOT_DIR, "config/ext") },
   MONGODB_CONNECTION_URL: {
     type: "string",
-    default: "mongodb://127.0.0.1/genieacs"
+    default: "mongodb://127.0.0.1/genieacs",
   },
 
   CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
@@ -48,7 +48,6 @@ const options = {
   CWMP_SSL_KEY: { type: "string", default: "" },
   CWMP_LOG_FILE: { type: "path", default: "" },
   CWMP_ACCESS_LOG_FILE: { type: "path", default: "" },
-  CWMP_KEEP_ALIVE_TIMEOUT: { type: "int", default: 0 },
 
   NBI_WORKER_PROCESSES: { type: "int", default: 0 },
   NBI_PORT: { type: "int", default: 7557 },
@@ -63,7 +62,7 @@ const options = {
   FS_INTERFACE: { type: "string", default: "0.0.0.0" },
   FS_SSL_CERT: { type: "string", default: "" },
   FS_SSL_KEY: { type: "string", default: "" },
-  FS_HOSTNAME: { type: "string", default: "" },
+  FS_URL_PREFIX: { type: "string", default: "" },
   FS_LOG_FILE: { type: "path", default: "" },
   FS_ACCESS_LOG_FILE: { type: "path", default: "" },
 
@@ -77,6 +76,7 @@ const options = {
   UI_JWT_SECRET: { type: "string", default: "" },
 
   UDP_CONNECTION_REQUEST_PORT: { type: "int", default: 0 },
+  FORWARDED_HEADER: { type: "string", default: "" },
 
   DOWNLOAD_TIMEOUT: { type: "int", default: 3600 },
   EXT_TIMEOUT: { type: "int", default: 3000 },
@@ -99,16 +99,8 @@ const options = {
   CONNECTION_REQUEST_ALLOW_BASIC_AUTH: { type: "bool", default: false },
   MAX_COMMIT_ITERATIONS: { type: "int", default: 32 },
 
-  // XML configuration
-  XML_RECOVER: { type: "bool", default: false },
-  XML_IGNORE_ENC: { type: "bool", default: false },
-  XML_FORMAT: { type: "bool", default: false },
-  XML_NO_DECL: { type: "bool", default: false },
-  XML_NO_EMPTY: { type: "bool", default: false },
-  XML_LIBXMLJS: { type: "bool", default: false },
-
   // Should probably never be changed
-  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 }
+  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
 };
 
 const allConfig: { [name: string]: string | number } = {};
@@ -120,37 +112,20 @@ function setConfig(name, value, commandLineArgument = false): boolean {
   if (name === "CONFIG_DIR" || name === "config-dir")
     configDir = configDir || resolve(ROOT_DIR, value);
 
-  if (name === "CWMP_SSL" || name === "cwmp-ssl") {
+  if (name === "CWMP_SSL" || name === "cwmp-ssl")
-    cwmpSsl =
+    cwmpSsl = cwmpSsl || String(value).toLowerCase().trim();
-      cwmpSsl ||
-      String(value)
-        .toLowerCase()
-        .trim();
-  }
 
-  if (name === "NBI_SSL" || name === "nbi-ssl") {
+  if (name === "NBI_SSL" || name === "nbi-ssl")
-    nbiSsl =
+    nbiSsl = nbiSsl || String(value).toLowerCase().trim();
-      nbiSsl ||
-      String(value)
-        .toLowerCase()
-        .trim();
-  }
 
-  if (name === "FS_SSL" || name === "fs-ssl") {
+  if (name === "FS_SSL" || name === "fs-ssl")
-    fsSsl =
+    fsSsl = fsSsl || String(value).toLowerCase().trim();
-      fsSsl ||
-      String(value)
-        .toLowerCase()
-        .trim();
-  }
 
-  if (name === "UI_SSL" || name === "ui-ssl") {
+  if (name === "UI_SSL" || name === "ui-ssl")
-    uiSsl =
+    uiSsl = uiSsl || String(value).toLowerCase().trim();
-      uiSsl ||
+
-      String(value)
+  if (name === "FS_HOSTNAME" || name === "fs-hostname")
-        .toLowerCase()
+    fsHostname = fsHostname || String(value).trim();
-        .trim();
-  }
 
   // For compatibility with v1.0
   if (name === "PRESETS_CACHE_DURATION" || name === "presets-cache-duration")
@@ -168,12 +143,6 @@ function setConfig(name, value, commandLineArgument = false): boolean {
   )
     setConfig("GPV_BATCH_SIZE", value);
 
-  if (name === "XML_PARSE_IGNORE_ENC" || name === "xml-parse-ignore-enc")
-    setConfig("XML_IGNORE_ENC", value);
-
-  if (name === "XML_PARSE_RECOVER" || name === "xml-parse-recover")
-    setConfig("XML_RECOVER", value);
-
   if (name === "FS_IP" || name === "fs-ip") setConfig("FS_HOSTNAME", value);
 
   function cast(val, type): string | number | boolean {
@@ -181,11 +150,7 @@ function setConfig(name, value, commandLineArgument = false): boolean {
       case "int":
         return Number(val);
       case "bool":
-        return ["true", "1"].includes(
+        return ["true", "1"].includes(String(val).trim().toLowerCase());
-          String(val)
-            .trim()
-            .toLowerCase()
-        );
       case "string":
         return String(val);
       case "path":
@@ -275,6 +240,15 @@ if (["true", "1"].includes(uiSsl)) {
   setConfig("UI_SSL_KEY", `${d}/cwmp.key`);
 }
 
+if (fsHostname) {
+  const FS_PORT = allConfig["FS_PORT"] || 7567;
+  const FS_SSL = !!allConfig["FS_SSL_CERT"];
+  setConfig(
+    "FS_URL_PREFIX",
+    (FS_SSL ? "https" : "http") + `://${fsHostname}:${FS_PORT}/`
+  );
+}
+
 // Defaults
 for (const [k, v] of Object.entries(options))
   if (v["default"] != null) setConfig(k, v["default"]);
@@ -310,7 +284,7 @@ export function get(
   return null;
 }
 
-export function getDefault(optionName): string | number | boolean {
+export function getDefault(optionName: string): string | number | boolean {
   const option = options[optionName];
   if (!option) return null;
 

lib/connection-request.ts

@@ -29,15 +29,13 @@ import * as debug from "./debug";
 
 async function extractAuth(
   exp: Expression,
-  context: {},
-  now: number,
   dflt: any
 ): Promise<[string, string, Expression]> {
   let username, password;
   const _exp = await evaluateAsync(
     exp,
-    context,
+    {},
-    now,
+    0,
     async (e: Expression): Promise<Expression> => {
       if (!username && Array.isArray(e) && e[0] === "FUNC") {
         if (e[1] === "EXT") {
@@ -50,8 +48,8 @@ async function extractAuth(
           return fault ? null : value;
         } else if (e[1] === "AUTH") {
           if (!Array.isArray(e[2]) && !Array.isArray(e[3])) {
-            username = e[2];
+            username = e[2] || "";
-            password = e[3];
+            password = e[3] || "";
           }
           return dflt;
         }
@@ -67,10 +65,10 @@ function httpGet(
   timeout,
   _debug: boolean,
   deviceId: string
-): Promise<{ statusCode: number; headers: {} }> {
+): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
   return new Promise((resolve, reject) => {
     const req = http
-      .get(options, res => {
+      .get(options, (res) => {
         res.resume();
         resolve({ statusCode: res.statusCode, headers: res.headers });
         if (_debug) {
@@ -78,12 +76,12 @@ function httpGet(
           debug.incomingHttpResponse(res, deviceId, null);
         }
       })
-      .on("error", err => {
+      .on("error", (err) => {
         req.abort();
         reject(new Error("Device is offline"));
         if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
       })
-      .on("socket", socket => {
+      .on("socket", (socket) => {
         socket.setTimeout(timeout);
         socket.on("timeout", () => {
           req.abort();
@@ -95,23 +93,21 @@ function httpGet(
 export async function httpConnectionRequest(
   address: string,
   authExp: Expression,
-  context: {},
   allowBasicAuth: boolean,
   timeout: number,
   _debug: boolean,
   deviceId: string
 ): Promise<void> {
-  const now = Date.now();
   const options: http.RequestOptions = parse(address);
   if (options.protocol !== "http:")
     throw new Error("Invalid connection request URL or protocol");
 
   options.agent = new http.Agent({
     maxSockets: 1,
-    keepAlive: true
+    keepAlive: true,
   });
 
-  let authHeader: {};
+  let authHeader: Record<string, string>;
   let username: string;
   let password: string;
 
@@ -125,8 +121,8 @@ export async function httpConnectionRequest(
         opts = Object.assign(
           {
             headers: {
-              Authorization: auth.basic(username || "", password || "")
+              Authorization: auth.basic(username || "", password || ""),
-            }
+            },
           },
           options
         );
@@ -141,8 +137,8 @@ export async function httpConnectionRequest(
                 "GET",
                 null,
                 authHeader
-              )
+              ),
-            }
+            },
           },
           options
         );
@@ -163,12 +159,7 @@ export async function httpConnectionRequest(
       authHeader = auth.parseWwwAuthenticateHeader(
         res.headers["www-authenticate"]
       );
-      [username, password, authExp] = await extractAuth(
+      [username, password, authExp] = await extractAuth(authExp, false);
-        authExp,
-        context,
-        now,
-        false
-      );
     } else {
       throw new Error(
         `Unexpected response code from device: ${res.statusCode}`
@@ -181,8 +172,7 @@ export async function httpConnectionRequest(
 export async function udpConnectionRequest(
   address: string,
   authExp: Expression,
-  context: {},
+  sourcePort = 0,
-  sourcePort: number = 0,
   _debug: boolean,
   deviceId: string
 ): Promise<void> {
@@ -202,12 +192,7 @@ export async function udpConnectionRequest(
   let username: string;
   let password: string;
 
-  [username, password, authExp] = await extractAuth(
+  [username, password, authExp] = await extractAuth(authExp, null);
-    authExp,
-    context,
-    now,
-    null
-  );
 
   if (username == null) username = "";
   if (password == null) password = "";
@@ -233,12 +218,7 @@ export async function udpConnectionRequest(
       });
     }
 
-    [username, password, authExp] = await extractAuth(
+    [username, password, authExp] = await extractAuth(authExp, null);
-      authExp,
-      context,
-      now,
-      null
-    );
   }
   client.close();
 }

lib/db.ts

@@ -19,7 +19,7 @@
 
 import { MongoClient, ObjectID, Collection } from "mongodb";
 import { get } from "./config";
-import { escapeRegExp } from "./common";
+import { encodeTag, escapeRegExp } from "./common";
 import { parse } from "./common/expression-parser";
 import {
   DeviceData,
@@ -27,7 +27,7 @@ import {
   SessionFault,
   Task,
   Operation,
-  Expression
+  Expression,
 } from "./types";
 import Path from "./common/path";
 
@@ -44,13 +44,22 @@ export let tasksCollection: Collection,
   usersCollection: Collection,
   configCollection: Collection;
 
+let clientPromise: Promise<MongoClient>;
 export let client: MongoClient;
 
+function compareAccessLists(list1: string[], list2: string[]): boolean {
+  if (list1.length !== list2.length) return false;
+  for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
+  return true;
+}
+
 export async function connect(): Promise<void> {
-  client = await MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
-    useNewUrlParser: true
+    useNewUrlParser: true,
+    useUnifiedTopology: true,
   });
 
+  client = await clientPromise;
   const db = client.db();
 
   tasksCollection = db.collection("tasks");
@@ -70,12 +79,12 @@ export async function connect(): Promise<void> {
 }
 
 export async function disconnect(): Promise<void> {
-  if (client) await client.close();
+  if (clientPromise) await (await clientPromise).close();
 }
 
 // Optimize projection by removing overlaps
 // This can modify the object
-function optimizeProjection(obj: {}): {} {
+function optimizeProjection(obj: { [path: string]: 1 }): { [path: string]: 1 } {
   if (obj[""]) return { "": obj[""] };
 
   const keys = Object.keys(obj).sort();
@@ -102,13 +111,13 @@ export async function fetchDevice(
     [
       Path.parse("Events"),
       timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 0] }
+      { object: [timestamp, 1], writable: [timestamp, 0] },
     ],
     [
       Path.parse("DeviceID"),
       timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 0] }
+      { object: [timestamp, 1], writable: [timestamp, 0] },
-    ]
+    ],
   ];
 
   const device = await devicesCollection.findOne({ _id: id });
@@ -116,6 +125,8 @@ export async function fetchDevice(
 
   function storeParams(obj, path: string, pathLength: number, ts): void {
     if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
+    if (obj["_attributesTimestamp"])
+      obj["_attributesTimestamp"] = +obj["_attributesTimestamp"];
 
     const attrs: Attributes = {};
     let t = obj["_timestamp"] || 1;
@@ -133,6 +144,16 @@ export async function fetchDevice(
 
     if (obj["_object"] != null) attrs.object = [t, obj["_object"] ? 1 : 0];
 
+    if (obj["_notification"] != null) {
+      attrs.notification = [
+        obj["_attributesTimestamp"] || 1,
+        obj["_notification"],
+      ];
+    }
+
+    if (obj["_accessList"] != null)
+      attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
+
     res.push([Path.parse(path.slice(0, -1)), t, attrs]);
 
     for (const [k, v] of Object.entries(obj)) {
@@ -155,8 +176,8 @@ export async function fetchDevice(
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]]
+            value: [+v, [+v, "xsd:dateTime"]],
-          }
+          },
         ]);
         break;
       case "_lastBoot":
@@ -166,8 +187,8 @@ export async function fetchDevice(
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]]
+            value: [+v, [+v, "xsd:dateTime"]],
-          }
+          },
         ]);
         break;
       case "_lastBootstrap":
@@ -177,8 +198,8 @@ export async function fetchDevice(
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]]
+            value: [+v, [+v, "xsd:dateTime"]],
-          }
+          },
         ]);
         break;
       case "_registered":
@@ -189,8 +210,8 @@ export async function fetchDevice(
           {
             object: [timestamp, 0],
             writable: [timestamp, 0],
-            value: [timestamp, [+v, "xsd:dateTime"]]
+            value: [timestamp, [+v, "xsd:dateTime"]],
-          }
+          },
         ]);
         break;
       case "_id":
@@ -200,8 +221,8 @@ export async function fetchDevice(
           {
             object: [timestamp, 0],
             writable: [timestamp, 0],
-            value: [timestamp, [v as string, "xsd:string"]]
+            value: [timestamp, [v as string, "xsd:string"]],
-          }
+          },
         ]);
         break;
       case "_tags":
@@ -209,20 +230,19 @@ export async function fetchDevice(
           res.push([
             Path.parse("Tags"),
             timestamp,
-            { object: [timestamp, 1], writable: [timestamp, 0] }
+            { object: [timestamp, 1], writable: [timestamp, 0] },
           ]);
         }
 
-        for (let t of v as string[]) {
+        for (const t of v as string[]) {
-          t = t.replace(/[^a-zA-Z0-9-]+/g, "_");
           res.push([
-            Path.parse("Tags." + t),
+            Path.parse("Tags." + encodeTag(t)),
             timestamp,
             {
               object: [timestamp, 0],
               writable: [timestamp, 1],
-              value: [timestamp, [true, "xsd:boolean"]]
+              value: [timestamp, [true, "xsd:boolean"]],
-            }
+            },
           ]);
         }
         break;
@@ -234,8 +254,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_Manufacturer"], "xsd:string"]]
+              value: [timestamp, [v["_Manufacturer"], "xsd:string"]],
-            }
+            },
           ]);
         }
 
@@ -246,8 +266,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_OUI"], "xsd:string"]]
+              value: [timestamp, [v["_OUI"], "xsd:string"]],
-            }
+            },
           ]);
         }
 
@@ -258,8 +278,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_ProductClass"], "xsd:string"]]
+              value: [timestamp, [v["_ProductClass"], "xsd:string"]],
-            }
+            },
           ]);
         }
 
@@ -270,8 +290,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_SerialNumber"], "xsd:string"]]
+              value: [timestamp, [v["_SerialNumber"], "xsd:string"]],
-            }
+            },
           ]);
         }
     }
@@ -329,6 +349,12 @@ export async function saveDevice(
     const object2 = ((diff[2] || {}).object || [])[1];
     const writable2 = ((diff[2] || {}).writable || [])[1];
     const writable1 = ((diff[1] || {}).writable || [])[1];
+    const attributesTimestamp1 = ((diff[1] || {}).notification || [])[0];
+    const attributesTimestamp2 = ((diff[2] || {}).notification || [])[0];
+    const notification1 = ((diff[1] || {}).notification || [])[1];
+    const notification2 = ((diff[2] || {}).notification || [])[1];
+    const accessList1 = ((diff[1] || {}).accessList || [])[1];
+    const accessList2 = ((diff[2] || {}).accessList || [])[1];
 
     switch (path.segments[0]) {
       case "Events":
@@ -396,7 +422,7 @@ export async function saveDevice(
           } else {
             if (!update["$pull"]["_tags"]) {
               update["$pull"]["_tags"] = {
-                $in: []
+                $in: [],
               };
             }
             update["$pull"]["_tags"]["$in"].push(path.segments[1]);
@@ -455,6 +481,46 @@ export async function saveDevice(
                     path.length ? path.toString() + "._writable" : "_writable"
                   ] = !!writable2;
                 }
+
+                break;
+              case "notification":
+                if (
+                  !diff[1] ||
+                  !diff[1].notification ||
+                  notification2 !== notification1
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._notification"
+                      : "_notification"
+                  ] = notification2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][
+                    path.toString() + "._attributesTimestamp"
+                  ] = new Date(attributesTimestamp2);
+                }
+
+                break;
+              case "accessList":
+                if (
+                  !diff[1] ||
+                  !diff[1].accessList ||
+                  !compareAccessLists(accessList2, accessList1)
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._accessList"
+                      : "_accessList"
+                  ] = accessList2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][
+                    path.toString() + "._attributesTimestamp"
+                  ] = new Date(attributesTimestamp2);
+                }
             }
           }
         }
@@ -470,6 +536,12 @@ export async function saveDevice(
               if (attrName === "value") {
                 update["$unset"][p + "_type"] = 1;
                 update["$unset"][p + "_timestamp"] = 1;
+              } else if (attrName === "notification") {
+                if (accessList2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
+              } else if (attrName === "accessList") {
+                if (notification2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
               }
             }
           }
@@ -508,7 +580,7 @@ export async function saveDevice(
   }
 
   const result = await devicesCollection.updateOne({ _id: deviceId }, update, {
-    upsert: isNew
+    upsert: isNew,
   });
 
   if (result.result.n !== 1)
@@ -521,7 +593,7 @@ export async function saveDevice(
 }
 
 export async function getFaults(
-  deviceId
+  deviceId: string
 ): Promise<{ [channel: string]: SessionFault }> {
   const res = await faultsCollection
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
@@ -541,26 +613,33 @@ export async function getFaults(
   return faults;
 }
 
-export async function saveFault(deviceId, channel, fault): Promise<void> {
+export async function saveFault(
+  deviceId: string,
+  channel: string,
+  fault: SessionFault
+): Promise<void> {
   const id = `${deviceId}:${channel}`;
-  fault = Object.assign({}, fault);
+  const f = Object.assign({}, fault) as Record<string, any>;
-  fault._id = id;
+  f["_id"] = id;
-  fault.device = deviceId;
+  f["device"] = deviceId;
-  fault.channel = channel;
+  f["channel"] = channel;
-  fault.timestamp = new Date(fault.timestamp);
+  f["timestamp"] = new Date(fault.timestamp);
-  fault.provisions = JSON.stringify(fault.provisions);
+  f["provisions"] = JSON.stringify(fault.provisions);
-  await faultsCollection.replaceOne({ _id: id }, fault, { upsert: true });
+  await faultsCollection.replaceOne({ _id: id }, f, { upsert: true });
 }
 
-export async function deleteFault(deviceId, channel): Promise<void> {
+export async function deleteFault(
+  deviceId: string,
+  channel: string
+): Promise<void> {
   await faultsCollection.deleteOne({ _id: `${deviceId}:${channel}` });
 }
 
 export async function getDueTasks(
-  deviceId,
+  deviceId: string,
-  timestamp
+  timestamp: number
 ): Promise<[Task[], number]> {
-  const cur = tasksCollection.find({ device: deviceId }).sort(["timestamp"]);
+  const cur = tasksCollection.find({ device: deviceId }).sort({ timestamp: 1 });
   const tasks = [] as Task[];
 
   for await (const task of cur) {
@@ -591,14 +670,17 @@ export async function getDueTasks(
   return [tasks, null];
 }
 
-export async function clearTasks(deviceId, taskIds): Promise<void> {
+export async function clearTasks(
+  deviceId: string,
+  taskIds: string[]
+): Promise<void> {
   await tasksCollection.deleteMany({
-    _id: { $in: taskIds.map(id => new ObjectID(id)) }
+    _id: { $in: taskIds.map((id) => new ObjectID(id)) },
   });
 }
 
 export async function getOperations(
-  deviceId
+  deviceId: string
 ): Promise<{ [commandKey: string]: Operation }> {
   const res = await operationsCollection
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
@@ -608,6 +690,11 @@ export async function getOperations(
   for (const r of res) {
     const commandKey = r._id.slice(deviceId.length + 1);
     delete r._id;
+    // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
+    if (typeof r.provisions !== "string") {
+      operations[commandKey] = r;
+      continue;
+    }
     r.timestamp = +r.timestamp;
     if (r.args) r.args = JSON.parse(r.args);
     r.provisions = JSON.parse(r.provisions);
@@ -618,43 +705,46 @@ export async function getOperations(
 }
 
 export async function saveOperation(
-  deviceId,
+  deviceId: string,
-  commandKey,
+  commandKey: string,
-  operation
+  operation: Operation
 ): Promise<void> {
   const id = `${deviceId}:${commandKey}`;
-  operation = Object.assign({}, operation);
+  const o = Object.assign({}, operation) as Record<string, any>;
-  operation._id = id;
+  o["_id"] = id;
-  operation.timestamp = new Date(operation.timestamp);
+  o["timestamp"] = new Date(operation.timestamp);
-  operation.provisions = JSON.stringify(operation.provisions);
+  o["provisions"] = JSON.stringify(operation.provisions);
-  operation.retries = JSON.stringify(operation.retries);
+  o["retries"] = JSON.stringify(operation.retries);
-  operation.args = JSON.stringify(operation.args);
+  o["args"] = JSON.stringify(operation.args);
-  await operationsCollection.replaceOne({ _id: id }, operation, {
+  await operationsCollection.replaceOne({ _id: id }, o, {
-    upsert: true
+    upsert: true,
   });
 }
 
-export async function deleteOperation(deviceId, commandKey): Promise<void> {
+export async function deleteOperation(
+  deviceId: string,
+  commandKey: string
+): Promise<void> {
   await operationsCollection.deleteOne({ _id: `${deviceId}:${commandKey}` });
 }
 
-export async function getPresets(): Promise<{}[]> {
+export async function getPresets(): Promise<Record<string, any>[]> {
   return presetsCollection.find().toArray();
 }
 
-export async function getObjects(): Promise<{}[]> {
+export async function getObjects(): Promise<Record<string, any>[]> {
   return objectsCollection.find().toArray();
 }
 
-export async function getProvisions(): Promise<{}[]> {
+export async function getProvisions(): Promise<Record<string, any>[]> {
   return provisionsCollection.find().toArray();
 }
 
-export async function getVirtualParameters(): Promise<{}[]> {
+export async function getVirtualParameters(): Promise<Record<string, any>[]> {
   return virtualParametersCollection.find().toArray();
 }
 
-export function getFiles(): Promise<{}[]> {
+export function getFiles(): Promise<Record<string, any>[]> {
   return filesCollection.find().toArray();
 }
 
@@ -662,9 +752,9 @@ export async function getConfig(): Promise<
   { id: string; value: Expression }[]
 > {
   const res = await configCollection.find().toArray();
-  return res.map(c => ({
+  return res.map((c) => ({
     id: c["_id"],
-    value: parse(c["value"])
+    value: parse(c["value"]),
   }));
 }
 

lib/debug.ts

@@ -21,7 +21,7 @@ import {
   IncomingMessage,
   ServerResponse,
   ClientRequest,
-  RequestOptions
+  RequestOptions,
 } from "http";
 import { Socket } from "net";
 import { appendFileSync } from "fs";
@@ -49,7 +49,7 @@ export function incomingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
   const msg = {
     event: "incoming HTTP request",
     timestamp: now,
@@ -60,7 +60,7 @@ export function incomingHttpRequest(
     method: httpRequest.method,
     url: httpRequest.url,
     headers: httpRequest.headers,
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -77,7 +77,7 @@ export function outgoingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
   const msg = {
     event: "outgoing HTTP response",
     timestamp: now,
@@ -86,7 +86,7 @@ export function outgoingHttpResponse(
     connection: getConnectionTimestamp(con),
     statusCode: httpResponse.statusCode,
     headers: httpResponse.getHeaders(),
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -104,7 +104,7 @@ export function outgoingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
@@ -115,7 +115,7 @@ export function outgoingHttpRequest(
     method: options.method || "GET",
     url: options.path,
     headers: httpRequest.getHeaders(),
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -143,7 +143,7 @@ export function outgoingHttpRequestError(
     method: options.method,
     url: options.path,
     headers: httpRequest.getHeaders(),
-    error: err.message
+    error: err.message,
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -160,16 +160,16 @@ export function incomingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
   const msg = {
     event: "incoming HTTP response",
     timestamp: now,
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
-    connection: getConnectionTimestamp(httpResponse.connection),
+    connection: getConnectionTimestamp(httpResponse.socket),
     statusCode: httpResponse.statusCode,
     headers: httpResponse.headers,
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -193,7 +193,7 @@ export function outgoingUdpMessage(
     remoteAddress: remoteAddress,
     deviceId: deviceId,
     remotePort: remotePort,
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")

lib/default-provisions.ts

@@ -21,90 +21,191 @@ import Path from "./common/path";
 import * as config from "./config";
 import * as device from "./device";
 import * as scheduling from "./scheduling";
+import { SessionContext, Declaration } from "./types";
 
 const MAX_DEPTH = +config.get("MAX_DEPTH");
 
-export function refresh(sessionContext, provision, declarations): boolean {
+export function refresh(
-  const segments = Path.parse(provision[1]).segments.slice();
+  sessionContext: SessionContext,
-  const l = segments.length;
+  provision: (string | number | boolean)[],
-  segments.length = MAX_DEPTH;
+  declarations: Declaration[]
-  segments.fill("*", l);
+): boolean {
-  const path = Path.parse(segments.join("."));
+  if (
-  const every = 1000 * (provision[2] || 1);
+    (provision.length !== 2 || typeof provision[1] !== "string") &&
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number") &&
+    (provision.length < 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number" ||
+      typeof provision[3] !== "boolean")
+  )
+    throw new Error("Invalid arguments");
+
+  const every = 1000 * ((provision[2] as number) || 1);
   const offset = scheduling.variance(sessionContext.deviceId, every);
   const t = scheduling.interval(sessionContext.timestamp, every, offset);
 
-  for (let i = l; i < path.length; ++i) {
+  let attrGet;
+  let refreshChildren;
+  if (provision[3] == null) {
+    refreshChildren = true;
+    attrGet = { object: 1, writable: 1, value: t };
+  } else {
+    attrGet = {};
+    refreshChildren = !!provision[3];
+    for (const a of provision.slice(4)) attrGet[a as string] = t;
+  }
+
+  let path = Path.parse(provision[1]);
+  let l = path.length;
+  if (refreshChildren) {
+    const segments = path.segments.slice();
+    l = segments.length;
+    segments.length = MAX_DEPTH;
+    segments.fill("*", l);
+    path = Path.parse(segments.join("."));
+  }
+
+  for (let i = l; i <= path.length; ++i) {
     declarations.push({
       path: path.slice(0, i),
       pathGet: t,
       pathSet: null,
-      attrGet: { object: 1, writable: 1, value: t },
+      attrGet: attrGet,
       attrSet: null,
-      defer: true
+      defer: true,
     });
   }
 
   return true;
 }
 
-export function value(sessionContext, provision, declarations): boolean {
+export function value(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (
+    provision.length < 3 ||
+    provision.length > 4 ||
+    typeof provision[1] !== "string"
+  )
+    throw new Error("Invalid arguments");
+
+  let attr: string, val: any;
+
+  if (provision.length === 3) {
+    attr = "value";
+    val = provision[2];
+  } else {
+    attr = (provision[2] as string) || "";
+    val = provision[3];
+  }
+
+  if (attr === "accessList") {
+    val = (val || "")
+      .split(",")
+      .map((s) => s.trim())
+      .filter((s) => !!s);
+  } else if (attr === "value") {
+    val = [val];
+  }
+
   declarations.push({
     path: Path.parse(provision[1]),
     pathGet: 1,
     pathSet: null,
-    attrGet: { value: 1 },
+    attrGet: { [attr]: 1 },
-    attrSet: { value: [provision[2]] },
+    attrSet: { [attr]: val },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
-export function tag(sessionContext, provision, declarations): boolean {
+export function tag(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (
+    provision.length !== 3 ||
+    typeof provision[1] !== "string" ||
+    typeof provision[2] !== "boolean"
+  )
+    throw new Error("Invalid arguments");
+
   declarations.push({
     path: Path.parse(`Tags.${provision[1]}`),
     pathGet: 1,
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [provision[2]] },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
-export function reboot(sessionContext, provision, declarations): boolean {
+export function reboot(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
   declarations.push({
     path: Path.parse("Reboot"),
     pathGet: 1,
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
-export function reset(sessionContext, provision, declarations): boolean {
+export function reset(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
   declarations.push({
     path: Path.parse("FactoryReset"),
     pathGet: 1,
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
-export function download(sessionContext, provision, declarations): boolean {
+export function download(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string") &&
+    (provision.length !== 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string" ||
+      typeof provision[3] !== "string")
+  )
+    throw new Error("Invalid arguments");
+
   const alias = [
     `FileType:${JSON.stringify(provision[1] || "")}`,
     `FileName:${JSON.stringify(provision[2] || "")}`,
-    `TargetFileName:${JSON.stringify(provision[3] || "")}`
+    `TargetFileName:${JSON.stringify(provision[3] || "")}`,
   ].join(",");
 
   declarations.push({
@@ -113,7 +214,7 @@ export function download(sessionContext, provision, declarations): boolean {
     pathSet: 1,
     attrGet: null,
     attrSet: null,
-    defer: true
+    defer: true,
   });
 
   declarations.push({
@@ -122,22 +223,25 @@ export function download(sessionContext, provision, declarations): boolean {
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
 export function instances(
-  sessionContext,
+  sessionContext: SessionContext,
-  provision,
+  provision: (string | number | boolean)[],
-  declarations,
+  declarations: Declaration[],
-  startRevision,
+  startRevision: number,
-  endRevision
+  endRevision: number
 ): boolean {
+  if (provision.length !== 3 || typeof provision[1] !== "string")
+    throw new Error("Invalid arguments");
+
   let count = Number(provision[2]);
 
-  if (Number.isNaN(count)) return true;
+  if (Number.isNaN(count)) throw new Error("Invalid arguments");
 
   const path = Path.parse(provision[1]);
 
@@ -148,7 +252,7 @@ export function instances(
       pathSet: null,
       attrGet: null,
       attrSet: null,
-      defer: true
+      defer: true,
     });
 
     if (endRevision === startRevision) return false;
@@ -167,7 +271,7 @@ export function instances(
     pathSet: count,
     attrGet: null,
     attrSet: null,
-    defer: true
+    defer: true,
   });
 
   return true;

lib/device.ts

@@ -18,12 +18,20 @@
  */
 
 import Path from "./common/path";
-import { DeviceData, Declaration, Clear } from "./types";
+import {
+  DeviceData,
+  Declaration,
+  Clear,
+  AttributeTimestamps,
+  Attributes,
+} from "./types";
 
 const CHANGE_FLAGS = {
   object: 2,
   writable: 4,
-  value: 8
+  value: 8,
+  notification: 16,
+  accessList: 32,
 };
 
 function parseBool(v): boolean {
@@ -36,7 +44,7 @@ function parseBool(v): boolean {
 
 export function sanitizeParameterValue(
   parameterValue: [string | number | boolean, string]
-): [string | number | boolean, string?] {
+): [string | number | boolean, string] {
   if (parameterValue[0] != null) {
     switch (parameterValue[1]) {
       case "xsd:boolean":
@@ -96,8 +104,8 @@ export function getAliasDeclarations(
       pathSet: null,
       attrGet: attrGet,
       attrSet: null,
-      defer: true
+      defer: true,
-    }
+    },
   ];
 
   if (path.alias) {
@@ -107,7 +115,7 @@ export function getAliasDeclarations(
         for (const [p] of alias as [Path, string][]) {
           decs = decs.concat(
             getAliasDeclarations(parent.concat(p), timestamp, {
-              value: timestamp
+              value: timestamp,
             })
           );
         }
@@ -189,10 +197,10 @@ export function unpack(
 }
 
 export function clear(
-  deviceData,
+  deviceData: DeviceData,
-  path,
+  path: Path,
-  timestamp,
+  timestamp: number,
-  attributes,
+  attributes: AttributeTimestamps,
   changeFlags = 0
 ): void {
   const changeTrackers = {};
@@ -271,8 +279,8 @@ function compareEquality(a, b): boolean {
 export function set(
   deviceData: DeviceData,
   path: Path,
-  timestamp,
+  timestamp: number,
-  attributes,
+  attributes: Attributes,
   toClear?: Clear[]
 ): Clear[] {
   path = deviceData.paths.add(path);
@@ -300,7 +308,7 @@ export function set(
       attributes.object[1] &&
       attributes.object[0] >= (attributes.value ? attributes.value[0] : 0)
     )
-      attributes.value = [attributes.object[0]];
+      attributes.value = [attributes.object[0], null];
 
     const newAttributes = Object.assign({}, currentAttributes, attributes);
 
@@ -376,7 +384,7 @@ export function track(
   deviceData: DeviceData,
   path: Path,
   marker: string,
-  attributes?
+  attributes?: string[]
 ): void {
   path = deviceData.paths.add(path);
   let f = 1;
@@ -393,7 +401,10 @@ export function track(
   cur[marker] |= f;
 }
 
-export function clearTrackers(deviceData: DeviceData, tracker): void {
+export function clearTrackers(
+  deviceData: DeviceData,
+  tracker: string | string[]
+): void {
   if (Array.isArray(tracker)) {
     for (const v of deviceData.trackers.values())
       for (const t of tracker) delete v[t];

lib/extensions.ts

@@ -21,6 +21,7 @@ import { spawn, ChildProcess } from "child_process";
 import * as crypto from "crypto";
 import * as config from "./config";
 import { Fault } from "./types";
+import { ROOT_DIR } from "./config";
 
 const TIMEOUT = +config.get("EXT_TIMEOUT");
 
@@ -35,28 +36,29 @@ function messageHandler(message): void {
   }
 }
 
-export function run(args): Promise<{ fault: Fault; value: any }> {
+export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
-  return new Promise(resolve => {
+  return new Promise((resolve) => {
     const scriptName = args[0];
 
     const id = crypto.randomBytes(8).toString("hex");
     jobs.set(id, resolve);
 
     if (!processes[scriptName]) {
-      const p = spawn("bin/genieacs-ext", [scriptName], {
+      const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
-        stdio: ["inherit", "inherit", "inherit", "ipc"]
+        stdio: ["inherit", "inherit", "inherit", "ipc"],
       });
       processes[scriptName] = p;
 
-      p.on("error", err => {
+      p.on("error", (err) => {
         if (processes[scriptName] === p) {
           if (jobs.delete(id)) {
             resolve({
               fault: { code: err.name, message: err.message },
-              value: null
+              value: null,
             });
           }
 
+          // eslint-disable-next-line @typescript-eslint/no-floating-promises
           kill(processes[scriptName]);
           delete processes[scriptName];
         }
@@ -73,7 +75,7 @@ export function run(args): Promise<{ fault: Fault; value: any }> {
       if (jobs.delete(id)) {
         resolve({
           fault: { code: "timeout", message: "Extension timed out" },
-          value: null
+          value: null,
         });
       }
     }, TIMEOUT);
@@ -85,7 +87,7 @@ export function run(args): Promise<{ fault: Fault; value: any }> {
 }
 
 function kill(process: ChildProcess): Promise<void> {
-  return new Promise(resolve => {
+  return new Promise((resolve) => {
     const timeToKill = Date.now() + 5000;
 
     process.kill();

lib/forwarded.ts

@@ -0,0 +1,175 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { IncomingMessage } from "http";
+import { TLSSocket } from "tls";
+import { parseCIDR, parse, IPv6, IPv4 } from "ipaddr.js";
+import * as config from "./config";
+
+interface RequestOrigin {
+  localAddress: string;
+  localPort: number;
+  remoteAddress: string;
+  remotePort: number;
+  host: string;
+  encrypted: boolean;
+}
+
+const FORWARDED_HEADER = "" + config.get("FORWARDED_HEADER");
+const cache = new WeakMap<IncomingMessage, RequestOrigin>();
+const cidrs: [IPv4 | IPv6, number][] = [];
+
+for (const str of FORWARDED_HEADER.split(",").map((s) => s.trim())) {
+  try {
+    cidrs.push(parseCIDR(str));
+  } catch (err) {
+    // Not a valid CIDR format, try parsing as IP
+    try {
+      const ip = parse(str);
+      cidrs.push([ip, ip.toByteArray().length * 8]);
+    } catch (err) {
+      // Not a valid IP either, ignore
+    }
+  }
+}
+
+function parseForwardedHeader(str: string): { [name: string]: string } {
+  str = str.toLowerCase();
+  const res: { [name: string]: string } = {};
+  let keyIdx = 0;
+  let valueIdx = -1;
+  let key: string;
+  for (let i = 0; i < str.length; ++i) {
+    const char = str.charCodeAt(i);
+    if (char === 61 /* = */) {
+      if (keyIdx >= 0) {
+        key = str.slice(keyIdx, i).trim();
+        keyIdx = -1;
+        valueIdx = i + 1;
+      }
+    } else if (char === 59 /* ; */) {
+      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
+      valueIdx = -1;
+      keyIdx = i + 1;
+    } else if (char === 44 /* , */) {
+      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
+      return res;
+    } else if (char === 34 /* " */) {
+      if (valueIdx >= 0) {
+        const quoteIdx = i;
+        if (!str.slice(valueIdx, quoteIdx).trim()) {
+          for (i = i + 1; i < str.length; ++i) {
+            const c = str.charCodeAt(i);
+            if (c === 92 /* \ */) ++i;
+            if (c === 34 /* " */) {
+              res[key] = JSON.parse(str.slice(quoteIdx, i + 1).trim());
+              valueIdx = -1;
+              keyIdx = i + 1;
+              break;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  if (valueIdx >= 0) res[key] = str.slice(valueIdx).trim();
+
+  return res;
+}
+
+export function getRequestOrigin(request: IncomingMessage): RequestOrigin {
+  let origin = cache.get(request);
+  if (!origin) {
+    const soc = request.socket;
+    origin = {
+      localAddress: soc.localAddress,
+      localPort: soc.localPort,
+      remoteAddress: soc.remoteAddress,
+      remotePort: soc.remotePort,
+      host: request.headers["host"],
+      encrypted: !!(request.socket as TLSSocket).encrypted,
+    };
+
+    const header = request.headers["forwarded"];
+    if (header) {
+      const ip = parse(soc.remoteAddress) as IPv4;
+      if (cidrs.some((cidr) => ip.match(cidr as [IPv4, number]))) {
+        const parsed = parseForwardedHeader(header);
+
+        if (parsed["proto"] === "https") {
+          origin.encrypted = true;
+          origin.localPort = 443;
+        } else if (parsed["proto"] === "http") {
+          origin.encrypted = false;
+          origin.localPort = 80;
+        }
+
+        if (parsed["host"]) {
+          origin.host = parsed["host"];
+          const [, port] = parsed["host"].split(":", 2);
+          origin.localPort = +port || origin.localPort;
+        }
+
+        if (parsed["for"]) {
+          if (parsed["for"].startsWith("[")) {
+            const i = parsed["for"].lastIndexOf("]");
+            if (i >= 0) {
+              origin.remoteAddress = parsed["for"].slice(1, i);
+              origin.remotePort =
+                parseInt(parsed["for"].slice(i + 2)) || origin.remotePort;
+            }
+          } else {
+            const i = parsed["for"].lastIndexOf(":");
+            if (i >= 0) {
+              origin.remoteAddress = parsed["for"].slice(0, i);
+              origin.remotePort =
+                parseInt(parsed["for"].slice(i + 1)) || origin.remotePort;
+            } else {
+              origin.remoteAddress = parsed["for"];
+            }
+          }
+        }
+
+        if (parsed["by"]) {
+          if (parsed["by"].startsWith("[")) {
+            const i = parsed["by"].lastIndexOf("]");
+            if (i >= 0) {
+              origin.localAddress = parsed["by"].slice(1, i);
+              origin.localPort =
+                parseInt(parsed["by"].slice(i + 2)) || origin.localPort;
+            }
+          } else {
+            const i = parsed["by"].lastIndexOf(":");
+            if (i >= 0) {
+              origin.localAddress = parsed["by"].slice(0, i);
+              origin.localPort =
+                parseInt(parsed["by"].slice(i + 1)) || origin.localPort;
+            } else {
+              origin.localAddress = parsed["by"];
+            }
+          }
+        }
+      }
+    }
+
+    cache.set(request, origin);
+  }
+  return origin;
+}

lib/fs.ts

@@ -19,11 +19,16 @@
 
 import * as url from "url";
 import * as querystring from "querystring";
+import { IncomingMessage, ServerResponse } from "http";
 import { GridFSBucket } from "mongodb";
 import * as db from "./db";
 import * as logger from "./logger";
+import { getRequestOrigin } from "./forwarded";
 
-export function listener(request, response): void {
+export function listener(
+  request: IncomingMessage,
+  response: ServerResponse
+): void {
   const urlParts = url.parse(request.url, true);
   if (request.method === "GET") {
     const filename = querystring.unescape(urlParts.pathname.substring(1));
@@ -31,7 +36,7 @@ export function listener(request, response): void {
     const log = {
       message: "Fetch file",
       filename: filename,
-      remoteAddress: request.connection.remoteAddress
+      remoteAddress: getRequestOrigin(request).remoteAddress,
     };
 
     db.filesCollection.findOne({ _id: filename }, (err, file) => {
@@ -47,7 +52,7 @@ export function listener(request, response): void {
 
       response.writeHead(200, {
         "Content-Type": file.contentType || "application/octet-stream",
-        "Content-Length": file.length
+        "Content-Length": file.length,
       });
 
       const bucket = new GridFSBucket(db.client.db());

lib/gpn-heuristic.ts

@@ -1,3 +1,5 @@
+import Path from "./common/path";
+
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -27,9 +29,12 @@ const UNDISCOVERED_DEPTH = 7;
 // flags is an int where its bits mark the segments in the pattern that
 // need refreshing. Leading 0s indicate that the pattern up to that
 // point has been discovered.
-export function estimateGpnCount(gpnPatterns, depth = 0): number {
+export function estimateGpnCount(
-  const children = {};
+  gpnPatterns: [Path, number][],
-  const wildcardChildren = [];
+  depth = 0
+): number {
+  const children: { [segment: string]: [Path, number][] } = {};
+  const wildcardChildren: [Path, number][] = [];
   let wildcardDiscovered = false;
   let gpnCount = 0;
 
@@ -37,7 +42,7 @@ export function estimateGpnCount(gpnPatterns, depth = 0): number {
     const path = pattern[0];
     const flags = pattern[1] >> depth;
 
-    const k = path[depth];
+    const k = path.segments[depth] as string;
 
     if (!k) {
       if (flags & 1) gpnCount = 1;

lib/init.ts

@@ -60,7 +60,7 @@ const INFORM_SCRIPT = `
 const username = declare("DeviceID.ID", {value: 1}).value[0]
 
 // Password will be fixed a given device because Math.random() is seeded with devcie ID by default.
-const password = "" + Math.random();
+const password = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER).toString(36);
 
 const informInterval = 300;
 
@@ -95,11 +95,11 @@ export async function getStatus(): Promise<Status> {
     filters: !Object.keys(ui["filters"]).length,
     device: !Object.keys(ui["device"]).length,
     index: !Object.keys(ui["index"]).length,
-    overview: !Object.keys(ui["overview"]).length
+    overview: !Object.keys(ui["overview"]).length,
   };
 }
 
-export async function seed(options): Promise<void> {
+export async function seed(options: Record<string, boolean>): Promise<void> {
   const resources = {};
   const proms = [];
 
@@ -117,12 +117,12 @@ export async function seed(options): Promise<void> {
         role: "admin",
         resource: "virtualParameters",
         access: 3,
-        validate: "true"
+        validate: "true",
-      }
+      },
     ];
 
     resources["users"] = [
-      { username: "admin", password: "admin", roles: ["admin"] }
+      { username: "admin", password: "admin", roles: ["admin"] },
     ];
   }
 
@@ -135,8 +135,7 @@ export async function seed(options): Promise<void> {
       { _id: "ui.filters.1.parameter", value: "DeviceID.ProductClass" },
       { _id: "ui.filters.1.type", value: "'string'" },
       { _id: "ui.filters.2.label", value: "'Tag'" },
-      { _id: "ui.filters.2.parameter", value: "tag" },
+      { _id: "ui.filters.2.type", value: "'tag'" },
-      { _id: "ui.filters.2.type", value: "'string'" }
     ]);
   }
 
@@ -150,118 +149,118 @@ export async function seed(options): Promise<void> {
       { _id: "ui.device.2.parameters.0.label", value: "'Last inform'" },
       {
         _id: "ui.device.2.parameters.0.components.0.type",
-        value: "'parameter'"
+        value: "'parameter'",
       },
       { _id: "ui.device.2.parameters.0.components.1.chart", value: "'online'" },
       {
         _id: "ui.device.2.parameters.0.components.1.type",
-        value: "'overview-dot'"
+        value: "'overview-dot'",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.type",
-        value: "'summon-button'"
+        value: "'summon-button'",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.0",
-        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.1",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.2",
         value:
-          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress"
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.3",
         value:
-          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress"
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.4",
-        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID"
+        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.5",
         value:
-          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase"
+          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.6",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName"
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.7",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress"
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.8",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress"
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress",
       },
       {
         _id: "ui.device.2.parameters.0.parameter",
-        value: "DATE_STRING(Events.Inform)"
+        value: "DATE_STRING(Events.Inform)",
       },
       { _id: "ui.device.2.parameters.1.label", value: "'Serial number'" },
       {
         _id: "ui.device.2.parameters.1.parameter",
-        value: "DeviceID.SerialNumber"
+        value: "DeviceID.SerialNumber",
       },
       { _id: "ui.device.2.parameters.2.label", value: "'Product class'" },
       {
         _id: "ui.device.2.parameters.2.parameter",
-        value: "DeviceID.ProductClass"
+        value: "DeviceID.ProductClass",
       },
       { _id: "ui.device.2.parameters.3.label", value: "'OUI'" },
       {
         _id: "ui.device.2.parameters.3.parameter",
-        value: "DeviceID.OUI"
+        value: "DeviceID.OUI",
       },
       { _id: "ui.device.2.parameters.4.label", value: "'Manufacturer'" },
       {
         _id: "ui.device.2.parameters.4.parameter",
-        value: "DeviceID.Manufacturer"
+        value: "DeviceID.Manufacturer",
       },
       { _id: "ui.device.2.parameters.5.label", value: "'Hardware version'" },
       {
         _id: "ui.device.2.parameters.5.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
       },
       { _id: "ui.device.2.parameters.6.label", value: "'Software version'" },
       {
         _id: "ui.device.2.parameters.6.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
       },
       { _id: "ui.device.2.parameters.7.label", value: "'MAC'" },
       {
         _id: "ui.device.2.parameters.7.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress"
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress",
       },
       { _id: "ui.device.2.parameters.8.label", value: "'IP'" },
       {
         _id: "ui.device.2.parameters.8.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
       },
       { _id: "ui.device.2.parameters.9.label", value: "'WLAN SSID'" },
       {
         _id: "ui.device.2.parameters.9.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
       },
       { _id: "ui.device.2.parameters.10.label", value: "'WLAN passphrase'" },
       {
         _id: "ui.device.2.parameters.10.parameter",
         value:
-          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase"
+          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
       },
       { _id: "ui.device.3.type", value: "'parameter-table'" },
       { _id: "ui.device.3.label", value: "'WLAN hosts'" },
       {
         _id: "ui.device.3.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host"
+        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host",
       },
       { _id: "ui.device.3.childParameters.0.label", value: "'Host name'" },
       { _id: "ui.device.3.childParameters.0.parameter", value: "HostName" },
@@ -279,12 +278,12 @@ export async function seed(options): Promise<void> {
       { _id: "ui.device.5.element", value: "'div.container-full-width'" },
       {
         _id: "ui.device.5.components.0.components.0",
-        value: "'All parameters'"
+        value: "'All parameters'",
       },
       { _id: "ui.device.5.components.0.element", value: "'h3'" },
       { _id: "ui.device.5.components.0.type", value: "'container'" },
       { _id: "ui.device.5.components.1.type", value: "'all-parameters'" },
-      { _id: "ui.device.6.type", value: "'device-actions'" }
+      { _id: "ui.device.6.type", value: "'device-actions'" },
     ]);
   }
 
@@ -299,18 +298,18 @@ export async function seed(options): Promise<void> {
       { _id: "ui.index.2.label", value: "'Software version'" },
       {
         _id: "ui.index.2.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
       },
       { _id: "ui.index.3.label", value: "'IP'" },
       {
         _id: "ui.index.3.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
       },
       { _id: "ui.index.4.label", value: "'SSID'" },
       {
         _id: "ui.index.4.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
       },
       { _id: "ui.index.5.type", value: "'container'" },
       { _id: "ui.index.5.label", value: "'Last inform'" },
@@ -323,7 +322,7 @@ export async function seed(options): Promise<void> {
       { _id: "ui.index.6.label", value: "'Tags'" },
       { _id: "ui.index.6.parameter", value: "Tags" },
       { _id: "ui.index.6.unsortable", value: "true" },
-      { _id: "ui.index.6.writable", value: "false" }
+      { _id: "ui.index.6.writable", value: "false" },
     ]);
   }
 
@@ -332,43 +331,44 @@ export async function seed(options): Promise<void> {
       { _id: "ui.overview.charts.online.label", value: "'Online status'" },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.color",
-        value: "'#31a354'"
+        value: "'#31a354'",
       },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.filter",
-        value: "Events.Inform > NOW() - 5 * 60 * 1000"
+        value: "Events.Inform > NOW() - 5 * 60 * 1000",
       },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.label",
-        value: "'Online now'"
+        value: "'Online now'",
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.color",
-        value: "'#a1d99b'"
+        value: "'#a1d99b'",
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.filter",
         value:
-          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)"
+          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)",
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.label",
-        value: "'Past 24 hours'"
+        value: "'Past 24 hours'",
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.color",
-        value: "'#e5f5e0'"
+        value: "'#e5f5e0'",
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.filter",
-        value: "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)"
+        value:
+          "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)",
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.label",
-        value: "'Others'"
+        value: "'Others'",
       },
       { _id: "ui.overview.groups.online.label", value: "''" },
-      { _id: "ui.overview.groups.online.charts.0", value: "'online'" }
+      { _id: "ui.overview.groups.online.charts.0", value: "'online'" },
     ]);
   }
 
@@ -378,16 +378,17 @@ export async function seed(options): Promise<void> {
         _id: "bootstrap",
         weight: 0,
         channel: "bootstrap",
-        provision: "bootstrap"
+        events: "0 BOOTSTRAP",
+        provision: "bootstrap",
       },
       { _id: "default", weight: 0, channel: "default", provision: "default" },
-      { _id: "inform", weight: 0, channel: "inform", provision: "inform" }
+      { _id: "inform", weight: 0, channel: "inform", provision: "inform" },
     ];
 
     resources["provisions"] = [
       { _id: "bootstrap", script: BOOTSTRAP_SCRIPT },
       { _id: "default", script: DEFAULT_SCRIPT },
-      { _id: "inform", script: INFORM_SCRIPT }
+      { _id: "inform", script: INFORM_SCRIPT },
     ];
   }
 

lib/local-cache.ts

@@ -22,30 +22,39 @@ import * as crypto from "crypto";
 import * as config from "./config";
 import * as db from "./db";
 import * as cache from "./cache";
-import { mongoQueryToFilter } from "./mongodb-functions";
+import { convertOldPrecondition } from "./mongodb-functions";
 import * as logger from "./logger";
 import * as scheduling from "./scheduling";
 import * as expression from "./common/expression";
 import { parse } from "./common/expression-parser";
-import { Preset, Expression } from "./types";
+import {
+  Preset,
+  Expression,
+  Provisions,
+  VirtualParameters,
+  Files,
+  Users,
+  Permissions,
+  Config,
+  UiConfig,
+} from "./types";
 
-interface Permissions {
+interface Snapshot {
-  [role: string]: {
+  presets: Preset[];
-    [access: number]: {
+  provisions: Provisions;
-      [resource: string]: {
+  virtualParameters: VirtualParameters;
-        access: number;
+  files: Files;
-        filter: Expression;
+  permissions: Permissions;
-        validate?: Expression;
+  users: Users;
-      };
+  config: Config;
-    };
+  ui: UiConfig;
-  };
 }
 
 const REFRESH = 3000;
 const EVICT_TIMEOUT = 60000;
 
-const snapshots = new Map();
+const snapshots = new Map<string, Snapshot>();
-let currentSnapshot = null;
+let currentSnapshot: string = null;
 let nextRefresh = 1;
 
 function computeHash(snapshot): string {
@@ -98,11 +107,15 @@ function computeHash(snapshot): string {
   return h.digest("hex");
 }
 
-function flattenObject(src, prefix = "", dst = {}): {} {
+function flattenObject(
+  src: Record<string, unknown>,
+  prefix = "",
+  dst = {}
+): Record<string, unknown> {
   for (const k of Object.keys(src)) {
     const v = src[k];
     if (typeof v === "object" && !Array.isArray(v))
-      flattenObject(v, `${prefix}${k}.`, dst);
+      flattenObject(v as Record<string, unknown>, `${prefix}${k}.`, dst);
     else dst[`${prefix}${k}`] = v;
   }
   return dst;
@@ -112,14 +125,14 @@ async function fetchPresets(): Promise<Preset[]> {
   const res = await db.getPresets();
   let objects = await db.getObjects();
 
-  objects = objects.map(obj => {
+  objects = objects.map((obj) => {
     // Flatten object
     obj = flattenObject(obj);
 
     // If no keys are defined, consider all parameters as keys to keep the
     // same behavior from v1.0
     if (!obj["_keys"] || !obj["_keys"].length)
-      obj["_keys"] = Object.keys(obj).filter(k => !k.startsWith("_"));
+      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
 
     return obj;
   });
@@ -136,12 +149,9 @@ async function fetchPresets(): Promise<Preset[]> {
     if (preset["schedule"]) {
       const parts = preset["schedule"].trim().split(/\s+/);
       schedule = {
-        md5: crypto
+        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
-          .createHash("md5")
-          .update(preset["schedule"])
-          .digest("hex"),
         duration: null,
-        schedule: null
+        schedule: null,
       };
 
       try {
@@ -151,7 +161,7 @@ async function fetchPresets(): Promise<Preset[]> {
         logger.warn({
           message: "Invalid preset schedule",
           preset: preset["_id"],
-          schedule: preset["schedule"]
+          schedule: preset["schedule"],
         });
         schedule.schedule = false;
       }
@@ -163,7 +173,9 @@ async function fetchPresets(): Promise<Preset[]> {
       try {
         precondition = parse(preset["precondition"]);
       } catch (error) {
-        precondition = mongoQueryToFilter(JSON.parse(preset["precondition"]));
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"])
+        );
       }
 
       // Simplify expression
@@ -176,7 +188,7 @@ async function fetchPresets(): Promise<Preset[]> {
     for (const c of preset["configurations"]) {
       switch (c.type) {
         case "age":
-          _provisions.push(["refresh", c.name, c.age]);
+          _provisions.push(["refresh", c.name, +c.age]);
           break;
 
         case "value":
@@ -199,7 +211,7 @@ async function fetchPresets(): Promise<Preset[]> {
           for (const obj of objects) {
             if (obj["_id"] === c.object) {
               const alias = obj["_keys"]
-                .map(k => `${k}:${JSON.stringify(obj[k])}`)
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
                 .join(",");
               const p = `${c.name}.[${alias}]`;
               _provisions.push(["instances", p, 1]);
@@ -217,7 +229,7 @@ async function fetchPresets(): Promise<Preset[]> {
           for (const obj of objects) {
             if (obj["_id"] === c.object) {
               const alias = obj["_keys"]
-                .map(k => `${k}:${JSON.stringify(obj[k])}`)
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
                 .join(",");
               const p = `${c.name}.[${alias}]`;
               _provisions.push(["instances", p, 0]);
@@ -237,14 +249,14 @@ async function fetchPresets(): Promise<Preset[]> {
       schedule: schedule,
       events: events,
       precondition: precondition,
-      provisions: _provisions
+      provisions: _provisions,
     });
   }
 
   return presets;
 }
 
-async function fetchProvisions(): Promise<{}> {
+async function fetchProvisions(): Promise<Provisions> {
   const res = await db.getProvisions();
 
   const provisions = {};
@@ -254,7 +266,9 @@ async function fetchProvisions(): Promise<{}> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    provisions[r["_id"]].script = new vm.Script(
+    provisions[
+      r["_id"]
+    ].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -263,7 +277,7 @@ async function fetchProvisions(): Promise<{}> {
   return provisions;
 }
 
-async function fetchVirtualParameters(): Promise<{}> {
+async function fetchVirtualParameters(): Promise<VirtualParameters> {
   const res = await db.getVirtualParameters();
 
   const virtualParameters = {};
@@ -273,7 +287,9 @@ async function fetchVirtualParameters(): Promise<{}> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    virtualParameters[r["_id"]].script = new vm.Script(
+    virtualParameters[
+      r["_id"]
+    ].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -292,7 +308,7 @@ async function fetchPermissions(): Promise<Permissions> {
 
     permissions[p.role][p.access][p.resource] = {
       access: p.access,
-      filter: parse(p.filter || "true")
+      filter: parse(p.filter || "true"),
     };
     if (p.validate)
       permissions[p.role][p.access][p.resource].validate = parse(p.validate);
@@ -301,7 +317,7 @@ async function fetchPermissions(): Promise<Permissions> {
   return permissions;
 }
 
-async function fetchFiles(): Promise<{}> {
+async function fetchFiles(): Promise<Files> {
   const res = await db.getFiles();
   const files = {};
 
@@ -316,7 +332,7 @@ async function fetchFiles(): Promise<{}> {
   return files;
 }
 
-async function fetchUsers(): Promise<{}> {
+async function fetchUsers(): Promise<Users> {
   const _users = await db.getUsers();
   const users = {};
 
@@ -324,14 +340,14 @@ async function fetchUsers(): Promise<{}> {
     users[user._id] = {
       password: user.password,
       salt: user.salt,
-      roles: user.roles.split(",").map(s => s.trim())
+      roles: user.roles.split(",").map((s) => s.trim()),
     };
   }
 
   return users;
 }
 
-async function fetchConfig(): Promise<[{}, {}]> {
+async function fetchConfig(): Promise<[Config, UiConfig]> {
   const conf = await db.getConfig();
 
   conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
@@ -340,7 +356,7 @@ async function fetchConfig(): Promise<[{}, {}]> {
     filters: {},
     device: {},
     index: {},
-    overview: {}
+    overview: {},
   };
   const _config = {};
 
@@ -367,7 +383,7 @@ async function fetchConfig(): Promise<[{}, {}]> {
 
 async function refresh(): Promise<void> {
   if (!nextRefresh) {
-    await new Promise(resolve => setTimeout(resolve, 20));
+    await new Promise((resolve) => setTimeout(resolve, 20));
     await refresh();
     return;
   }
@@ -391,7 +407,7 @@ async function refresh(): Promise<void> {
     fetchFiles(),
     fetchPermissions(),
     fetchUsers(),
-    fetchConfig()
+    fetchConfig(),
   ]);
 
   const snapshot = {
@@ -402,7 +418,7 @@ async function refresh(): Promise<void> {
     permissions: res[4],
     users: res[5],
     config: res[6][0],
-    ui: res[6][1]
+    ui: res[6][1],
   };
 
   if (currentSnapshot) {
@@ -426,45 +442,40 @@ export async function getCurrentSnapshot(): Promise<string> {
   return currentSnapshot;
 }
 
-export function hasSnapshot(hash): boolean {
+export function hasSnapshot(hash: string): boolean {
   return snapshots.has(hash);
 }
 
-export function getPresets(snapshotKey): Preset[] {
+export function getPresets(snapshotKey: string): Preset[] {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.presets;
 }
 
-export function getProvisions(
+export function getProvisions(snapshotKey: string): Provisions {
-  snapshotKey
-): { [name: string]: { md5: string; script: vm.Script } } {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.provisions;
 }
 
-export function getVirtualParameters(
+export function getVirtualParameters(snapshotKey: string): VirtualParameters {
-  snapshotKey
-): { [name: string]: { md5: string; script: vm.Script } } {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.virtualParameters;
 }
 
-export function getFiles(
+export function getFiles(snapshotKey: string): Files {
-  snapshotKey
-): { [name: string]: { length: number; md5: string; contentType: string } } {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.files;
 }
 
 export function getConfig(
-  snapshotKey,
+  snapshotKey: string,
-  key,
+  key: string,
-  context?,
+  context: Record<string, unknown>,
-  now?
+  now: number,
+  cb?: (e: Expression) => Expression
 ): string | number | boolean | null {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
@@ -484,39 +495,52 @@ export function getConfig(
       "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
     "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
     "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
-    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT"
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
   };
 
   if (!(key in snapshot.config)) {
-    if (key in oldOpts)
+    if (key in oldOpts) {
-      return config.get(oldOpts[key], context ? context.id : null);
+      let id;
-    else return null;
+      if (context && context["id"]) {
+        id = context["id"];
+      } else if (cb) {
+        id = cb(["PARAM", "DeviceID.ID"]);
+        if (Array.isArray(id)) id = null;
       }
-  return expression.evaluate(snapshot.config[key], context, now || Date.now());
+      return config.get(oldOpts[key], id);
+    }
+    return null;
+  }
+
+  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+  return Array.isArray(v) ? null : v;
 }
 
-export function getConfigExpression(snapshotKey, key): Expression {
+export function getConfigExpression(
+  snapshotKey: string,
+  key: string
+): Expression {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 
   return snapshot.config[key];
 }
 
-export function getUsers(snapshotKey): {} {
+export function getUsers(snapshotKey: string): Users {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 
   return snapshot.users;
 }
 
-export function getPermissions(snapshotKey): {} {
+export function getPermissions(snapshotKey: string): Permissions {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 
   return snapshot.permissions;
 }
 
-export function getUiConfig(snapshotKey): {} {
+export function getUiConfig(snapshotKey: string): UiConfig {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 

lib/logger.ts

@@ -21,6 +21,15 @@ import * as fs from "fs";
 import * as os from "os";
 
 import * as config from "./config";
+import { getRequestOrigin } from "./forwarded";
+import {
+  SessionContext,
+  AcsRequest,
+  CpeRequest,
+  CpeFault,
+  InformRequest,
+  Fault,
+} from "./types";
 
 const REOPEN_EVERY = 60000;
 
@@ -66,7 +75,7 @@ function reopen(): void {
       if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
         logStream.end();
         logStream = fs.createWriteStream(null, {
-          fd: fs.openSync(LOG_FILE, "a")
+          fd: fs.openSync(LOG_FILE, "a"),
         });
         logStat = fs.fstatSync(logStream.fd);
       }
@@ -90,7 +99,7 @@ function reopen(): void {
       ) {
         accessLogStream.end();
         accessLogStream = fs.createWriteStream(null, {
-          fd: fs.openSync(ACCESS_LOG_FILE, "a")
+          fd: fs.openSync(ACCESS_LOG_FILE, "a"),
         });
         accessLogStat = fs.fstatSync(accessLogStream.fd);
       }
@@ -104,7 +113,7 @@ function reopen(): void {
     setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
 }
 
-export function init(service, version): void {
+export function init(service: string, version: string): void {
   defaultMeta.hostname = os.hostname();
   defaultMeta.pid = process.pid;
   defaultMeta.name = `genieacs-${service}`;
@@ -120,7 +129,7 @@ export function init(service, version): void {
 
   if (ACCESS_LOG_FILE) {
     accessLogStream = fs.createWriteStream(null, {
-      fd: fs.openSync(ACCESS_LOG_FILE, "a")
+      fd: fs.openSync(ACCESS_LOG_FILE, "a"),
     });
     accessLogStat = fs.fstatSync(accessLogStream.fd);
   }
@@ -146,61 +155,75 @@ export function close(): void {
   logStream.end();
 }
 
-export function flatten(details): {} {
+export function flatten(
+  details: Record<string, unknown>
+): Record<string, unknown> {
   if (details.sessionContext) {
-    details.deviceId = details.sessionContext.deviceId;
+    const sessionContext = details.sessionContext as SessionContext;
-    details.remoteAddress =
+    details.deviceId = sessionContext.deviceId;
-      details.sessionContext.httpRequest.connection.remoteAddress;
+    details.remoteAddress = getRequestOrigin(
+      sessionContext.httpRequest
+    ).remoteAddress;
     delete details.sessionContext;
   }
 
   if (details.exception) {
-    details.exceptionName = details.exception.name;
+    const err = details.exception as Error;
-    details.exceptionMessage = details.exception.message;
+    details.exceptionName = err.name;
-    details.exceptionStack = details.exception.stack;
+    details.exceptionMessage = err.message;
+    details.exceptionStack = err.stack;
     delete details.exception;
   }
 
   if (details.task) {
-    details.taskId = details.task._id;
+    details.taskId = details.task["_id"];
     delete details.task;
   }
 
   if (details.rpc) {
-    if (details.rpc.acsRequest) {
+    const rpc = details.rpc as {
-      details.acsRequestId = details.rpc.id;
+      id: string;
-      details.acsRequestName = details.rpc.acsRequest.name;
+      acsRequest?: AcsRequest;
-      if (details.rpc.acsRequest.commandKey)
+      cpeRequest?: CpeRequest;
-        details.acsRequestCommandKey = details.rpc.acsRequest.commandKey;
+      cpeFault?: CpeFault;
-    } else if (details.rpc.cpeRequest) {
+    };
-      details.cpeRequestId = details.rpc.id;
+    if (rpc.acsRequest) {
-      if (details.rpc.cpeRequest.name === "Inform") {
+      details.acsRequestId = rpc.id;
-        details.informEvent = details.rpc.cpeRequest.event.join(",");
+      details.acsRequestName = rpc.acsRequest.name;
-        details.informRetryCount = details.rpc.cpeRequest.retryCount;
+      if (rpc.acsRequest["commandKey"])
+        details.acsRequestCommandKey = rpc.acsRequest["commandKey"];
+    } else if (rpc.cpeRequest) {
+      details.cpeRequestId = rpc.id;
+      if (rpc.cpeRequest.name === "Inform") {
+        details.informEvent = (rpc.cpeRequest as InformRequest).event.join(",");
+        details.informRetryCount = (rpc.cpeRequest as InformRequest).retryCount;
       } else {
-        details.cpeRequestName = details.rpc.cpeRequest.name;
+        details.cpeRequestName = rpc.cpeRequest.name;
-        if (details.rpc.cpeRequest.commandKey)
+        if (rpc.cpeRequest["commandKey"])
-          details.cpeRequestCommandKey = details.rpc.cpeRequest.commandKey;
+          details.cpeRequestCommandKey = rpc.cpeRequest["commandKey"];
       }
-    } else if (details.rpc.cpeFault) {
+    } else if (rpc.cpeFault) {
-      details.acsRequestId = details.rpc.id;
+      details.acsRequestId = rpc.id;
-      details.cpeFaultCode = details.rpc.cpeFault.detail.faultCode;
+      details.cpeFaultCode = rpc.cpeFault.detail.faultCode;
-      details.cpeFaultString = details.rpc.cpeFault.detail.faultString;
+      details.cpeFaultString = rpc.cpeFault.detail.faultString;
     }
     delete details.rpc;
   }
 
   if (details.fault) {
-    details.faultCode = details.fault.code;
+    const fault = details.fault as Fault;
-    details.faultMessage = details.fault.message;
+    details.faultCode = fault.code;
+    details.faultMessage = fault.message;
     delete details.fault;
   }
 
   // For genieacs-ui
   if (details.context) {
-    details.remoteAddress = details.context.request.ip;
+    details.remoteAddress = getRequestOrigin(
-    if (details.context.state.user)
+      details.context["req"]
-      details.user = details.context.state.user.username;
+    ).remoteAddress;
+    if (details.context["state"].user)
+      details.user = details.context["state"].user.username;
     delete details.context;
   }
 
@@ -210,7 +233,10 @@ export function flatten(details): {} {
   return details;
 }
 
-function formatJson(details, systemd): string {
+function formatJson(
+  details: Record<string, unknown>,
+  systemd: boolean
+): string {
   if (systemd) {
     let severity = "";
     if (details.severity === "info") severity = "<6>";
@@ -223,14 +249,17 @@ function formatJson(details, systemd): string {
   return `${JSON.stringify(flatten(details))}${os.EOL}`;
 }
 
-function formatSimple(details, systemd): string {
+function formatSimple(
+  details: Record<string, unknown>,
+  systemd: boolean
+): string {
   const skip = {
     user: true,
     remoteAddress: true,
     severity: true,
     timestamp: true,
     message: true,
-    deviceId: !!details.sessionContext
+    deviceId: !!details.sessionContext,
   };
 
   flatten(details);
@@ -263,12 +292,14 @@ function formatSimple(details, systemd): string {
     return `${severity}${remote}${details.message}${meta}${os.EOL}`;
   }
 
-  return `${details.timestamp} [${details.severity.toUpperCase()}] ${remote}${
+  return `${
+    details.timestamp
+  } [${(details.severity as string).toUpperCase()}] ${remote}${
     details.message
   }${meta}${os.EOL}`;
 }
 
-function log(details): void {
+function log(details: Record<string, unknown>): void {
   details.timestamp = new Date().toISOString();
   if (LOG_FORMAT === "json") {
     details = Object.assign({}, defaultMeta, details);
@@ -278,22 +309,22 @@ function log(details): void {
   }
 }
 
-export function info(details): void {
+export function info(details: Record<string, unknown>): void {
   details.severity = "info";
   log(details);
 }
 
-export function warn(details): void {
+export function warn(details: Record<string, unknown>): void {
   details.severity = "warn";
   log(details);
 }
 
-export function error(details): void {
+export function error(details: Record<string, unknown>): void {
   details.severity = "error";
   log(details);
 }
 
-export function accessLog(details): void {
+export function accessLog(details: Record<string, unknown>): void {
   details.timestamp = new Date().toISOString();
   if (ACCESS_LOG_FORMAT === "json") {
     Object.assign(details, defaultMeta);
@@ -303,17 +334,17 @@ export function accessLog(details): void {
   }
 }
 
-export function accessInfo(details): void {
+export function accessInfo(details: Record<string, unknown>): void {
   details.severity = "info";
   accessLog(details);
 }
 
-export function accessWarn(details): void {
+export function accessWarn(details: Record<string, unknown>): void {
   details.severity = "warn";
   accessLog(details);
 }
 
-export function accessError(details): void {
+export function accessError(details: Record<string, unknown>): void {
   details.severity = "error";
   accessLog(details);
 }

lib/mongodb-functions.ts

@@ -18,14 +18,15 @@
  */
 
 import { ObjectID } from "mongodb";
-import { map, parse, stringify } from "./common/expression-parser";
+import { map, parse, stringify, parseList } from "./common/expression-parser";
-import { likePatternToRegExp, evaluate } from "./common/expression";
+import { likePatternToRegExp } from "./common/expression";
 import { Expression, Fault, Task } from "./types";
+import { decodeTag, encodeTag } from "./common";
 
 const isArray = Array.isArray;
 
-export function processDeviceFilter(filter): Expression {
+export function processDeviceFilter(filter: Expression): Expression {
-  return map(filter, exp => {
+  return map(filter, (exp) => {
     if (!isArray(exp)) return exp;
 
     if (exp[0] === "PARAM") {
@@ -34,8 +35,6 @@ export function processDeviceFilter(filter): Expression {
       else if (p === "DeviceID") return ["PARAM", "_deviceId"];
       else if (p.startsWith("DeviceID."))
         return ["PARAM", "_deviceId._" + p.slice(9)];
-      else if (p === "Tags") return ["PARAM", "_tags"];
-      else if (p === "tag") return ["PARAM", "_tags"];
       else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
       else if (p === "Events.Registered") return ["PARAM", "_registered"];
       else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
@@ -45,15 +44,26 @@ export function processDeviceFilter(filter): Expression {
       exp[1][0] === "PARAM" &&
       exp[1][1].startsWith("Tags.")
     ) {
-      const t = exp[1][1].slice(5);
+      const t = decodeTag(exp[1][1].slice(5));
-      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
+      if (exp[0] === "IS NULL") return ["_tags", t, false];
-      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
-      else if (exp[0] === "=" && exp[2] === true)
+      else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
-        return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
     } else if (
-      ["=", "<>", ">", ">=", "<", "<=", "LIKE", "NOT LIKE"].includes(exp[0])
+      [
+        "=",
+        "<>",
+        ">",
+        ">=",
+        "<",
+        "<=",
+        "LIKE",
+        "NOT LIKE",
+        "IS NULL",
+        "IS NOT NULL",
+      ].includes(exp[0])
     ) {
-      let e = map(exp, ee => {
+      let e = map(exp, (ee) => {
         if (
           isArray(ee) &&
           ee[0] === "PARAM" &&
@@ -75,8 +85,8 @@ export function processDeviceFilter(filter): Expression {
   });
 }
 
-export function processTasksFilter(filter): Expression {
+export function processTasksFilter(filter: Expression): Expression {
-  return map(filter, exp => {
+  return map(filter, (exp) => {
     if (!isArray(exp)) return exp;
     if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
       const e = exp.slice();
@@ -91,8 +101,8 @@ export function processTasksFilter(filter): Expression {
   });
 }
 
-export function processFaultsFilter(filter): Expression {
+export function processFaultsFilter(filter: Expression): Expression {
-  return map(filter, exp => {
+  return map(filter, (exp) => {
     if (!isArray(exp)) return exp;
     if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
       const e = exp.slice();
@@ -105,140 +115,137 @@ export function processFaultsFilter(filter): Expression {
   });
 }
 
-export function filterToMongoQuery(exp: Expression): {} {
+export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
-  const ops = {
+  function recursive(filter, negate): Record<string, unknown> {
-    OR: 0,
-    AND: 0,
-    NOT: 0,
-    "=": 1,
-    "<>": 1,
-    ">": 1,
-    ">=": 1,
-    "<": 1,
-    "<=": 1,
-    LIKE: 1,
-    "NOT LIKE": 1,
-    "IS NULL": 1,
-    "IS NOT NULL": 1
-  };
-
-  function recursive(filter, negate, res = {}): {} {
     const op = filter[0];
 
-    if (ops[op] === 0) {
+    if (op === "AND") {
-      for (let i = 1; i < filter.length; ++i) {
+      filter = filter.filter((f) => f !== true);
-        if (!isArray(filter[i]) || ops[filter[i][0]] == null)
+      if (filter.length === 2) return recursive(filter[1], negate);
-          throw new Error(`Invalid expression in ${op} clause`);
     }
 
-      if ((!negate && op === "AND") || (negate && op === "OR")) {
+    if (op === "OR") {
-        res["$and"] = res["$and"] || [];
+      filter = filter.filter((f) => f !== false);
-        for (let i = 1; i < filter.length; ++i)
+      if (filter.length === 2) return recursive(filter[1], negate);
-          res["$and"].push(recursive(filter[i], negate));
-      } else if ((!negate && op === "OR") || (negate && op === "AND")) {
-        res["$or"] = res["$or"] || [];
-
-        for (let i = 1; i < filter.length; ++i)
-          res["$or"].push(recursive(filter[i], negate));
-      } else if (op === "NOT") {
-        recursive(filter[1], !negate, res);
-      }
-    } else if (ops[op] === 1) {
-      if (isArray(filter[2])) {
-        throw new Error(`Right operand of ${op} clause is not a primitive`);
-      } else if (
-        ["LIKE", "NOT LIKE"].includes(op) &&
-        isArray(filter[1]) &&
-        filter[1][0] === "FUNC" &&
-        ["UPPER", "LOWER"].includes(filter[1][1])
-      ) {
-        if (
-          (filter[1][1] === "UPPER" && filter[2] !== filter[2].toUpperCase()) ||
-          (filter[1][1] === "LOWER" && filter[2] !== filter[2].toLowerCase())
-        ) {
-          throw new Error(
-            `Cannot compare ${
-              filter[1][1]
-            }() against non-${filter[1][1].toLowerCase()}case pattern`
-          );
-        }
-      } else if (!isArray(filter[1]) || filter[1][0] !== "PARAM") {
-        throw new Error(`Left operand of ${op} clause is not a parameter`);
     }
 
-      if (op === "=") {
+    if ((!negate && op === "AND") || (negate && op === "OR"))
-        const param = filter[1][1];
+      return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
-        const p = (res[param] = res[param] || {});
+
-        if (negate) p["$ne"] = filter[2];
+    if ((!negate && op === "OR") || (negate && op === "AND"))
-        else p["$eq"] = filter[2];
+      return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
-      } else if (op === "<>") {
+
-        const param = filter[1][1];
+    if (op === "NOT") return recursive(filter[1], !negate);
-        let p = (res[param] = res[param] || {});
+
-        if (negate) p = p["$not"] = p["$not"] || {};
+    if (isArray(filter[2]))
-        p["$ne"] = filter[2];
+      throw new Error(`Invalid RHS operand of ${op} clause`);
-        if (param !== "_tags") p["$exists"] = true;
+
-      } else if (op === ">") {
+    if (op === "LIKE" || op === "NOT LIKE") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gt"] = filter[2];
-      } else if (op === ">=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gte"] = filter[2];
-      } else if (op === "<") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lt"] = filter[2];
-      } else if (op === "<=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lte"] = filter[2];
-      } else if (op === "IS NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: negate };
-      } else if (op === "IS NOT NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: !negate };
-      } else {
       if (op === "NOT LIKE") negate = !negate;
       let param;
       let flags;
+      if (!Array.isArray(filter[1]))
+        throw new Error(`Invalid LHS operand of ${op} clause`);
       if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
+        if (filter[2] !== filter[2].toUpperCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
         param = filter[1][2][1];
         flags = "i";
       } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
+        if (filter[2] !== filter[2].toLowerCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
         param = filter[1][2][1];
         flags = "i";
-        } else {
+      } else if (filter[1][0] === "PARAM") {
         param = filter[1][1];
         flags = "";
+      } else {
+        throw new Error(`Invalid LHS operand of ${op} clause`);
       }
       const r = likePatternToRegExp(filter[2], filter[3], flags);
-        if (negate) res[param] = { $not: r };
+      if (negate) return { [param]: { $nin: [r, null] } };
-        else res[param] = r;
+      else return { [param]: r };
     }
-    } else {
+
+    if (op === "_tags") {
+      let t = filter[2];
+      if (negate) t = !t;
+      if (t) return { _tags: filter[1] };
+      else return { _tags: { $ne: filter[1] } };
+    }
+
+    if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
+      throw new Error(`Invalid LHS operand of ${op} clause`);
+
+    if (op === "IS NULL") {
+      const p = filter[1][1];
+      return { [p]: null };
+    }
+
+    if (op === "IS NOT NULL") {
+      const p = filter[1][1];
+      return { [p]: { $ne: null } };
+    }
+
+    if (isArray(filter[2]) || filter[2] == null)
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (!negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === "<>") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === ">") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lte: v } };
+      else return { [p]: { $gt: v } };
+    }
+
+    if (op === ">=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lt: v } };
+      else return { [p]: { $gte: v } };
+    }
+
+    if (op === "<") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gte: v } };
+      else return { [p]: { $lt: v } };
+    }
+
+    if (op === "<=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gt: v } };
+      else return { [p]: { $lte: v } };
+    }
+
     throw new Error(`Unrecognized operator ${op}`);
   }
 
-    return res;
+  if (!isArray(exp)) {
-  }
+    if (exp === true) return {};
-
-  const _exp = evaluate(exp, null, Date.now());
-
-  if (!isArray(_exp)) {
-    if (_exp === true) return {};
     throw new Error("Primitives are not valid queries");
   }
 
-  return recursive(_exp, false);
+  return recursive(exp, false);
 }
 
-export function processDeviceProjection(projection: {}): {} {
+export function processDeviceProjection(
+  projection: Record<string, 1>
+): Record<string, 1> {
   if (!projection) return projection;
   const p = {};
   for (const [k, v] of Object.entries(projection)) {
@@ -264,7 +271,9 @@ export function processDeviceProjection(projection: {}): {} {
   return p;
 }
 
-export function processDeviceSort(sort: {}): {} {
+export function processDeviceSort(
+  sort: Record<string, number>
+): Record<string, number> {
   if (!sort) return sort;
   const s = {};
   for (const [k, v] of Object.entries(sort)) {
@@ -280,70 +289,81 @@ export function processDeviceSort(sort: {}): {} {
   return s;
 }
 
-function parseDate(d): number | string {
+function parseDate(d: Date): number | string {
   const n = +d;
   return isNaN(n) ? "" + d : n;
 }
 
-interface FlatDevice {
+interface FlatAttributes {
-  [param: string]: {
+  object?: boolean;
-    object: boolean;
+  objectTimestamp?: number;
-    objectTimestamp: number;
+  writable?: boolean;
-    writable: boolean;
+  writableTimestamp?: number;
-    writableTimestamp: number;
+  value?: [string | number | boolean, string];
-    value: [string | number | boolean, string];
+  valueTimestamp?: number;
-    valueTimestamp: number;
+  notification?: number;
-  };
+  notificationTimestamp?: number;
+  accessList?: string[];
+  accessListTimestamp?: number;
 }
 
-export function flattenDevice(device): FlatDevice {
+interface FlatDevice {
-  function recursive(input, root, output, timestamp): void {
+  [param: string]: FlatAttributes;
+}
+
+export function flattenDevice(device: Record<string, unknown>): FlatDevice {
+  function recursive(
+    input,
+    root: string,
+    output: FlatDevice,
+    timestamp: number
+  ): void {
     for (const [name, tree] of Object.entries(input)) {
       if (!root) {
         if (name === "_lastInform") {
           output["Events.Inform"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
+            value: [parseDate(tree as Date), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_registered") {
           output["Events.Registered"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
+            value: [parseDate(tree as Date), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_lastBoot") {
           output["Events.1_BOOT"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
+            value: [parseDate(tree as Date), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_lastBootstrap") {
           output["Events.0_BOOTSTRAP"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
+            value: [parseDate(tree as Date), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_id") {
           output["DeviceID.ID"] = {
-            value: [tree, "xsd:string"],
+            value: [tree as string, "xsd:string"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_deviceId") {
           output["DeviceID.Manufacturer"] = {
@@ -352,7 +372,7 @@ export function flattenDevice(device): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
           output["DeviceID.OUI"] = {
             value: [tree["_OUI"], "xsd:string"],
@@ -360,7 +380,7 @@ export function flattenDevice(device): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
           output["DeviceID.ProductClass"] = {
             value: [tree["_ProductClass"], "xsd:string"],
@@ -368,7 +388,7 @@ export function flattenDevice(device): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
           output["DeviceID.SerialNumber"] = {
             value: [tree["_SerialNumber"], "xsd:string"],
@@ -376,24 +396,24 @@ export function flattenDevice(device): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
         } else if (name === "_tags") {
           output["Tags"] = {
-            writable: true,
+            writable: false,
             writableTimestamp: timestamp,
             object: true,
-            objectTimestamp: timestamp
+            objectTimestamp: timestamp,
           };
 
           for (const t of tree as string[]) {
-            output[`Tags.${t}`] = {
+            output[`Tags.${encodeTag(t)}`] = {
               value: [true, "xsd:boolean"],
               valueTimestamp: timestamp,
-              writable: false,
+              writable: true,
               writableTimestamp: timestamp,
               object: false,
-              objectTimestamp: timestamp
+              objectTimestamp: timestamp,
             };
           }
         }
@@ -407,55 +427,67 @@ export function flattenDevice(device): FlatDevice {
       else if (+input["_timestamp"] > timestamp)
         childrenTimestamp = +input["_timestamp"];
 
-      const attrs = {};
+      const attrs: FlatAttributes = {};
       if (tree["_value"] != null) {
-        attrs["value"] = [
+        attrs.value = [
           tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
-          tree["_type"]
+          tree["_type"],
         ];
-        attrs["valueTimestamp"] = +(tree["_timestamp"] || childrenTimestamp);
+        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
-        attrs["object"] = false;
+        attrs.object = false;
-        attrs["objectTimestamp"] = childrenTimestamp;
+        attrs.objectTimestamp = childrenTimestamp;
       } else if (tree["_object"] != null) {
-        attrs["object"] = tree["_object"];
+        attrs.object = tree["_object"];
-        attrs["objectTimestamp"] = childrenTimestamp;
+        attrs.objectTimestamp = childrenTimestamp;
       }
 
       if (tree["_writable"] != null) {
-        attrs["writable"] = tree["_writable"];
+        attrs.writable = tree["_writable"];
-        attrs["writableTimestamp"] = childrenTimestamp;
+        attrs.writableTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_notification"] != null) {
+        attrs.notification = tree["_notification"];
+        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      if (tree["_accessList"] != null) {
+        attrs.accessList = tree["_accessList"];
+        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
       }
 
       const r = root ? `${root}.${name}` : name;
       output[r] = attrs;
 
-      if (attrs["object"] || tree["object"] == null)
+      if (attrs.object || tree["object"] == null)
         recursive(tree, r, output, childrenTimestamp);
     }
   }
 
-  const newDevice = {};
+  const newDevice: FlatDevice = {};
-  const timestamp = new Date(device["_lastInform"] || 1).getTime();
+  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
   recursive(device, "", newDevice, timestamp);
   return newDevice;
 }
 
-export function flattenFault(fault): Fault {
+export function flattenFault(fault: unknown): Fault {
-  const f = Object.assign({}, fault);
+  const f = Object.assign({}, fault) as Fault;
   if (f.timestamp) f.timestamp = +f.timestamp;
-  if (f.expiry) f.expiry = +f.expiry;
+  if (f["expiry"]) f["expiry"] = +f["expiry"];
-  return f;
+  return f as Fault;
 }
 
-export function flattenTask(task): Task {
+export function flattenTask(task: unknown): Task {
-  const t = Object.assign({}, task);
+  const t = Object.assign({}, task) as Task;
   t._id = "" + t._id;
-  if (t.timestamp) t.timestamp = +t.timestamp;
+  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
   if (t.expiry) t.expiry = +t.expiry;
   return t;
 }
 
-export function mongoQueryToFilter(query): Expression {
+export function convertOldPrecondition(
+  query: Record<string, unknown>
+): Expression {
   function recursive(_query): Expression {
     const expressions: Expression[] = [];
     for (const [k, v] of Object.entries(_query)) {
@@ -474,14 +506,27 @@ export function mongoQueryToFilter(query): Expression {
       } else if (k === "_tags") {
         if (typeof v === "object") {
           if (isArray(v)) throw new Error(`Invalid type`);
-
+          const conjs: Expression[] = [];
-          if (v.hasOwnProperty("$ne"))
+          for (const [op, val] of Object.entries(v)) {
-            expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]);
+            if (op === "$ne") {
-          else if (v.hasOwnProperty("$eq"))
+              if (typeof v["$ne"] !== "string")
-            expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v["$eq"]}`]]);
+                throw new Error("Only string values are allowed for _tags");
-          else throw new Error(`Invalid tag query`);
+              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
             } else {
-          expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
+              throw new Error(`Invalid tag query`);
+            }
+          }
+          if (conjs.length === 1) expressions.push(conjs[0]);
+          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
+        } else {
+          expressions.push([
+            "IS NOT NULL",
+            ["PARAM", `Tags.${encodeTag(v as string)}`],
+          ]);
         }
       } else if (k.startsWith("Tags.")) {
         let exists: boolean;
@@ -497,41 +542,41 @@ export function mongoQueryToFilter(query): Expression {
 
         const exps: Expression[] = [];
         for (const [kk, vv] of Object.entries(v)) {
-          let op;
+          if (kk === "$eq") {
-          switch (kk) {
+            exps.push(["=", ["PARAM", k], vv]);
-            case "$eq":
+          } else if (kk === "$ne") {
-              op = "=";
+            exps.push([
-              break;
+              "OR",
-            case "$ne":
+              ["<>", ["PARAM", k], vv],
-              op = "<>";
+              ["IS NULL", ["PARAM", k]],
-              break;
+            ]);
-            case "$lt":
+          } else if (kk === "$lt") {
-              op = "<";
+            exps.push(["<", ["PARAM", k], vv]);
-              break;
+          } else if (kk === "$lte") {
-            case "$lte":
+            exps.push(["<=", ["PARAM", k], vv]);
-              op = "<=";
+          } else if (kk === "$gt") {
-              break;
+            exps.push([">", ["PARAM", k], vv]);
-            case "$gt":
+          } else if (kk === "$gte") {
-              op = ">";
+            exps.push([">=", ["PARAM", k], vv]);
-              break;
+          } else {
-            case "$gte":
-              op = ">=";
-              break;
-            default:
             throw new Error(`Operator ${kk} not supported`);
           }
-          exps.push([op, ["PARAM", k], vv]);
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
         }
-        if (exps.length === 1) return exps[0];
+        if (exps.length === 1) {
+          expressions.push(exps[0]);
+        } else if (exps.length > 1) {
           const and: Expression = ["AND"];
           expressions.push(and.concat(exps));
+        }
       } else {
         expressions.push(["=", ["PARAM", k], v]);
       }
     }
     if (expressions.length === 1) return expressions[0];
-    const and: Expression = ["AND"];
+    if (expressions.length === 0) return true;
-    return and.concat(expressions);
+    return ["AND", ...expressions];
   }
 
   // empty filter
@@ -540,15 +585,21 @@ export function mongoQueryToFilter(query): Expression {
   return recursive(query);
 }
 
-export function flattenPreset(preset): {} {
+export function flattenPreset(
+  preset: Record<string, unknown>
+): Record<string, unknown> {
   const p = Object.assign({}, preset);
   if (p.precondition) {
     try {
       // Try parse to check expression validity
-      parse(p.precondition);
+      parse(p.precondition as string);
     } catch (error) {
-      p.precondition = mongoQueryToFilter(JSON.parse(p.precondition));
+      p.precondition = convertOldPrecondition(
-      p.precondition = p.precondition.length ? stringify(p.precondition) : "";
+        JSON.parse(p.precondition as string)
+      );
+      p.precondition = (p.precondition as string).length
+        ? stringify(p.precondition as Expression)
+        : "";
     }
   }
 
@@ -560,14 +611,14 @@ export function flattenPreset(preset): {} {
 
   const provision = p.configurations[0];
   if (
-    p.configurations.length === 1 &&
+    (p.configurations as any[]).length === 1 &&
     provision.type === "provision" &&
     provision.name &&
     provision.name.length
   ) {
     p.provision = provision.name;
     p.provisionArgs = provision.args
-      ? JSON.stringify(provision.args).slice(1, -1)
+      ? provision.args.map((a) => stringify(a)).join(", ")
       : "";
   }
 
@@ -575,30 +626,34 @@ export function flattenPreset(preset): {} {
   return p;
 }
 
-export function flattenFile(file): {} {
+export function flattenFile(
+  file: Record<string, unknown>
+): Record<string, unknown> {
   const f = {};
   f["_id"] = file["_id"];
   if (file.metadata) {
-    f["metadata.fileType"] = file.metadata.fileType || "";
+    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
-    f["metadata.oui"] = file.metadata.oui || "";
+    f["metadata.oui"] = file["metadata"]["oui"] || "";
-    f["metadata.productClass"] = file.metadata.productClass || "";
+    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
-    f["metadata.version"] = file.metadata.version || "";
+    f["metadata.version"] = file["metadata"]["version"] || "";
   }
   return f;
 }
 
-export function preProcessPreset(data): {} {
+export function preProcessPreset(
+  data: Record<string, unknown>
+): Record<string, unknown> {
   const preset = Object.assign({}, data);
 
   if (!preset.precondition) preset.precondition = "";
   // Try parse to check expression validity
-  parse(preset.precondition);
+  parse(preset.precondition as string);
 
-  preset.weight = parseInt(preset.weight) || 0;
+  preset.weight = parseInt(preset.weight as string) || 0;
 
   const events = {};
   if (preset.events) {
-    for (let e of preset.events.split(",")) {
+    for (let e of (preset.events as string).split(",")) {
       let v = true;
       e = e.trim();
       if (e.startsWith("-")) {
@@ -616,11 +671,11 @@ export function preProcessPreset(data): {} {
   const configuration = {
     type: "provision",
     name: preset.provision,
-    args: null
+    args: null,
   };
 
   if (preset.provisionArgs)
-    configuration.args = JSON.parse(`[${preset.provisionArgs}]`);
+    configuration.args = parseList(preset.provisionArgs as string);
 
   delete preset.provision;
   delete preset.provisionArgs;

lib/nbi.ts

@@ -21,13 +21,14 @@ import * as url from "url";
 import * as mongodb from "mongodb";
 import * as querystring from "querystring";
 import * as vm from "vm";
-import { exec } from "child_process";
 import * as config from "./config";
 import * as db from "./db";
 import * as query from "./query";
 import * as apiFunctions from "./api-functions";
+import { IncomingMessage, ServerResponse } from "http";
 import * as cache from "./cache";
 import { version as VERSION } from "../package.json";
+import { ping } from "./ping";
 
 const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
 const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
@@ -42,7 +43,7 @@ const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
 const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
 
-function throwError(err, httpResponse?): never {
+function throwError(err: Error, httpResponse?: ServerResponse): never {
   if (httpResponse) {
     httpResponse.writeHead(500, { Connection: "close" });
     httpResponse.end(`${err.name}: ${err.message}`);
@@ -50,7 +51,10 @@ function throwError(err, httpResponse?): never {
   throw err;
 }
 
-export function listener(request, response): void {
+export function listener(
+  request: IncomingMessage,
+  response: ServerResponse
+): void {
   const chunks = [];
   let bytes = 0;
   response.setHeader("GenieACS-Version", VERSION);
@@ -85,7 +89,7 @@ export function listener(request, response): void {
           { _id: presetName },
           preset,
           { upsert: true },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             cache
@@ -94,7 +98,7 @@ export function listener(request, response): void {
                 response.writeHead(200);
                 response.end();
               })
-              .catch(err => {
+              .catch((err) => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -102,7 +106,7 @@ export function listener(request, response): void {
           }
         );
       } else if (request.method === "DELETE") {
-        db.presetsCollection.deleteOne({ _id: presetName }, err => {
+        db.presetsCollection.deleteOne({ _id: presetName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -111,7 +115,7 @@ export function listener(request, response): void {
               response.writeHead(200);
               response.end();
             })
-            .catch(err => {
+            .catch((err) => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -132,7 +136,7 @@ export function listener(request, response): void {
           { _id: objectName },
           object,
           { upsert: true },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             cache
@@ -141,7 +145,7 @@ export function listener(request, response): void {
                 response.writeHead(200);
                 response.end();
               })
-              .catch(err => {
+              .catch((err) => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -149,7 +153,7 @@ export function listener(request, response): void {
           }
         );
       } else if (request.method === "DELETE") {
-        db.objectsCollection.deleteOne({ _id: objectName }, err => {
+        db.objectsCollection.deleteOne({ _id: objectName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -158,7 +162,7 @@ export function listener(request, response): void {
               response.writeHead(200);
               response.end();
             })
-            .catch(err => {
+            .catch((err) => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -175,7 +179,7 @@ export function listener(request, response): void {
       if (request.method === "PUT") {
         const object = {
           _id: provisionName,
-          script: body.toString()
+          script: body.toString(),
         };
 
         try {
@@ -190,7 +194,7 @@ export function listener(request, response): void {
           { _id: provisionName },
           object,
           { upsert: true },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             cache
@@ -199,7 +203,7 @@ export function listener(request, response): void {
                 response.writeHead(200);
                 response.end();
               })
-              .catch(err => {
+              .catch((err) => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -207,7 +211,7 @@ export function listener(request, response): void {
           }
         );
       } else if (request.method === "DELETE") {
-        db.provisionsCollection.deleteOne({ _id: provisionName }, err => {
+        db.provisionsCollection.deleteOne({ _id: provisionName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -216,7 +220,7 @@ export function listener(request, response): void {
               response.writeHead(200);
               response.end();
             })
-            .catch(err => {
+            .catch((err) => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -233,7 +237,7 @@ export function listener(request, response): void {
       if (request.method === "PUT") {
         const object = {
           _id: virtualParameterName,
-          script: body.toString()
+          script: body.toString(),
         };
 
         try {
@@ -248,7 +252,7 @@ export function listener(request, response): void {
           { _id: virtualParameterName },
           object,
           { upsert: true },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             cache
@@ -257,7 +261,7 @@ export function listener(request, response): void {
                 response.writeHead(200);
                 response.end();
               })
-              .catch(err => {
+              .catch((err) => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -267,7 +271,7 @@ export function listener(request, response): void {
       } else if (request.method === "DELETE") {
         db.virtualParametersCollection.deleteOne(
           { _id: virtualParameterName },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             cache
@@ -276,7 +280,7 @@ export function listener(request, response): void {
                 response.writeHead(200);
                 response.end();
               })
-              .catch(err => {
+              .catch((err) => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -295,7 +299,7 @@ export function listener(request, response): void {
         db.devicesCollection.updateOne(
           { _id: deviceId },
           { $addToSet: { _tags: tag } },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
             response.writeHead(200);
             response.end();
@@ -305,7 +309,7 @@ export function listener(request, response): void {
         db.devicesCollection.updateOne(
           { _id: deviceId },
           { $pull: { _tags: tag } },
-          err => {
+          (err) => {
             if (err) return void throwError(err, response);
 
             response.writeHead(200);
@@ -323,12 +327,12 @@ export function listener(request, response): void {
         );
         const deviceId = faultId.split(":", 1)[0];
         const channel = faultId.slice(deviceId.length + 1);
-        db.faultsCollection.deleteOne({ _id: faultId }, err => {
+        db.faultsCollection.deleteOne({ _id: faultId }, (err) => {
           if (err) return void throwError(err, response);
 
           if (channel.startsWith("task_")) {
             const objId = new mongodb.ObjectID(channel.slice(5));
-            return void db.tasksCollection.deleteOne({ _id: objId }, err => {
+            return void db.tasksCollection.deleteOne({ _id: objId }, (err) => {
               if (err) return void throwError(err, response);
 
               cache
@@ -337,7 +341,7 @@ export function listener(request, response): void {
                   response.writeHead(200);
                   response.end();
                 })
-                .catch(err => {
+                .catch((err) => {
                   setTimeout(() => {
                     throwError(err, response);
                   });
@@ -351,7 +355,7 @@ export function listener(request, response): void {
               response.writeHead(200);
               response.end();
             })
-            .catch(err => {
+            .catch((err) => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -382,17 +386,20 @@ export function listener(request, response): void {
                         const taskTimeout =
                           (urlParts.query.timeout &&
                             parseInt(urlParts.query.timeout as string)) ||
-                          config.get("DEVICE_ONLINE_THRESHOLD", deviceId);
+                          (config.get(
+                            "DEVICE_ONLINE_THRESHOLD",
+                            deviceId
+                          ) as number);
 
                         apiFunctions
                           .watchTask(deviceId, task._id, taskTimeout)
-                          .then(status => {
+                          .then((status) => {
                             if (status === "timeout") {
                               response.writeHead(
                                 202,
                                 "Task queued but not processed",
                                 {
-                                  "Content-Type": "application/json"
+                                  "Content-Type": "application/json",
                                 }
                               );
                               response.end(JSON.stringify(task));
@@ -404,44 +411,44 @@ export function listener(request, response): void {
                                     return void throwError(err, response);
 
                                   response.writeHead(202, "Task faulted", {
-                                    "Content-Type": "application/json"
+                                    "Content-Type": "application/json",
                                   });
                                   response.end(JSON.stringify(task2));
                                 }
                               );
                             } else {
                               response.writeHead(200, {
-                                "Content-Type": "application/json"
+                                "Content-Type": "application/json",
                               });
                               response.end(JSON.stringify(task));
                             }
                           })
-                          .catch(err => {
+                          .catch((err) => {
                             setTimeout(() => {
                               throwError(err, response);
                             });
                           });
                       })
-                      .catch(err => {
+                      .catch((err) => {
                         response.writeHead(202, err.message, {
-                          "Content-Type": "application/json"
+                          "Content-Type": "application/json",
                         });
                         response.end(JSON.stringify(task));
                       });
                   } else {
                     response.writeHead(202, {
-                      "Content-Type": "application/json"
+                      "Content-Type": "application/json",
                     });
                     response.end(JSON.stringify(task));
                   }
                 })
-                .catch(err => {
+                .catch((err) => {
                   setTimeout(() => {
                     throwError(err, response);
                   });
                 });
             })
-            .catch(err => {
+            .catch((err) => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -454,7 +461,7 @@ export function listener(request, response): void {
               response.writeHead(200);
               response.end();
             })
-            .catch(err => {
+            .catch((err) => {
               response.writeHead(504);
               response.end(`${err.name}: ${err.message}`);
             });
@@ -487,12 +494,12 @@ export function listener(request, response): void {
               const deviceId = task.device;
               db.tasksCollection.deleteOne(
                 { _id: new mongodb.ObjectID(taskId) },
-                err => {
+                (err) => {
                   if (err) return void throwError(err, response);
 
                   db.faultsCollection.deleteOne(
                     { _id: `${deviceId}:task_${taskId}` },
-                    err => {
+                    (err) => {
                       if (err) return void throwError(err, response);
 
                       cache
@@ -501,7 +508,7 @@ export function listener(request, response): void {
                           response.writeHead(200);
                           response.end();
                         })
-                        .catch(err => {
+                        .catch((err) => {
                           setTimeout(() => {
                             throwError(err, response);
                           });
@@ -527,7 +534,7 @@ export function listener(request, response): void {
               const deviceId = task.device;
               db.faultsCollection.deleteOne(
                 { _id: `${deviceId}:task_${taskId}` },
-                err => {
+                (err) => {
                   if (err) return void throwError(err, response);
 
                   cache
@@ -536,7 +543,7 @@ export function listener(request, response): void {
                       response.writeHead(200);
                       response.end();
                     })
-                    .catch(err => {
+                    .catch((err) => {
                       setTimeout(() => {
                         throwError(err, response);
                       });
@@ -562,14 +569,19 @@ export function listener(request, response): void {
           fileType: request.headers.filetype,
           oui: request.headers.oui,
           productClass: request.headers.productclass,
-          version: request.headers.version
+          version: request.headers.version,
         };
         const bucket = new mongodb.GridFSBucket(db.client.db());
-        const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
+        bucket.delete((filename as unknown) as mongodb.ObjectId, () => {
-          metadata: metadata
+          const uploadStream = bucket.openUploadStreamWithId(
-        });
+            filename,
+            filename,
+            {
+              metadata: metadata,
+            }
+          );
 
-        uploadStream.on("error", err => {
+          uploadStream.on("error", (err) => {
             throwError(err, response);
           });
 
@@ -577,10 +589,18 @@ export function listener(request, response): void {
             response.writeHead(201);
             response.end();
           });
+        });
       } else if (request.method === "DELETE") {
         const bucket = new mongodb.GridFSBucket(db.client.db());
-        bucket.delete((filename as unknown) as mongodb.ObjectId, err => {
+        bucket.delete((filename as unknown) as mongodb.ObjectId, (err) => {
-          if (err) return void throwError(err, response);
+          if (err) {
+            if (err.message.startsWith("FileNotFound")) {
+              response.writeHead(404);
+              response.end("404 Not Found");
+              return;
+            }
+            return void throwError(err, response);
+          }
 
           response.writeHead(200);
           response.end();
@@ -591,8 +611,13 @@ export function listener(request, response): void {
       }
     } else if (PING_REGEX.test(urlParts.pathname)) {
       const host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1]);
-      exec(`ping -w 1 -i 0.2 -c 3 ${host}`, (err, stdout) => {
+      ping(host, (err, res, stdout) => {
         if (err) {
+          if (!res) {
+            response.writeHead(500, { Connection: "close" });
+            response.end(`${err.name}: ${err.message}`);
+            return;
+          }
           response.writeHead(404, { "Cache-Control": "no-cache" });
           response.end(`${err.name}: ${err.message}`);
           return;
@@ -600,7 +625,7 @@ export function listener(request, response): void {
 
         response.writeHead(200, {
           "Content-Type": "text/plain",
-          "Cache-Control": "no-cache"
+          "Cache-Control": "no-cache",
         });
         response.end(stdout);
       });
@@ -620,7 +645,7 @@ export function listener(request, response): void {
           response.writeHead(200);
           response.end();
         })
-        .catch(err => {
+        .catch((err) => {
           setTimeout(() => {
             throwError(err, response);
           });
@@ -668,15 +693,15 @@ export function listener(request, response): void {
           break;
         case "tasks":
           q = query.sanitizeQueryTypes(q, {
-            _id: v => new mongodb.ObjectID(v),
+            _id: (v) => new mongodb.ObjectID(v as string),
-            timestamp: v => new Date(v),
+            timestamp: (v) => new Date(v as number),
-            retries: Number
+            retries: Number,
           });
           break;
         case "faults":
           q = query.sanitizeQueryTypes(q, {
-            timestamp: v => new Date(v),
+            timestamp: (v) => new Date(v as number),
-            retries: Number
+            retries: Number,
           });
       }
 
@@ -712,7 +737,7 @@ export function listener(request, response): void {
 
         response.writeHead(200, {
           "Content-Type": "application/json",
-          total: total
+          total: total,
         });
 
         if (request.method === "HEAD") {
@@ -723,11 +748,11 @@ export function listener(request, response): void {
         response.write("[\n");
         i = 0;
         cur.forEach(
-          item => {
+          (item) => {
             if (i++) response.write(",\n");
             response.write(JSON.stringify(item));
           },
-          err => {
+          (err) => {
             if (err) return void throwError(err);
             response.end("\n]");
           }

lib/ping.ts

@@ -0,0 +1,89 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { platform } from "os";
+import { exec } from "child_process";
+
+export interface PingResult {
+  packetsTransmitted: number;
+  packetsReceived: number;
+  packetLoss: number;
+  min: number;
+  avg: number;
+  max: number;
+  mdev: number;
+}
+
+export function ping(
+  host: string,
+  callback: (err: Error, res?: PingResult, stdout?: string) => void
+): void {
+  let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
+  switch (platform()) {
+    case "linux":
+      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
+      parseRegExp1 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+      parseRegExp2 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
+      break;
+
+    case "freebsd":
+      // Send a single packet because on FreeBSD only superuser can send
+      // packets that are only 200 ms apart.
+      cmd = `ping -t 1 -c 3 ${host}`;
+      parseRegExp1 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
+      parseRegExp2 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
+      break;
+
+    default:
+      return callback(new Error("Platform not supported"));
+  }
+
+  exec(cmd, (err, stdout) => {
+    let parsed: PingResult;
+    if (stdout) {
+      const m1 = stdout.match(parseRegExp1);
+      if (m1) {
+        parsed = {
+          packetsTransmitted: +m1[1],
+          packetsReceived: +m1[2],
+          packetLoss: +m1[3],
+          min: +m1[4],
+          avg: +m1[5],
+          max: +m1[6],
+          mdev: +m1[7],
+        };
+      } else {
+        const m2 = stdout.match(parseRegExp2);
+        if (m2) {
+          parsed = {
+            packetsTransmitted: +m2[1],
+            packetsReceived: +m2[2],
+            packetLoss: +m2[3],
+            min: null,
+            avg: null,
+            max: null,
+            mdev: null,
+          };
+        }
+      }
+    }
+
+    callback(err, parsed, stdout);
+  });
+}

lib/query.ts

@@ -17,7 +17,9 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import * as common from "./common";
+function isObject(obj: any): boolean {
+  return Object.prototype.toString.call(obj) === "[object Object]";
+}
 
 function stringToRegexp(input, flags?): RegExp | false {
   if (input.indexOf("*") === -1) return false;
@@ -34,8 +36,8 @@ function stringToRegexp(input, flags?): RegExp | false {
 }
 
 function normalize(input): any {
-  if (common.typeOf(input) === common.STRING_TYPE) {
+  if (typeof input === "string") {
-    const vals = [input];
+    const vals: any = [input];
     const m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input);
     if (m) vals.push({ $regex: new RegExp(m[1], m[2]) });
 
@@ -52,14 +54,14 @@ function normalize(input): any {
   return input;
 }
 
-export function expandValue(value): any[] {
+function expandValue(value: unknown): unknown[] {
-  if (common.typeOf(value) === common.ARRAY_TYPE) {
+  if (Array.isArray(value)) {
     let a = [];
     for (const j of value) a = a.concat(expandValue(j));
     return [a];
-  } else if (common.typeOf(value) !== common.OBJECT_TYPE) {
+  } else if (!isObject(value)) {
     const n = normalize(value);
-    if (common.typeOf(n) !== common.ARRAY_TYPE) return [n];
+    if (!Array.isArray(n)) return [n];
     else return n;
   }
 
@@ -103,12 +105,14 @@ function permute(param, val): any[] {
   return conditions;
 }
 
-export function expand(query): {} {
+export function expand(
+  query: Record<string, unknown>
+): Record<string, unknown> {
   const newQuery = {};
   for (const [k, v] of Object.entries(query)) {
     if (k[0] === "$") {
       // Operator
-      newQuery[k] = (v as any[]).map(e => expand(e));
+      newQuery[k] = (v as any[]).map((e) => expand(e));
     } else {
       const conditions = permute(k, v);
       if (conditions.length > 1) {
@@ -129,100 +133,16 @@ export function expand(query): {} {
   return newQuery;
 }
 
-function testExpressions(params, expressions, lop): boolean {
+export function sanitizeQueryTypes(
-  for (const f of expressions) {
+  query: Record<string, unknown>,
-    const res = test(params, f);
+  types: Record<string, (v: unknown) => unknown>
-    switch (lop) {
+): Record<string, unknown> {
-      case "$and":
-        if (!res) return false;
-        break;
-      case "$or":
-        if (res) return true;
-        break;
-      case "$nor":
-        if (res) return false;
-    }
-  }
-
-  switch (lop) {
-    case "$and":
-      return true;
-    case "$or":
-      return false;
-    case "$nor":
-      return true;
-    default:
-      throw new Error("Unknown logical operator");
-  }
-}
-
-export function test(params, query): boolean {
-  let res;
-  for (const [k, v] of Object.entries(query)) {
-    if (k.charAt(0) === "$") {
-      // Logical operator
-      res = testExpressions(params, v, k);
-    } else {
-      const value = params[k];
-      if (common.typeOf(v) !== common.OBJECT_TYPE) {
-        if (common.typeOf(value) === common.ARRAY_TYPE) {
-          // TODO comparing array to regex, array to array, and object to object
-          res = value.indexOf(v) !== -1;
-        } else {
-          if (common.typeOf(v) === common.REGEXP_TYPE)
-            res = (v as RegExp).test(value);
-          else res = v === value;
-        }
-      } else {
-        for (const [k2, v2] of Object.entries(v)) {
-          switch (k2) {
-            case "$ne":
-              if (common.typeOf(value) === common.ARRAY_TYPE)
-                res = value.indexOf(v2) === -1;
-              else res = value !== v2;
-              break;
-            case "$lt":
-              res = value < v2;
-              break;
-            case "$lte":
-              res = value <= v2;
-              break;
-            case "$gt":
-              res = value > v2;
-              break;
-            case "$gte":
-              res = value >= v2;
-              break;
-            case "$regex":
-              res = v2.test(value);
-              break;
-            case "$in":
-              throw new Error("Operator not supported");
-            case "$nin":
-              throw new Error("Operator not supported");
-            case "$all":
-              throw new Error("Operator not supported");
-            case "$exists":
-              throw new Error("Operator not supported");
-            default:
-              throw new Error("Operator not supported");
-          }
-        }
-      }
-    }
-
-    if (!res) return false;
-  }
-  return true;
-}
-
-export function sanitizeQueryTypes(query, types): {} {
   for (const [k, v] of Object.entries(query)) {
     if (k[0] === "$") {
       // Logical operator
       for (const vv of v as any[]) sanitizeQueryTypes(vv, types);
     } else if (k in types) {
-      if (common.typeOf(v) === common.OBJECT_TYPE) {
+      if (isObject(v)) {
         for (const [kk, vv] of Object.entries(v)) {
           switch (kk) {
             case "$in":

lib/sandbox.ts

@@ -109,6 +109,16 @@ class SandboxDate {
 
     return t;
   }
+
+  public static parse(dateString: string): number {
+    return Date.parse(dateString);
+  }
+
+  public static UTC(
+    ...args: [number, number?, number?, number?, number?, number?, number?]
+  ): number {
+    return Date.UTC(...args);
+  }
 }
 
 function random(): number {
@@ -117,7 +127,7 @@ function random(): number {
   return state.rng();
 }
 
-random.seed = function(s) {
+random.seed = function (s) {
   state.rng = seedrandom(s);
 };
 
@@ -125,7 +135,7 @@ class ParameterWrapper {
   public constructor(path: Path, attributes, unpacked?, unpackedRevision?) {
     for (const attrName of attributes) {
       Object.defineProperty(this, attrName, {
-        get: function() {
+        get: function () {
           if (state.uncommitted) commit();
 
           if (state.revision !== unpackedRevision) {
@@ -147,12 +157,12 @@ class ParameterWrapper {
           if (!attr) return UNDEFINED;
 
           return attr[1];
-        }
+        },
       });
     }
 
     Object.defineProperty(this, "path", {
-      get: function() {
+      get: function () {
         if (state.uncommitted) commit();
 
         if (state.revision !== unpackedRevision) {
@@ -167,11 +177,11 @@ class ParameterWrapper {
         if (!unpacked.length) return UNDEFINED;
 
         return unpacked[0].toString();
-      }
+      },
     });
 
     Object.defineProperty(this, "size", {
-      get: function() {
+      get: function () {
         if (state.uncommitted) commit();
 
         if (state.revision !== unpackedRevision) {
@@ -186,10 +196,10 @@ class ParameterWrapper {
         if (!unpacked.length) return UNDEFINED;
 
         return unpacked.length;
-      }
+      },
     });
 
-    this[Symbol.iterator] = function*() {
+    this[Symbol.iterator] = function* () {
       if (state.uncommitted) commit();
 
       if (state.revision !== unpackedRevision) {
@@ -225,7 +235,7 @@ function declare(
     pathSet: null,
     attrGet: null,
     attrSet: null,
-    defer: true
+    defer: true,
   };
 
   const attrs = new Set();
@@ -280,9 +290,9 @@ function commit(): void {
   }
 }
 
-function ext(): any {
+function ext(...args: unknown[]): any {
   ++state.extCounter;
-  const extCall = Array.from(arguments).map(String);
+  const extCall = args.map(String);
   const key = `${state.revision}: ${JSON.stringify(extCall)}`;
 
   if (key in state.sessionContext.extensionsCache)
@@ -292,11 +302,11 @@ function ext(): any {
   throw EXT;
 }
 
-function log(msg: string, meta: {}): void {
+function log(msg: string, meta: Record<string, unknown>): void {
   if (state.revision === state.maxRevision && state.extCounter >= 0) {
     const details = Object.assign({}, meta, {
       sessionContext: state.sessionContext,
-      message: `Script: ${msg}`
+      message: `Script: ${msg}`,
     });
 
     delete details["hostname"];
@@ -332,8 +342,8 @@ function errorToFault(err: Error): Fault {
     message: err.message,
     detail: {
       name: err.name,
-      message: err.message
+      message: err.message,
-    }
+    },
   };
 
   if (err.stack) {
@@ -355,7 +365,7 @@ function errorToFault(err: Error): Fault {
 
 export async function run(
   script: vm.Script,
-  globals: {},
+  globals: Record<string, unknown>,
   sessionContext: SessionContext,
   startRevision: number,
   maxRevision: number,
@@ -370,7 +380,7 @@ export async function run(
     extensions: {},
     clear: [],
     rng: null,
-    extCounter: extCounter
+    extCounter: extCounter,
   };
 
   for (const n of Object.keys(context)) delete context[n];
@@ -393,7 +403,7 @@ export async function run(
         clear: null,
         declare: null,
         done: false,
-        returnValue: null
+        returnValue: null,
       };
     }
   }
@@ -413,7 +423,7 @@ export async function run(
       clear: null,
       declare: null,
       done: false,
-      returnValue: null
+      returnValue: null,
     };
   }
 
@@ -433,6 +443,6 @@ export async function run(
     clear: _state.clear,
     declare: _state.declarations,
     done: status === 0,
-    returnValue: ret
+    returnValue: ret,
   };
 }

lib/scheduling.ts

@@ -21,10 +21,7 @@ import * as crypto from "crypto";
 import * as later from "later";
 
 function md532(str): number {
-  const digest = crypto
+  const digest = crypto.createHash("md5").update(str).digest();
-    .createHash("md5")
-    .update(str)
-    .digest();
   return (
     digest.readUInt32LE(0) ^
     digest.readUInt32LE(4) ^
@@ -33,29 +30,37 @@ function md532(str): number {
   );
 }
 
-export function variance(deviceId, vrnc): number {
+export function variance(deviceId: string, vrnc: number): number {
   return (md532(deviceId) >>> 0) % vrnc;
 }
 
-export function interval(timestamp, intrvl, offset = 0): number {
+export function interval(
+  timestamp: number,
+  intrvl: number,
+  offset = 0
+): number {
   return Math.trunc((timestamp + offset) / intrvl) * intrvl - offset;
 }
 
-export function parseCron(cronExp): any {
+export function parseCron(cronExp: string): any {
   const parts = cronExp.trim().split(/\s+/);
   if (parts.length === 5) parts.unshift("*");
 
   return later.schedule(later.parse.cron(parts.join(" "), true));
 }
 
-export function cron(timestamp, schedule, offset = 0): number[] {
+export function cron(
+  timestamp: number,
+  schedule: unknown,
+  offset = 0
+): number[] {
   // TODO later.js doesn't throw erorr if expression is invalid!
   const ret = [0, 0];
 
-  const prev = schedule.prev(1, new Date(timestamp + offset));
+  const prev = (schedule as any).prev(1, new Date(timestamp + offset));
   if (prev) ret[0] = prev.setMilliseconds(0) - offset;
 
-  const next = schedule.next(1, new Date(timestamp + offset + 1000));
+  const next = (schedule as any).next(1, new Date(timestamp + offset + 1000));
   if (next) ret[1] = next.setMilliseconds(0) - offset;
 
   return ret;

lib/server.ts

@@ -20,6 +20,7 @@
 import * as fs from "fs";
 import * as http from "http";
 import * as https from "https";
+import { Socket } from "net";
 import * as path from "path";
 import { ROOT_DIR } from "./config";
 
@@ -46,39 +47,48 @@ function closeServer(timeout, callback): void {
 
     const cb = callback;
     callback = null;
-    cb();
+    // Allow some time for connection close events to fire
+    setTimeout(cb, 50);
   });
 }
 
+interface ServerOptions {
+  port?: number;
+  host?: string;
+  ssl?: { key: string; cert: string };
+  timeout?: number;
+  keepAliveTimeout?: number;
+  onConnection?: (socket: Socket) => void;
+}
+
 export function start(
-  port,
+  options: ServerOptions,
-  networkInterface,
+  _listener: http.RequestListener
-  ssl,
-  _listener,
-  onConnection?,
-  keepAliveTimeout: number = 0
 ): void {
   listener = _listener;
 
-  if (ssl && ssl.key && ssl.cert) {
+  if (options.ssl) {
-    const options = {
+    const opts = {
-      key: ssl.key
+      key: options.ssl.key
         .split(":")
-        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
+        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
-      cert: ssl.cert
+      cert: options.ssl.cert
         .split(":")
-        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim())))
+        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
     };
 
-    server = https.createServer(options, listener);
+    server = https.createServer(opts, listener);
-    if (onConnection != null) server.on("secureConnection", onConnection);
+    if (options.onConnection)
+      server.on("secureConnection", options.onConnection);
   } else {
     server = http.createServer(listener);
-    if (onConnection != null) server.on("connection", onConnection);
+    if (options.onConnection) server.on("connection", options.onConnection);
   }
 
-  if (keepAliveTimeout) server.keepAliveTimeout = keepAliveTimeout;
+  server.timeout = options.timeout || 0;
-  server.listen(port, networkInterface);
+  if (options.keepAliveTimeout != null)
+    server.keepAliveTimeout = options.keepAliveTimeout;
+  server.listen({ port: options.port, host: options.host });
 }
 
 export function stop(): Promise<void> {

lib/soap-libxmljs.ts

@@ -1,745 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { parseXml, Document, Element, ParserOptions } from "libxmljs";
-import * as config from "./config";
-import { version as VERSION } from "../package.json";
-import {
-  CpeGetResponse,
-  CpeSetResponse,
-  InformRequest,
-  AcsResponse,
-  CpeRequest,
-  FaultStruct,
-  SpvFault,
-  CpeFault,
-  SoapMessage,
-  TransferCompleteRequest
-} from "./types";
-
-const SERVER_NAME = `GenieACS/${VERSION}`;
-
-const NAMESPACES = {
-  "1.0": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-0"
-  },
-  "1.1": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-1"
-  },
-  "1.2": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
-  },
-  "1.3": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
-  },
-  "1.4": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-3"
-  }
-};
-
-const LIBXMLJS_PARSE_OPTIONS: ParserOptions = {
-  nocdata: true,
-  recover: !!config.get("XML_RECOVER"),
-  /* eslint-disable-next-line @typescript-eslint/camelcase */
-  ignore_enc: !!config.get("XML_IGNORE_ENC")
-};
-
-interface WriterOptions {
-  declaration?: boolean;
-  format?: boolean;
-  selfCloseEmpty?: boolean;
-  whitespace?: boolean;
-  type?: "XML" | "HTML" | "XHTML";
-}
-
-declare module "libxmljs" {
-  interface Document {
-    toString(formatting: boolean | WriterOptions): string;
-  }
-}
-
-const LIBXMLJS_SAVE_OPTIONS: WriterOptions = {
-  format: !!config.get("XML_FORMAT"),
-  declaration: !config.get("XML_NO_DECL"),
-  selfCloseEmpty: !config.get("XML_NO_EMPTY")
-};
-
-let warnings;
-
-function parseBool(v): boolean {
-  v = "" + v;
-  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
-  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
-    return false;
-  else return null;
-}
-
-function event(xml): string[] {
-  return xml
-    .childNodes()
-    .filter(n => n.name() === "EventStruct")
-    .map(c =>
-      c
-        .childNodes()
-        .find(n => n.name() === "EventCode")
-        .text()
-        .trim()
-    );
-}
-
-function parameterInfoList(xml): [string, boolean][] {
-  return xml
-    .childNodes()
-    .filter(e => e.name() === "ParameterInfoStruct")
-    .map(e => {
-      let param, value;
-      for (const c of e.childNodes()) {
-        switch (c.name()) {
-          case "Name":
-            param = c.text();
-            break;
-          case "Writable":
-            value = c.text();
-            break;
-        }
-      }
-
-      let parsed = parseBool(value);
-
-      if (parsed === null) {
-        warnings.push({
-          message: "Invalid writable attribute",
-          parameter: param
-        });
-        parsed = false;
-      }
-
-      return [param, parsed];
-    });
-}
-
-function parameterValueList(
-  xml
-): [string, string | number | boolean, string][] {
-  return xml
-    .childNodes()
-    .filter(e => e.name() === "ParameterValueStruct")
-    .map(e => {
-      let valueElement, param;
-      for (const c of e.childNodes()) {
-        switch (c.name()) {
-          case "Name":
-            param = c.text();
-            break;
-          case "Value":
-            valueElement = c;
-            break;
-        }
-      }
-
-      const valueType = valueElement.attr("type").value();
-      const value = valueElement.text();
-      let parsed = value;
-      if (valueType === "xsd:boolean") {
-        parsed = parseBool(value);
-        if (parsed === null) {
-          warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
-          });
-          parsed = value;
-        }
-      } else if (valueType === "xsd:int" || valueType === "xsd:unsignedInt") {
-        parsed = parseInt(value);
-        if (isNaN(parsed)) {
-          warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
-          });
-          parsed = value;
-        }
-      } else if (valueType === "xsd:dateTime") {
-        parsed = Date.parse(value);
-        if (isNaN(parsed)) {
-          warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
-          });
-          parsed = value;
-        }
-      }
-
-      return [param, parsed, valueType];
-    });
-}
-
-function GetParameterNames(xml, methodRequest): void {
-  const el = xml.node("cwmp:GetParameterNames");
-  el.node("ParameterPath").text(methodRequest.parameterPath);
-  el.node("NextLevel").text(+methodRequest.nextLevel);
-}
-
-function GetParameterNamesResponse(xml): CpeGetResponse {
-  return {
-    name: "GetParameterNamesResponse",
-    parameterList: parameterInfoList(
-      xml.childNodes().find(n => n.name() === "ParameterList")
-    )
-  };
-}
-
-function GetParameterValues(xml, methodRequest): void {
-  const el = xml.node("cwmp:GetParameterValues").node("ParameterNames");
-  el.attr({
-    "soap-enc:arrayType": `xsd:string[${methodRequest.parameterNames.length}]`
-  });
-
-  for (const p of methodRequest.parameterNames) el.node("string").text(p);
-}
-
-function GetParameterValuesResponse(xml): CpeGetResponse {
-  return {
-    name: "GetParameterValuesResponse",
-    parameterList: parameterValueList(
-      xml.childNodes().find(n => n.name() === "ParameterList")
-    )
-  };
-}
-
-function SetParameterValues(xml, methodRequest): void {
-  const el = xml.node("cwmp:SetParameterValues");
-  const paramList = el.node("ParameterList");
-  paramList.attr({
-    "soap-enc:arrayType": `cwmp:ParameterValueStruct[${methodRequest.parameterList.length}]`
-  });
-
-  for (const p of methodRequest.parameterList) {
-    const pvs = paramList.node("ParameterValueStruct");
-    pvs.node("Name").text(p[0]);
-    let val = p[1];
-    if (p[2] === "xsd:dateTime" && typeof val === "number") {
-      val = new Date(val).toISOString();
-      if (methodRequest.DATETIME_MILLISECONDS === false)
-        val = val.replace(".000", "");
-    }
-    if (p[2] === "xsd:boolean" && typeof val === "boolean")
-      if (methodRequest.BOOLEAN_LITERAL === false) val = +val;
-
-    pvs
-      .node("Value")
-      .attr({ "xsi:type": p[2] })
-      .text(val);
-  }
-
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-function SetParameterValuesResponse(xml): CpeSetResponse {
-  return {
-    name: "SetParameterValuesResponse",
-    status: parseInt(
-      xml
-        .childNodes()
-        .find(n => n.name() === "Status")
-        .text()
-    )
-  };
-}
-
-function AddObject(xml, methodRequest): void {
-  const el = xml.node("cwmp:AddObject");
-  el.node("ObjectName").text(methodRequest.objectName);
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-function AddObjectResponse(xml): CpeSetResponse {
-  let instanceNumber, status;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "InstanceNumber":
-        instanceNumber = parseInt(c.text());
-        break;
-      case "Status":
-        status = parseInt(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "AddObjectResponse",
-    instanceNumber: instanceNumber,
-    status: status
-  };
-}
-
-function DeleteObject(xml, methodRequest): void {
-  const el = xml.node("cwmp:DeleteObject");
-  el.node("ObjectName").text(methodRequest.objectName);
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-function DeleteObjectResponse(xml): CpeSetResponse {
-  return {
-    name: "DeleteObjectResponse",
-    status: parseInt(
-      xml
-        .childNodes()
-        .find(n => n.name() === "Status")
-        .text()
-    )
-  };
-}
-
-function Reboot(xml, methodRequest): void {
-  const el = xml.node("cwmp:Reboot");
-  el.node("CommandKey").text(methodRequest.commandKey || "");
-}
-
-function RebootResponse(): CpeSetResponse {
-  return {
-    name: "RebootResponse"
-  };
-}
-
-function FactoryReset(xml): void {
-  xml.node("cwmp:FactoryReset");
-}
-
-function FactoryResetResponse(): CpeSetResponse {
-  return {
-    name: "FactoryResetResponse"
-  };
-}
-
-function Download(xml, methodRequest): void {
-  const el = xml.node("cwmp:Download");
-  el.node("CommandKey").text(methodRequest.commandKey || "");
-  el.node("FileType").text(methodRequest.fileType);
-  el.node("URL").text(methodRequest.url);
-  el.node("Username").text(methodRequest.username || "");
-  el.node("Password").text(methodRequest.password || "");
-  el.node("FileSize").text(methodRequest.fileSize || "0");
-  el.node("TargetFileName").text(methodRequest.targetFileName || "");
-  el.node("DelaySeconds").text(methodRequest.delaySeconds || "0");
-  el.node("SuccessURL").text(methodRequest.successUrl || "");
-  el.node("FailureURL").text(methodRequest.failureUrl || "");
-}
-
-function DownloadResponse(xml): CpeSetResponse {
-  let status, startTime, completeTime;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "Status":
-        status = parseInt(c.text());
-        break;
-      case "StartTime":
-        startTime = Date.parse(c.text());
-        break;
-      case "CompleteTime":
-        completeTime = Date.parse(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "DownloadResponse",
-    status: status,
-    startTime: startTime,
-    completeTime: completeTime
-  };
-}
-
-function Inform(xml): InformRequest {
-  let retryCount, evnt, parameterList;
-  const deviceId = {
-    Manufacturer: null,
-    OUI: null,
-    ProductClass: null,
-    SerialNumber: null
-  };
-
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "ParameterList":
-        parameterList = parameterValueList(c);
-        break;
-      case "DeviceId":
-        for (const cc of c.childNodes()) {
-          const n = cc.name();
-          if (n in deviceId) deviceId[n] = cc.text();
-        }
-        break;
-      case "Event":
-        evnt = event(c);
-        break;
-      case "RetryCount":
-        retryCount = parseInt(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "Inform",
-    parameterList: parameterList,
-    deviceId: deviceId,
-    event: evnt,
-    retryCount: retryCount
-  };
-}
-
-function InformResponse(xml): void {
-  xml
-    .node("cwmp:InformResponse")
-    .node("MaxEnvelopes")
-    .text(1);
-}
-
-function GetRPCMethods(): AcsResponse {
-  return { name: "GetRPCMethods" };
-}
-
-function GetRPCMethodsResponse(xml, methodResponse): void {
-  const el = xml.node("cwmp:GetRPCMethodsResponse").node("MethodList");
-  el.attr({
-    "soap-enc:arrayType": `xsd:string[${methodResponse.methodList.length}]`
-  });
-
-  for (const m of methodResponse.methodList) el.node("string").text(m);
-}
-
-function TransferComplete(xml): TransferCompleteRequest {
-  let commandKey, _faultStruct, startTime, completeTime;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "CommandKey":
-        commandKey = c.text();
-        break;
-      case "FaultStruct":
-        _faultStruct = faultStruct(c);
-        break;
-      case "StartTime":
-        startTime = Date.parse(c.text());
-        break;
-      case "CompleteTime":
-        completeTime = Date.parse(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "TransferComplete",
-    commandKey: commandKey,
-    faultStruct: _faultStruct,
-    startTime: startTime,
-    completeTime: completeTime
-  };
-}
-
-function TransferCompleteResponse(xml): void {
-  xml.node("cwmp:TransferCompleteResponse").text("");
-}
-
-function RequestDownload(xml): CpeRequest {
-  return {
-    name: "RequestDownload",
-    fileType: xml
-      .childNodes()
-      .find(n => n.name() === "FileType")
-      .text()
-  };
-}
-
-function RequestDownloadResponse(xml): void {
-  xml.node("cwmp:RequestDownloadResponse").text("");
-}
-
-function faultStruct(xml): FaultStruct {
-  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "FaultCode":
-        faultCode = c.text();
-        break;
-      case "FaultString":
-        faultString = c.text();
-        break;
-      case "SetParameterValuesFault":
-        setParameterValuesFault = setParameterValuesFault || [];
-        for (const cc of c.childNodes()) {
-          switch (cc.name()) {
-            case "ParameterName":
-              pn = cc.text();
-              break;
-            case "FaultCode":
-              fc = cc.text();
-              break;
-            case "FaultString":
-              fs = cc.text();
-              break;
-          }
-        }
-        setParameterValuesFault.push({
-          parameterName: pn,
-          faultCode: fc,
-          faultString: fs
-        });
-    }
-  }
-
-  return { faultCode, faultString, setParameterValuesFault };
-}
-
-function fault(xml): CpeFault {
-  let faultCode, faultString, detail;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "faultcode":
-        faultCode = c.text();
-        break;
-      case "faultstring":
-        faultString = c.text();
-        break;
-      case "detail":
-        detail = faultStruct(c.childNodes().find(n => n.name() === "Fault"));
-        break;
-    }
-  }
-
-  return { faultCode, faultString, detail };
-}
-
-export function request(data, cwmpVersion, warn): SoapMessage {
-  warnings = warn;
-
-  const rpc = {
-    id: null,
-    cwmpVersion: cwmpVersion,
-    sessionTimeout: null,
-    cpeRequest: null,
-    cpeFault: null,
-    cpeResponse: null
-  };
-
-  if (!data.length) return rpc;
-
-  const xml = parseXml(data, LIBXMLJS_PARSE_OPTIONS);
-
-  let headerElement, bodyElement;
-
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "Header":
-        headerElement = c;
-        break;
-      case "Body":
-        bodyElement = c;
-        break;
-    }
-  }
-
-  if (headerElement) {
-    for (const c of headerElement.childNodes()) {
-      switch (c.name()) {
-        case "ID":
-          rpc.id = c.text();
-          break;
-        case "sessionTimeout":
-          rpc.sessionTimeout = parseInt(c.text());
-          break;
-      }
-    }
-  }
-
-  for (const methodElement of bodyElement.childNodes()) {
-    const methodName = methodElement.name();
-    if (methodName === "text") continue;
-
-    if (!rpc.cwmpVersion && methodName !== "Fault") {
-      switch (methodElement.namespace().href()) {
-        case "urn:dslforum-org:cwmp-1-0":
-          rpc.cwmpVersion = "1.0";
-          break;
-        case "urn:dslforum-org:cwmp-1-1":
-          rpc.cwmpVersion = "1.1";
-          break;
-        case "urn:dslforum-org:cwmp-1-2":
-          if (rpc.sessionTimeout) rpc.cwmpVersion = "1.3";
-          else rpc.cwmpVersion = "1.2";
-
-          break;
-        case "urn:dslforum-org:cwmp-1-3":
-          rpc.cwmpVersion = "1.4";
-          break;
-        default:
-          throw new Error("Unrecognized CWMP version");
-      }
-    }
-
-    switch (methodName) {
-      case "Inform":
-        rpc.cpeRequest = Inform(methodElement);
-        break;
-      case "GetRPCMethods":
-        rpc.cpeRequest = GetRPCMethods();
-        break;
-      case "TransferComplete":
-        rpc.cpeRequest = TransferComplete(methodElement);
-        break;
-      case "RequestDownload":
-        rpc.cpeRequest = RequestDownload(methodElement);
-        break;
-      case "GetParameterNamesResponse":
-        rpc.cpeResponse = GetParameterNamesResponse(methodElement);
-        break;
-      case "GetParameterValuesResponse":
-        rpc.cpeResponse = GetParameterValuesResponse(methodElement);
-        break;
-      case "SetParameterValuesResponse":
-        rpc.cpeResponse = SetParameterValuesResponse(methodElement);
-        break;
-      case "AddObjectResponse":
-        rpc.cpeResponse = AddObjectResponse(methodElement);
-        break;
-      case "DeleteObjectResponse":
-        rpc.cpeResponse = DeleteObjectResponse(methodElement);
-        break;
-      case "RebootResponse":
-        rpc.cpeResponse = RebootResponse();
-        break;
-      case "FactoryResetResponse":
-        rpc.cpeResponse = FactoryResetResponse();
-        break;
-      case "DownloadResponse":
-        rpc.cpeResponse = DownloadResponse(methodElement);
-        break;
-      case "Fault":
-        rpc.cpeFault = fault(methodElement);
-        break;
-      default:
-        throw new Error(`8000 Method not supported ${methodName}`);
-    }
-  }
-
-  return rpc;
-}
-
-function createSoapEnv(cwmpVersion: string): Element {
-  const xml = new Document();
-  const env = xml.node("soap-env:Envelope");
-
-  for (const prefix of Object.keys(NAMESPACES[cwmpVersion]))
-    env.defineNamespace(prefix, NAMESPACES[cwmpVersion][prefix]);
-
-  return env;
-}
-
-export function response(rpc): { code: number; headers: {}; data: string } {
-  const headers = {
-    Server: SERVER_NAME,
-    SOAPServer: SERVER_NAME
-  };
-
-  if (!rpc) return { code: 204, headers: headers, data: "" };
-
-  const env = createSoapEnv(rpc.cwmpVersion);
-  const header = env.node("soap-env:Header");
-  header
-    .node("cwmp:ID")
-    .attr({
-      "soap-env:mustUnderstand": "1"
-    })
-    .text(rpc.id);
-
-  const body = env.node("soap-env:Body");
-
-  if (rpc.acsResponse) {
-    switch (rpc.acsResponse.name) {
-      case "InformResponse":
-        InformResponse(body);
-        break;
-      case "GetRPCMethodsResponse":
-        GetRPCMethodsResponse(body, rpc.acsResponse);
-        break;
-      case "TransferCompleteResponse":
-        TransferCompleteResponse(body);
-        break;
-      case "RequestDownloadResponse":
-        RequestDownloadResponse(body);
-        break;
-      default:
-        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
-    }
-  } else if (rpc.acsRequest) {
-    switch (rpc.acsRequest.name) {
-      case "GetParameterNames":
-        GetParameterNames(body, rpc.acsRequest);
-        break;
-      case "GetParameterValues":
-        GetParameterValues(body, rpc.acsRequest);
-        break;
-      case "SetParameterValues":
-        SetParameterValues(body, rpc.acsRequest);
-        break;
-      case "AddObject":
-        AddObject(body, rpc.acsRequest);
-        break;
-      case "DeleteObject":
-        DeleteObject(body, rpc.acsRequest);
-        break;
-      case "Reboot":
-        Reboot(body, rpc.acsRequest);
-        break;
-      case "FactoryReset":
-        FactoryReset(body);
-        break;
-      case "Download":
-        Download(body, rpc.acsRequest);
-        break;
-      default:
-        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
-    }
-  }
-
-  headers["Content-Type"] = 'text/xml; charset="utf-8"';
-  return {
-    code: 200,
-    headers: headers,
-    data: env.doc().toString(LIBXMLJS_SAVE_OPTIONS)
-  };
-}

lib/soap.ts

@@ -22,22 +22,34 @@ import {
   Element,
   parseAttrs,
   encodeEntities,
-  decodeEntities
+  decodeEntities,
 } from "./xml-parser";
 import memoize from "./common/memoize";
 import { version as VERSION } from "../package.json";
 import {
-  CpeGetResponse,
-  CpeSetResponse,
   InformRequest,
-  AcsResponse,
-  CpeRequest,
   FaultStruct,
   SpvFault,
   CpeFault,
   SoapMessage,
-  TransferCompleteRequest
+  TransferCompleteRequest,
+  AcsRequest,
+  GetParameterNamesResponse,
+  GetParameterValuesResponse,
+  GetParameterAttributesResponse,
+  SetParameterValuesResponse,
+  SetParameterAttributesResponse,
+  AddObjectResponse,
+  DeleteObject,
+  DeleteObjectResponse,
+  RebootResponse,
+  FactoryResetResponse,
+  DownloadResponse,
+  GetRPCMethodsRequest,
+  RequestDownloadRequest,
+  AcsResponse,
 } from "./types";
+import Path from "./common/path";
 
 const SERVER_NAME = `GenieACS/${VERSION}`;
 
@@ -47,60 +59,58 @@ const NAMESPACES = {
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-0"
+    cwmp: "urn:dslforum-org:cwmp-1-0",
   },
   "1.1": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-1"
+    cwmp: "urn:dslforum-org:cwmp-1-1",
   },
   "1.2": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
+    cwmp: "urn:dslforum-org:cwmp-1-2",
   },
   "1.3": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
+    cwmp: "urn:dslforum-org:cwmp-1-2",
   },
   "1.4": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-3"
+    cwmp: "urn:dslforum-org:cwmp-1-3",
-  }
+  },
 };
 
-let warnings;
+let warnings: Record<string, unknown>[];
 
 const memoizedParseAttrs = memoize(parseAttrs);
 
-function parseBool(v): boolean {
+function parseBool(v: string): boolean {
-  v = "" + v;
+  if (v === "true" || v === "1") return true;
-  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
+  if (v === "false" || v === "0") return false;
-  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+  return null;
-    return false;
-  else return null;
 }
 
 function event(xml: Element): string[] {
   return xml.children
-    .filter(n => n.localName === "EventStruct")
+    .filter((n) => n.localName === "EventStruct")
-    .map(c => c.children.find(n => n.localName === "EventCode").text.trim());
+    .map((c) => c.children.find((n) => n.localName === "EventCode").text);
 }
 
-function parameterInfoList(xml: Element): [string, boolean][] {
+function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
   return xml.children
-    .filter(e => e.localName === "ParameterInfoStruct")
+    .filter((e) => e.localName === "ParameterInfoStruct")
-    .map<[string, boolean]>(e => {
+    .map<[Path, boolean, boolean]>((e) => {
       let param: string, value: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -115,30 +125,34 @@ function parameterInfoList(xml: Element): [string, boolean][] {
 
       let parsed: boolean = parseBool(value);
 
-      if (parsed === null) {
+      if (parsed == null) {
         warnings.push({
-          message: "Invalid writable attribute",
+          message: "Missing or invalid XML node",
-          parameter: param
+          element: "Writable",
+          parameter: param,
         });
         parsed = false;
       }
 
-      return [param, parsed];
+      if (param && !param.endsWith("."))
+        return [Path.parse(param), false, parsed];
+      else return [Path.parse(param.slice(0, -1)), true, parsed];
     });
 }
 
-const getValueType = memoize((str: string) =>
+const getValueType = memoize((str: string) => {
-  parseAttrs(str)
+  const attrs = parseAttrs(str);
-    .find(s => s.localName === "type")
+  for (const attr of attrs) if (attr.localName === "type") return attr.value;
-    .value.trim()
+
-);
+  return null;
+});
 
 function parameterValueList(
   xml: Element
-): [string, string | number | boolean, string][] {
+): [Path, string | number | boolean, string][] {
   return xml.children
-    .filter(e => e.localName === "ParameterValueStruct")
+    .filter((e) => e.localName === "ParameterValueStruct")
-    .map<[string, string | number | boolean, string]>(e => {
+    .map<[Path, string | number | boolean, string]>((e) => {
       let valueElement: Element, param: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -151,16 +165,25 @@ function parameterValueList(
         }
       }
 
-      const valueType = getValueType(valueElement.attrs);
+      let valueType = getValueType(valueElement.attrs);
+      if (!valueType) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          attribute: "type",
+          parameter: param,
+        });
+        valueType = "xsd:string";
+      }
 
       const value = decodeEntities(valueElement.text);
       let parsed: string | number | boolean = value;
       if (valueType === "xsd:boolean") {
         parsed = parseBool(value);
-        if (parsed === null) {
+        if (parsed == null) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
-            parameter: param
+            element: "Value",
+            parameter: param,
           });
           parsed = value;
         }
@@ -168,8 +191,9 @@ function parameterValueList(
         parsed = parseInt(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
-            parameter: param
+            element: "Value",
+            parameter: param,
           });
           parsed = value;
         }
@@ -177,14 +201,54 @@ function parameterValueList(
         parsed = Date.parse(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
-            parameter: param
+            element: "Value",
+            parameter: param,
           });
           parsed = value;
         }
       }
 
-      return [param, parsed, valueType];
+      return [Path.parse(param), parsed, valueType];
+    });
+}
+
+function parameterAttributeList(xml: Element): [Path, number, string[]][] {
+  return xml.children
+    .filter((e) => e.localName === "ParameterAttributeStruct")
+    .map<[Path, number, string[]]>((e) => {
+      let notificationElement: Element,
+        accessListElement: Element,
+        param: string;
+      for (const c of e.children) {
+        switch (c.localName) {
+          case "Name":
+            param = c.text;
+            break;
+          case "Notification":
+            notificationElement = c;
+            break;
+          case "AccessList":
+            accessListElement = c;
+            break;
+        }
+      }
+
+      let notification = parseInt(notificationElement.text);
+      if (isNaN(notification)) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Notification",
+          parameter: param,
+        });
+        notification = 0;
+      }
+
+      const accessList = accessListElement.children
+        .filter((c) => c.localName === "string")
+        .map((c) => decodeEntities(c.text));
+
+      return [Path.parse(param), notification, accessList];
     });
 }
 
@@ -194,12 +258,12 @@ function GetParameterNames(methodRequest): string {
   }</ParameterPath><NextLevel>${+methodRequest.nextLevel}</NextLevel></cwmp:GetParameterNames>`;
 }
 
-function GetParameterNamesResponse(xml): CpeGetResponse {
+function GetParameterNamesResponse(xml): GetParameterNamesResponse {
   return {
     name: "GetParameterNamesResponse",
     parameterList: parameterInfoList(
-      xml.children.find(n => n.localName === "ParameterList")
+      xml.children.find((n) => n.localName === "ParameterList")
-    )
+    ),
   };
 }
 
@@ -207,21 +271,40 @@ function GetParameterValues(methodRequest): string {
   return `<cwmp:GetParameterValues><ParameterNames soap-enc:arrayType="xsd:string[${
     methodRequest.parameterNames.length
   }]">${methodRequest.parameterNames
-    .map(p => `<string>${p}</string>`)
+    .map((p) => `<string>${p}</string>`)
     .join("")}</ParameterNames></cwmp:GetParameterValues>`;
 }
 
-function GetParameterValuesResponse(xml: Element): CpeGetResponse {
+function GetParameterValuesResponse(xml: Element): GetParameterValuesResponse {
   return {
     name: "GetParameterValuesResponse",
     parameterList: parameterValueList(
-      xml.children.find(n => n.localName === "ParameterList")
+      xml.children.find((n) => n.localName === "ParameterList")
-    )
+    ),
+  };
+}
+
+function GetParameterAttributes(methodRequest): string {
+  return `<cwmp:GetParameterAttributes><ParameterNames soap-enc:arrayType="xsd:string[${
+    methodRequest.parameterNames.length
+  }]">${methodRequest.parameterNames
+    .map((p) => `<string>${p}</string>`)
+    .join("")}</ParameterNames></cwmp:GetParameterAttributes>`;
+}
+
+function GetParameterAttributesResponse(
+  xml: Element
+): GetParameterAttributesResponse {
+  return {
+    name: "GetParameterAttributesResponse",
+    parameterList: parameterAttributeList(
+      xml.children.find((n) => n.localName === "ParameterList")
+    ),
   };
 }
 
 function SetParameterValues(methodRequest): string {
-  const params = methodRequest.parameterList.map(p => {
+  const params = methodRequest.parameterList.map((p) => {
     let val = p[1];
     if (p[2] === "xsd:dateTime" && typeof val === "number") {
       val = new Date(val).toISOString();
@@ -237,32 +320,80 @@ function SetParameterValues(methodRequest): string {
 
   return `<cwmp:SetParameterValues><ParameterList soap-enc:arrayType="cwmp:ParameterValueStruct[${
     methodRequest.parameterList.length
-  }]">${params.join(
+  }]">${params.join("")}</ParameterList><ParameterKey>${
-    ""
+    methodRequest.parameterKey || ""
-  )}</ParameterList><ParameterKey>${methodRequest.parameterKey ||
+  }</ParameterKey></cwmp:SetParameterValues>`;
-    ""}</ParameterKey></cwmp:SetParameterValues>`;
 }
 
-function SetParameterValuesResponse(xml: Element): CpeSetResponse {
+function SetParameterValuesResponse(xml: Element): SetParameterValuesResponse {
+  let status: number;
+
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "Status":
+        status = parseInt(c.text);
+        break;
+    }
+  }
+
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
   return {
     name: "SetParameterValuesResponse",
-    status: parseInt(xml.children.find(n => n.localName === "Status").text)
+    status: status,
+  };
+}
+
+function SetParameterAttributes(methodRequest): string {
+  const params = methodRequest.parameterList.map((p) => {
+    return `<SetParameterAttributesStruct><Name>${
+      p[0]
+    }</Name><NotificationChange>${
+      p[1] == null ? "false" : "true"
+    }</NotificationChange><Notification>${
+      p[1] == null ? "" : p[1]
+    }</Notification><AccessListChange>${
+      p[2] == null ? "false" : "true"
+    }</AccessListChange><AccessList soap-enc:arrayType="xsd:string[${
+      (p[2] || []).length
+    }]">${
+      p[2] == null
+        ? ""
+        : p[2].map((s) => `<string>${encodeEntities(s)}</string>`).join("")
+    }</AccessList></SetParameterAttributesStruct>`;
+  });
+
+  return `<cwmp:SetParameterAttributes><ParameterList soap-enc:arrayType="cwmp:SetParameterAttributesStruct[${
+    methodRequest.parameterList.length
+  }]">${params.join("")}</ParameterList></cwmp:SetParameterAttributes>`;
+}
+
+function SetParameterAttributesResponse(): SetParameterAttributesResponse {
+  return {
+    name: "SetParameterAttributesResponse",
   };
 }
 
 function AddObject(methodRequest): string {
   return `<cwmp:AddObject><ObjectName>${
     methodRequest.objectName
-  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
+  }</ObjectName><ParameterKey>${
-    ""}</ParameterKey></cwmp:AddObject>`;
+    methodRequest.parameterKey || ""
+  }</ParameterKey></cwmp:AddObject>`;
 }
 
-function AddObjectResponse(xml: Element): CpeSetResponse {
+function AddObjectResponse(xml: Element): AddObjectResponse {
-  let instanceNumber, status;
+  let instanceNumber: string, status: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "InstanceNumber":
-        instanceNumber = parseInt(c.text);
+        instanceNumber = c.text;
         break;
       case "Status":
         status = parseInt(c.text);
@@ -270,35 +401,66 @@ function AddObjectResponse(xml: Element): CpeSetResponse {
     }
   }
 
+  if (!/^[0-9]+$/.test(instanceNumber))
+    throw new Error("Missing or invalid instance number");
+
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
   return {
     name: "AddObjectResponse",
     instanceNumber: instanceNumber,
-    status: status
+    status: status,
   };
 }
 
 function DeleteObject(methodRequest): string {
   return `<cwmp:DeleteObject><ObjectName>${
     methodRequest.objectName
-  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
+  }</ObjectName><ParameterKey>${
-    ""}</ParameterKey></cwmp:DeleteObject>`;
+    methodRequest.parameterKey || ""
+  }</ParameterKey></cwmp:DeleteObject>`;
 }
 
-function DeleteObjectResponse(xml: Element): CpeSetResponse {
+function DeleteObjectResponse(xml: Element): DeleteObjectResponse {
+  let status: number;
+
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "Status":
+        status = parseInt(c.text);
+        break;
+    }
+  }
+
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
   return {
     name: "DeleteObjectResponse",
-    status: parseInt(xml.children.find(n => n.localName === "Status").text)
+    status: status,
   };
 }
 
 function Reboot(methodRequest): string {
-  return `<cwmp:Reboot><CommandKey>${methodRequest.commandKey ||
+  return `<cwmp:Reboot><CommandKey>${
-    ""}</CommandKey></cwmp:Reboot>`;
+    methodRequest.commandKey || ""
+  }</CommandKey></cwmp:Reboot>`;
 }
 
-function RebootResponse(): CpeSetResponse {
+function RebootResponse(): RebootResponse {
   return {
-    name: "RebootResponse"
+    name: "RebootResponse",
   };
 }
 
@@ -306,33 +468,36 @@ function FactoryReset(): string {
   return "<cwmp:FactoryReset></cwmp:FactoryReset>";
 }
 
-function FactoryResetResponse(): CpeSetResponse {
+function FactoryResetResponse(): FactoryResetResponse {
   return {
-    name: "FactoryResetResponse"
+    name: "FactoryResetResponse",
   };
 }
 
 function Download(methodRequest): string {
-  return `<cwmp:Download><CommandKey>${methodRequest.commandKey ||
+  return `<cwmp:Download><CommandKey>${
-    ""}</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
+    methodRequest.commandKey || ""
+  }</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
     methodRequest.url
   }</URL><Username>${encodeEntities(
     methodRequest.username || ""
   )}</Username><Password>${encodeEntities(
     methodRequest.password || ""
-  )}</Password><FileSize>${methodRequest.fileSize ||
+  )}</Password><FileSize>${
-    "0"}</FileSize><TargetFileName>${encodeEntities(
+    methodRequest.fileSize || "0"
+  }</FileSize><TargetFileName>${encodeEntities(
     methodRequest.targetFileName || ""
-  )}</TargetFileName><DelaySeconds>${methodRequest.delaySeconds ||
+  )}</TargetFileName><DelaySeconds>${
-    "0"}</DelaySeconds><SuccessURL>${encodeEntities(
+    methodRequest.delaySeconds || "0"
+  }</DelaySeconds><SuccessURL>${encodeEntities(
     methodRequest.successUrl || ""
   )}</SuccessURL><FailureURL>${encodeEntities(
     methodRequest.failureUrl || ""
   )}</FailureURL></cwmp:Download>`;
 }
 
-function DownloadResponse(xml: Element): CpeSetResponse {
+function DownloadResponse(xml: Element): DownloadResponse {
-  let status, startTime, completeTime;
+  let status: number, startTime: number, completeTime: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "Status":
@@ -347,21 +512,46 @@ function DownloadResponse(xml: Element): CpeSetResponse {
     }
   }
 
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "DownloadResponse",
     status: status,
     startTime: startTime,
-    completeTime: completeTime
+    completeTime: completeTime,
   };
 }
 
 function Inform(xml: Element): InformRequest {
-  let retryCount, evnt, parameterList;
+  let retryCount: number, evnt: string[];
+  let parameterList: [Path, string | number | boolean, string][];
   const deviceId = {
     Manufacturer: null,
     OUI: null,
     ProductClass: null,
-    SerialNumber: null
+    SerialNumber: null,
   };
 
   for (const c of xml.children) {
@@ -384,12 +574,36 @@ function Inform(xml: Element): InformRequest {
     }
   }
 
+  if (!deviceId || !deviceId.SerialNumber || !deviceId.OUI)
+    throw new Error("Missing or invalid DeviceId element");
+
+  if (!parameterList) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "ParameterList",
+    });
+    parameterList = [];
+  }
+
+  if (!evnt) {
+    warnings.push({ message: "Missing or invalid XML node", element: "Event" });
+    evnt = [];
+  }
+
+  if (retryCount == null || isNaN(retryCount)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "RetryCount",
+    });
+    retryCount = 0;
+  }
+
   return {
     name: "Inform",
     parameterList: parameterList,
     deviceId: deviceId,
     event: evnt,
-    retryCount: retryCount
+    retryCount: retryCount,
   };
 }
 
@@ -397,7 +611,7 @@ function InformResponse(): string {
   return "<cwmp:InformResponse><MaxEnvelopes>1</MaxEnvelopes></cwmp:InformResponse>";
 }
 
-function GetRPCMethods(): AcsResponse {
+function GetRPCMethods(): GetRPCMethodsRequest {
   return { name: "GetRPCMethods" };
 }
 
@@ -405,12 +619,15 @@ function GetRPCMethodsResponse(methodResponse): string {
   return `<cwmp:GetRPCMethodsResponse><MethodList soap-enc:arrayType="xsd:string[${
     methodResponse.methodList.length
   }]">${methodResponse.methodList
-    .map(m => `<string>${m}</string>`)
+    .map((m) => `<string>${m}</string>`)
     .join("")}</MethodList></cwmp:GetRPCMethodsResponse>`;
 }
 
 function TransferComplete(xml: Element): TransferCompleteRequest {
-  let commandKey, _faultStruct, startTime, completeTime;
+  let commandKey: string,
+    _faultStruct: FaultStruct,
+    startTime: number,
+    completeTime: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "CommandKey":
@@ -428,12 +645,44 @@ function TransferComplete(xml: Element): TransferCompleteRequest {
     }
   }
 
+  if (commandKey == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CommandKey",
+    });
+    commandKey = "";
+  }
+
+  if (!_faultStruct) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultStruct",
+    });
+    _faultStruct = { faultCode: "0", faultString: "" };
+  }
+
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "TransferComplete",
     commandKey: commandKey,
     faultStruct: _faultStruct,
     startTime: startTime,
-    completeTime: completeTime
+    completeTime: completeTime,
   };
 }
 
@@ -441,10 +690,10 @@ function TransferCompleteResponse(): string {
   return "<cwmp:TransferCompleteResponse></cwmp:TransferCompleteResponse>";
 }
 
-function RequestDownload(xml: Element): CpeRequest {
+function RequestDownload(xml: Element): RequestDownloadRequest {
   return {
     name: "RequestDownload",
-    fileType: xml.children.find(n => n.localName === "FileType").text
+    fileType: xml.children.find((n) => n.localName === "FileType").text,
   };
 }
 
@@ -453,7 +702,12 @@ function RequestDownloadResponse(): string {
 }
 
 function faultStruct(xml: Element): FaultStruct {
-  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
+  let faultCode: string,
+    faultString: string,
+    setParameterValuesFault: SpvFault[],
+    pn: string,
+    fc: string,
+    fs: string;
   for (const c of xml.children) {
     switch (c.localName) {
       case "FaultCode":
@@ -464,6 +718,7 @@ function faultStruct(xml: Element): FaultStruct {
         break;
       case "SetParameterValuesFault":
         setParameterValuesFault = setParameterValuesFault || [];
+        pn = fc = fs = null;
         for (const cc of c.children) {
           switch (cc.localName) {
             case "ParameterName":
@@ -480,16 +735,32 @@ function faultStruct(xml: Element): FaultStruct {
         setParameterValuesFault.push({
           parameterName: pn,
           faultCode: fc,
-          faultString: fs
+          faultString: fs,
         });
     }
   }
 
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultCode",
+    });
+    faultCode = "";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultString",
+    });
+    faultString = "";
+  }
+
   return { faultCode, faultString, setParameterValuesFault };
 }
 
 function fault(xml: Element): CpeFault {
-  let faultCode, faultString, detail;
+  let faultCode: string, faultString: string, detail: FaultStruct;
   for (const c of xml.children) {
     switch (c.localName) {
       case "faultcode":
@@ -499,15 +770,37 @@ function fault(xml: Element): CpeFault {
         faultString = decodeEntities(c.text);
         break;
       case "detail":
-        detail = faultStruct(c.children.find(n => n.localName === "Fault"));
+        detail = faultStruct(c.children.find((n) => n.localName === "Fault"));
         break;
     }
   }
 
-  return { faultCode, faultString, detail };
+  if (!detail) throw new Error("Missing detail element");
+
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultcode",
+    });
+    faultCode = "Client";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultstring",
+    });
+    faultString = "CWMP fault";
+  }
+
+  return { faultCode, faultString, detail } as CpeFault;
 }
 
-export function request(body: string, cwmpVersion, warn): SoapMessage {
+export function request(
+  body: string,
+  cwmpVersion: string,
+  warn: Record<string, unknown>[]
+): SoapMessage {
   warnings = warn;
 
   const rpc = {
@@ -516,7 +809,7 @@ export function request(body: string, cwmpVersion, warn): SoapMessage {
     sessionTimeout: null,
     cpeRequest: null,
     cpeFault: null,
-    cpeResponse: null
+    cpeResponse: null,
   };
 
   if (!body.length) return rpc;
@@ -563,9 +856,9 @@ export function request(body: string, cwmpVersion, warn): SoapMessage {
         const attrs = memoizedParseAttrs(e.attrs);
         const attr = namespace
           ? attrs.find(
-              s => s.namespace === "xmlns" && s.localName === namespace
+              (s) => s.namespace === "xmlns" && s.localName === namespace
             )
-          : attrs.find(s => s.name === "xmlns");
+          : attrs.find((s) => s.name === "xmlns");
 
         if (attr) namespaceHref = attr.value;
       }
@@ -610,9 +903,15 @@ export function request(body: string, cwmpVersion, warn): SoapMessage {
     case "GetParameterValuesResponse":
       rpc.cpeResponse = GetParameterValuesResponse(methodElement);
       break;
+    case "GetParameterAttributesResponse":
+      rpc.cpeResponse = GetParameterAttributesResponse(methodElement);
+      break;
     case "SetParameterValuesResponse":
       rpc.cpeResponse = SetParameterValuesResponse(methodElement);
       break;
+    case "SetParameterAttributesResponse":
+      rpc.cpeResponse = SetParameterAttributesResponse();
+      break;
     case "AddObjectResponse":
       rpc.cpeResponse = AddObjectResponse(methodElement);
       break;
@@ -653,13 +952,18 @@ const namespacesAttrs = {
     .join(" "),
   "1.4": Object.entries(NAMESPACES["1.4"])
     .map(([k, v]) => `xmlns:${k}="${v}"`)
-    .join(" ")
+    .join(" "),
 };
 
-export function response(rpc): { code: number; headers: {}; data: string } {
+export function response(rpc: {
+  id: string;
+  acsRequest?: AcsRequest;
+  acsResponse?: AcsResponse;
+  cwmpVersion?: string;
+}): { code: number; headers: Record<string, string>; data: string } {
   const headers = {
     Server: SERVER_NAME,
-    SOAPServer: SERVER_NAME
+    SOAPServer: SERVER_NAME,
   };
 
   if (!rpc) return { code: 204, headers: headers, data: "" };
@@ -680,7 +984,11 @@ export function response(rpc): { code: number; headers: {}; data: string } {
         body = RequestDownloadResponse();
         break;
       default:
-        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
+        throw new Error(
+          `Unknown method response type ${
+            (rpc.acsResponse as AcsResponse).name
+          }`
+        );
     }
   } else if (rpc.acsRequest) {
     switch (rpc.acsRequest.name) {
@@ -690,9 +998,15 @@ export function response(rpc): { code: number; headers: {}; data: string } {
       case "GetParameterValues":
         body = GetParameterValues(rpc.acsRequest);
         break;
+      case "GetParameterAttributes":
+        body = GetParameterAttributes(rpc.acsRequest);
+        break;
       case "SetParameterValues":
         body = SetParameterValues(rpc.acsRequest);
         break;
+      case "SetParameterAttributes":
+        body = SetParameterAttributes(rpc.acsRequest);
+        break;
       case "AddObject":
         body = AddObject(rpc.acsRequest);
         break;
@@ -709,7 +1023,9 @@ export function response(rpc): { code: number; headers: {}; data: string } {
         body = Download(rpc.acsRequest);
         break;
       default:
-        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
+        throw new Error(
+          `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`
+        );
     }
   }
 
@@ -721,6 +1037,6 @@ export function response(rpc): { code: number; headers: {}; data: string } {
       namespacesAttrs[rpc.cwmpVersion]
     }><soap-env:Header><cwmp:ID soap-env:mustUnderstand="1">${
       rpc.id
-    }</cwmp:ID></soap-env:Header><soap-env:Body>${body}</soap-env:Body></soap-env:Envelope>`
+    }</cwmp:ID></soap-env:Header><soap-env:Body>${body}</soap-env:Body></soap-env:Envelope>`,
   };
 }

lib/types.ts

@@ -22,6 +22,7 @@ import PathSet from "./common/path-set";
 import VersionedMap from "./versioned-map";
 import InstanceSet from "./instance-set";
 import { IncomingMessage, ServerResponse } from "http";
+import { Script } from "vm";
 
 export type Expression = string | number | boolean | null | any[];
 
@@ -51,18 +52,24 @@ export interface Attributes {
   object?: [number, 1 | 0];
   writable?: [number, 1 | 0];
   value?: [number, [string | number | boolean, string]];
+  notification?: [number, number];
+  accessList?: [number, string[]];
 }
 
 export interface AttributeTimestamps {
   object?: number;
   writable?: number;
   value?: number;
+  notification?: number;
+  accessList?: number;
 }
 
 export interface AttributeValues {
   object?: boolean;
   writable?: boolean;
   value?: [string | number | boolean, string?];
+  notification?: number;
+  accessList?: string[];
 }
 
 export interface DeviceData {
@@ -75,12 +82,21 @@ export interface DeviceData {
 
 export type VirtualParameterDeclaration = [
   Path,
-  { path?: number; object?: number; writable?: number; value?: number }?,
+  {
+    path?: number;
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  }?,
   {
     path?: [number, number];
     object?: boolean;
     writable?: boolean;
     value?: [string | number | boolean, string?];
+    notification?: number;
+    accessList?: string[];
   }?
 ];
 
@@ -90,8 +106,11 @@ export interface SyncState {
     object: Set<Path>;
     writable: Set<Path>;
     value: Set<Path>;
+    notification: Set<Path>;
+    accessList: Set<Path>;
   };
   spv: Map<Path, [string | number | boolean, string]>;
+  spa: Map<Path, { notification: number; accessList: string[] }>;
   gpn: Set<Path>;
   gpnPatterns: Map<Path, number>;
   tags: Map<Path, boolean>;
@@ -131,7 +150,6 @@ export interface SessionContext {
   faults?: { [channel: string]: SessionFault };
   retries?: { [channel: string]: number };
   cacheSnapshot?: string;
-  configContext?: { [name: string]: any };
   httpResponse?: ServerResponse;
   httpRequest?: IncomingMessage;
   faultsTouched?: { [channel: string]: boolean };
@@ -139,6 +157,7 @@ export interface SessionContext {
   new?: boolean;
   debug?: boolean;
   state: number;
+  authState: number;
   tasks?: Task[];
   operations?: { [commandKey: string]: Operation };
   cacheUntil?: number;
@@ -151,7 +170,7 @@ export interface SessionContext {
 }
 
 export interface Task {
-  _id: string;
+  _id?: string;
   name: string;
   parameterNames?: string[];
   parameterValues?: [string, string | number | boolean, string?][];
@@ -160,6 +179,7 @@ export interface Task {
   fileName?: string;
   targetFileName?: string;
   expiry?: number;
+  provisions?: (string | number | boolean)[][];
 }
 
 export interface Operation {
@@ -176,44 +196,84 @@ export interface Operation {
   };
 }
 
-export interface AcsRequest {
+export type AcsRequest =
-  name: string;
+  | GetParameterNames
-  next?: string;
+  | GetParameterValues
+  | GetParameterAttributes
+  | SetParameterValues
+  | SetParameterAttributes
+  | AddObject
+  | DeleteObject
+  | FactoryReset
+  | Reboot
+  | Download;
+
+export interface GetParameterNames {
+  name: "GetParameterNames";
+  parameterPath: string;
+  nextLevel: boolean;
 }
 
-export interface GetAcsRequest extends AcsRequest {
+export interface GetParameterValues {
-  name: "GetParameterNames" | "GetParameterValues";
+  name: "GetParameterValues";
-  objectName?: string;
+  parameterNames: string[];
-  parameterNames?: string[];
-  parameterPath?: string;
-  nextLevel?: boolean;
-  instanceValues?: { [name: string]: string };
 }
 
-export interface SetAcsRequest extends AcsRequest {
+export interface GetParameterAttributes {
-  name:
+  name: "GetParameterAttributes";
-    | "SetParameterValues"
+  parameterNames: string[];
-    | "AddObject"
+}
-    | "DeleteObject"
+
-    | "FactoryReset"
+export interface SetParameterValues {
-    | "Reboot"
+  name: "SetParameterValues";
-    | "Download";
+  parameterList: [string, boolean | number | string, string][];
-  parameterList?: [string, string | number | boolean, string][];
+  parameterKey?: string;
-  instanceValues?: { [param: string]: string | number | boolean };
-  objectName?: string;
   DATETIME_MILLISECONDS?: boolean;
   BOOLEAN_LITERAL?: boolean;
-  commandKey?: string;
-  instance?: string;
-  fileType?: string;
-  fileSize?: number;
-  url?: string;
-  fileName?: string;
-  targetFileName?: string;
 }
 
-export interface CpeResponse {
+export interface SetParameterAttributes {
-  name: string;
+  name: "SetParameterAttributes";
+  parameterList: [string, number, string[]][];
+}
+
+export interface AddObject {
+  name: "AddObject";
+  objectName: string;
+  parameterKey?: string;
+  instanceValues: Record<string, string>;
+}
+
+export interface DeleteObject {
+  name: "DeleteObject";
+  objectName: string;
+  parameterKey?: string;
+}
+
+export interface FactoryReset {
+  name: "FactoryReset";
+  commandKey?: string;
+}
+
+export interface Reboot {
+  name: "Reboot";
+  commandKey?: string;
+}
+
+export interface Download {
+  name: "Download";
+  commandKey: string;
+  instance: string;
+  fileType: string;
+  fileName?: string;
+  url?: string;
+  username?: string;
+  password?: string;
+  fileSize?: number;
+  targetFileName?: string;
+  delaySecods?: number;
+  successUrl?: string;
+  failureUrl?: string;
 }
 
 export interface SpvFault {
@@ -229,38 +289,80 @@ export interface FaultStruct {
 }
 
 export interface CpeFault {
-  faultCode: string;
+  faultCode: "Client" | "Server";
-  faultString: string;
+  faultString: "CWMP fault";
   detail?: FaultStruct;
 }
 
-export interface CpeGetResponse extends CpeResponse {
+export type CpeResponse =
-  name: "GetParameterNamesResponse" | "GetParameterValuesResponse";
+  | GetParameterNamesResponse
-  parameterList?:
+  | GetParameterValuesResponse
-    | [string, boolean][]
+  | GetParameterAttributesResponse
-    | [string, string | number | boolean, string][];
+  | SetParameterValuesResponse
+  | SetParameterAttributesResponse
+  | AddObjectResponse
+  | DeleteObjectResponse
+  | RebootResponse
+  | FactoryResetResponse
+  | DownloadResponse;
+
+export interface GetParameterNamesResponse {
+  name: "GetParameterNamesResponse";
+  parameterList: [Path, boolean, boolean][];
 }
 
-export interface CpeSetResponse extends CpeResponse {
+export interface GetParameterValuesResponse {
-  name:
+  name: "GetParameterValuesResponse";
-    | "SetParameterValuesResponse"
+  parameterList: [Path, string | number | boolean, string][];
-    | "AddObjectResponse"
+}
-    | "DeleteObjectResponse"
+
-    | "RebootResponse"
+export interface GetParameterAttributesResponse {
-    | "FactoryResetResponse"
+  name: "GetParameterAttributesResponse";
-    | "DownloadResponse";
+  parameterList: [Path, number, string[]][];
-  status?: number;
+}
-  instanceNumber?: string;
+
+export interface SetParameterValuesResponse {
+  name: "SetParameterValuesResponse";
+  status: number;
+}
+
+export interface SetParameterAttributesResponse {
+  name: "SetParameterAttributesResponse";
+}
+
+export interface AddObjectResponse {
+  name: "AddObjectResponse";
+  instanceNumber: string;
+  status: number;
+}
+
+export interface DeleteObjectResponse {
+  name: "DeleteObjectResponse";
+  status: number;
+}
+
+export interface RebootResponse {
+  name: "RebootResponse";
+}
+
+export interface FactoryResetResponse {
+  name: "FactoryResetResponse";
+}
+
+export interface DownloadResponse {
+  name: "DownloadResponse";
+  status: number;
   startTime?: number;
   completeTime?: number;
 }
 
-export interface CpeRequest {
+export type CpeRequest =
-  name: string;
+  | InformRequest
-  fileType?: string;
+  | TransferCompleteRequest
-}
+  | GetRPCMethodsRequest
+  | RequestDownloadRequest;
 
-export interface InformRequest extends CpeRequest {
+export interface InformRequest {
   name: "Inform";
   deviceId: {
     Manufacturer: string;
@@ -270,10 +372,10 @@ export interface InformRequest extends CpeRequest {
   };
   event: string[];
   retryCount: number;
-  parameterList: [string, string | number | boolean, string][];
+  parameterList: [Path, string | number | boolean, string][];
 }
 
-export interface TransferCompleteRequest extends CpeRequest {
+export interface TransferCompleteRequest {
   name: "TransferComplete";
   commandKey?: string;
   faultStruct?: FaultStruct;
@@ -281,10 +383,36 @@ export interface TransferCompleteRequest extends CpeRequest {
   completeTime?: number;
 }
 
-export interface AcsResponse {
+export interface GetRPCMethodsRequest {
-  name: string;
+  name: "GetRPCMethods";
-  commandKey?: string;
+}
-  faultStruct?: FaultStruct;
+
+export interface RequestDownloadRequest {
+  name: "RequestDownload";
+  fileType: string;
+}
+
+export type AcsResponse =
+  | InformResponse
+  | GetRPCMethodsResponse
+  | TransferCompleteResponse
+  | RequestDownloadResponse;
+
+export interface InformResponse {
+  name: "InformResponse";
+}
+
+export interface GetRPCMethodsResponse {
+  name: "GetRPCMethodsResponse";
+  methodList: string[];
+}
+
+export interface TransferCompleteResponse {
+  name: "TransferCompleteResponse";
+}
+
+export interface RequestDownloadResponse {
+  name: "RequestDownloadResponse";
 }
 
 export interface QueryOptions {
@@ -299,12 +427,20 @@ export interface QueryOptions {
 export interface Declaration {
   path: Path;
   pathGet: number;
-  pathSet?: [number, number];
+  pathSet?: number | [number, number];
-  attrGet?: { object?: number; writable?: number; value?: number };
+  attrGet?: {
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  };
   attrSet?: {
     object?: boolean;
     writable?: boolean;
     value?: [string | number | boolean, string?];
+    notification?: number;
+    accessList?: string[];
   };
   defer: boolean;
 }
@@ -312,7 +448,13 @@ export interface Declaration {
 export type Clear = [
   Path,
   number,
-  { object?: number; writable?: number; value?: number }?,
+  {
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  }?,
   number?
 ];
 
@@ -321,8 +463,36 @@ export interface Preset {
   channel: string;
   schedule?: { md5: string; duration: number; schedule: any };
   events?: { [event: string]: boolean };
-  precondition?: {};
+  precondition?: Expression;
-  provisions: string[][];
+  provisions: [string, ...Expression[]][];
+}
+
+export interface Provisions {
+  [name: string]: { md5: string; script: Script };
+}
+
+export interface VirtualParameters {
+  [name: string]: { md5: string; script: Script };
+}
+
+export interface Files {
+  [name: string]: { length: number; md5: string; contentType: string };
+}
+
+export interface Users {
+  [name: string]: { password: string; salt: string; roles: string[] };
+}
+
+export interface Permissions {
+  [role: string]: {
+    [access: number]: {
+      [resource: string]: {
+        access: number;
+        filter: Expression;
+        validate?: Expression;
+      };
+    };
+  };
 }
 
 export type PermissionSet = {
@@ -333,6 +503,21 @@ export type PermissionSet = {
   };
 }[];
 
+export interface Config {
+  [name: string]: Expression;
+}
+
+export interface UiConfig {
+  filters: Record<string, unknown>;
+  device: Record<string, unknown>;
+  index: Record<string, unknown>;
+  overview: {
+    charts?: Record<string, unknown>;
+    groups?: Record<string, unknown>;
+  };
+  pageSize?: Expression;
+}
+
 export interface SoapMessage {
   id: string;
   cwmpVersion: string;

lib/ui.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Z_SYNC_FLUSH } from "zlib";
+import { constants } from "zlib";
 import Koa from "koa";
 import Router from "koa-router";
 import * as jwt from "jsonwebtoken";
@@ -31,9 +31,10 @@ import Authorizer from "./common/authorizer";
 import * as logger from "./logger";
 import * as localCache from "./local-cache";
 import { PermissionSet } from "./types";
-import { authSimple } from "./ui/api-functions";
+import { authLocal } from "./ui/api-functions";
 import * as init from "./init";
 import { version as VERSION } from "../package.json";
+import memoize from "./common/memoize";
 
 declare module "koa" {
   interface Request {
@@ -47,18 +48,21 @@ const router = new Router();
 const JWT_SECRET = "" + config.get("UI_JWT_SECRET");
 const JWT_COOKIE = "genieacs-ui-jwt";
 
-function getPermissionSets(ctx): PermissionSet[] {
+const getAuthorizer = memoize(
-  const allPermissions = localCache.getPermissions(ctx.state.configSnapshot);
+  (snapshot: string, rolesStr: string): Authorizer => {
-  const users = localCache.getUsers(ctx.state.configSnapshot);
+    const roles: string[] = JSON.parse(rolesStr);
-  const user = users[ctx.state.user.username];
+    const allPermissions = localCache.getPermissions(snapshot);
-  const permissionSets = user.roles.map(role =>
+    const permissionSets: PermissionSet[] = roles.map((r) =>
-    Object.values(allPermissions[role] || {})
+      Object.values(allPermissions[r] || {})
     );
-  return permissionSets;
+    return new Authorizer(permissionSets);
-}
+  }
+);
 
-koa.on("error", async err => {
+koa.on("error", async (err) => {
+  setTimeout(() => {
     throw err;
+  });
 });
 
 koa.use(async (ctx, next) => {
@@ -73,19 +77,43 @@ koa.use(
   koaJwt({
     secret: JWT_SECRET,
     passthrough: true,
-    cookie: JWT_COOKIE
+    cookie: JWT_COOKIE,
+    isRevoked: async (ctx, token) => {
+      if (token["authMethod"] === "local") {
+        return !localCache.getUsers(ctx.state.configSnapshot)[
+          token["username"]
+        ];
+      }
+
+      return true;
+    },
   })
 );
 
 koa.use(async (ctx, next) => {
-  if (ctx.state.user && ctx.state.user.username)
+  let roles: string[] = [];
-    ctx.state.authorizer = new Authorizer(getPermissionSets(ctx));
+
-  else ctx.state.authorizer = new Authorizer([]);
+  if (ctx.state.user && ctx.state.user.username) {
+    let user;
+    if (ctx.state.user.authMethod === "local") {
+      user = localCache.getUsers(ctx.state.configSnapshot)[
+        ctx.state.user.username
+      ];
+    } else {
+      throw new Error("Invalid auth method");
+    }
+    roles = user.roles || [];
+  }
+
+  ctx.state.authorizer = getAuthorizer(
+    ctx.state.configSnapshot,
+    JSON.stringify(roles)
+  );
 
   return next();
 });
 
-router.post("/login", async ctx => {
+router.post("/login", async (ctx) => {
   if (!JWT_SECRET) {
     ctx.status = 500;
     ctx.body = "UI_JWT_SECRET is not set";
@@ -100,12 +128,12 @@ router.post("/login", async ctx => {
     message: "Log in",
     context: ctx,
     username: username,
-    method: null
+    method: null,
   };
 
-  function success(method): void {
+  function success(authMethod): void {
-    log.method = method;
+    log.method = authMethod;
-    const token = jwt.sign({ username }, JWT_SECRET);
+    const token = jwt.sign({ username, authMethod }, JWT_SECRET);
     ctx.cookies.set(JWT_COOKIE, token, { sameSite: "lax" });
     ctx.body = JSON.stringify(token);
     logger.accessInfo(log);
@@ -118,20 +146,19 @@ router.post("/login", async ctx => {
     logger.accessWarn(log);
   }
 
-  const roles = await authSimple(ctx.state.configSnapshot, username, password);
+  if (await authLocal(ctx.state.configSnapshot, username, password))
-
+    return void success("local");
-  if (roles) return void success("simple");
 
   failure();
 });
 
-router.post("/logout", async ctx => {
+router.post("/logout", async (ctx) => {
   ctx.cookies.set(JWT_COOKIE); // Delete cookie
   ctx.body = "";
 
   logger.accessInfo({
     message: "Log out",
-    context: ctx
+    context: ctx,
   });
 });
 
@@ -145,11 +172,11 @@ koa.use(async (ctx, next) => {
 koa.use(koaBodyParser());
 router.use("/api", api.routes(), api.allowedMethods());
 
-router.get("/status", ctx => {
+router.get("/status", (ctx) => {
   ctx.body = "OK";
 });
 
-router.get("/init", async ctx => {
+router.get("/init", async (ctx) => {
   const status = await init.getStatus();
   if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
     if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
@@ -170,7 +197,7 @@ router.get("/init", async ctx => {
   ctx.body = status;
 });
 
-router.post("/init", async ctx => {
+router.post("/init", async (ctx) => {
   const status = ctx.request.body;
   if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
     if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
@@ -191,10 +218,8 @@ router.post("/init", async ctx => {
   ctx.body = "";
 });
 
-router.get("/", async ctx => {
+router.get("/", async (ctx) => {
-  let permissionSets = [];
+  const permissionSets: PermissionSet[] = ctx.state.authorizer.getPermissionSets();
-  if (ctx.state.user && ctx.state.user.username)
-    permissionSets = getPermissionSets(ctx);
 
   let wizard = "";
   if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
@@ -211,7 +236,7 @@ router.get("/", async ctx => {
     <noscript>GenieACS UI requires JavaScript to work. Please enable JavaScript in your browser.</noscript>
       <script>
         window.clientConfig = ${JSON.stringify({
-          ui: localCache.getUiConfig(ctx.state.configSnapshot)
+          ui: localCache.getUiConfig(ctx.state.configSnapshot),
         })};
         window.configSnapshot = ${JSON.stringify(ctx.state.configSnapshot)};
         window.genieacsVersion = ${JSON.stringify(VERSION)};
@@ -228,7 +253,12 @@ router.get("/", async ctx => {
 
 koa.use(
   koaCompress({
-    flush: Z_SYNC_FLUSH
+    gzip: {
+      flush: constants.Z_SYNC_FLUSH,
+    },
+    deflate: {
+      flush: constants.Z_SYNC_FLUSH,
+    },
   })
 );
 

lib/ui/api-functions.ts

@@ -17,7 +17,6 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { exec } from "child_process";
 import { ObjectID } from "mongodb";
 import * as db from "./db";
 import { del } from "../cache";
@@ -27,8 +26,9 @@ import {
   insertTasks,
   watchTask,
   connectionRequest,
-  deleteDevice
+  deleteDevice,
 } from "../api-functions";
+import { Task } from "../types";
 
 async function deleteFault(id): Promise<void> {
   const deviceId = id.split(":", 1)[0];
@@ -38,13 +38,16 @@ async function deleteFault(id): Promise<void> {
     db.deleteFault(id),
     channel.startsWith("task_")
       ? db.deleteTask(new ObjectID(channel.slice(5)))
-      : null
+      : null,
   ]);
 
   await del(`${deviceId}_tasks_faults_operations`);
 }
 
-export async function deleteResource(resource, id): Promise<void> {
+export async function deleteResource(
+  resource: string,
+  id: string
+): Promise<void> {
   switch (resource) {
     case "devices":
       await deleteDevice(id);
@@ -87,28 +90,30 @@ export async function deleteResource(resource, id): Promise<void> {
 }
 
 export async function postTasks(
-  deviceId,
+  deviceId: string,
-  tasks,
+  tasks: Task[],
-  timeout
+  timeout: number,
+  device: Record<string, { value?: [boolean | number | string, string] }>
 ): Promise<{ connectionRequest: string; tasks: any[] }> {
   for (const task of tasks) {
     delete task._id;
-    task.device = deviceId;
+    task["device"] = deviceId;
   }
 
   tasks = await insertTasks(tasks);
-  const statuses = tasks.map(t => {
+  const statuses = tasks.map((t) => {
     return { _id: t._id, status: "pending" };
   });
 
   await del(`${deviceId}_tasks_faults_operations`);
 
   try {
-    await connectionRequest(deviceId);
+    await connectionRequest(deviceId, device);
   } catch (err) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
     return {
       connectionRequest: err.message,
-      tasks: statuses
+      tasks: statuses,
     };
   }
 
@@ -131,48 +136,20 @@ export async function postTasks(
       r.status = "done";
     } else if (res[i * 2 + 1].length === 1) {
       r.status = "fault";
-      r.fault = res[i * 2 + 1][0];
+      r["fault"] = res[i * 2 + 1][0];
     }
-    db.deleteTask(r._id);
   }
 
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+
   return { connectionRequest: "OK", tasks: statuses };
 }
 
-interface PingResponse {
+export async function putResource(
-  packetsTransmitted: number;
+  resource: string,
-  packetsReceived: number;
+  id: string,
-  packetLoss: string;
+  data: Record<string, unknown>
-  min: number;
+): Promise<void> {
-  avg: number;
-  max: number;
-  mdev: number;
-}
-
-export function ping(host): Promise<PingResponse> {
-  return new Promise((resolve, reject) => {
-    exec(`ping -w 1 -i 0.2 -c 3 ${host}`, (err, stdout) => {
-      if (err) return void reject(err);
-
-      const m = stdout.match(
-        /(\d) packets transmitted, (\d) received, ([\d.%]+) packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/
-      );
-      if (!m) return void reject(new Error("Could not parse ping response"));
-
-      resolve({
-        packetsTransmitted: +m[1],
-        packetsReceived: +m[2],
-        packetLoss: m[3],
-        min: +m[4],
-        avg: +m[5],
-        max: +m[6],
-        mdev: +m[7]
-      });
-    });
-  });
-}
-
-export async function putResource(resource, id, data): Promise<void> {
   if (resource === "presets") {
     await db.putPreset(id, data);
   } else if (resource === "provisions") {
@@ -184,23 +161,27 @@ export async function putResource(resource, id, data): Promise<void> {
   } else if (resource === "permissions") {
     await db.putPermission(id, data);
   } else if (resource === "users") {
-    delete data.password;
+    delete data["password"];
-    delete data.salt;
+    delete data["salt"];
     await db.putUser(id, data);
   }
 
   await del("presets_hash");
 }
 
-export function authSimple(snapshot, username, password): Promise<string[]> {
+export function authLocal(
+  snapshot: string,
+  username: string,
+  password: string
+): Promise<boolean> {
   return new Promise((resolve, reject) => {
     const users = getUsers(snapshot);
     const user = users[username];
     if (!user || !user.password) return void resolve(null);
     hashPassword(password, user.salt)
-      .then(hash => {
+      .then((hash) => {
-        if (hash === user.password) resolve(user.roles);
+        if (hash === user.password) resolve(true);
-        else resolve(null);
+        else resolve(false);
       })
       .catch(reject);
   });

lib/ui/api.ts

@@ -28,6 +28,10 @@ import { getConfig } from "../local-cache";
 import { QueryOptions, Expression } from "../types";
 import { generateSalt, hashPassword } from "../auth";
 import { del } from "../cache";
+import Authorizer from "../common/authorizer";
+import { ping } from "../ping";
+import * as url from "url";
+import { decodeTag } from "../common";
 
 const router = new Router();
 export default router;
@@ -50,7 +54,7 @@ const RESOURCE_IDS = {
   permissions: "_id",
   users: "_id",
   faults: "_id",
-  tasks: "_id"
+  tasks: "_id",
 };
 
 const resources = {
@@ -63,21 +67,26 @@ const resources = {
   permissions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   users: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   faults: 0 | RESOURCE_DELETE,
-  tasks: 0
+  tasks: 0,
 };
 
-router.get(`/devices/:id.csv`, async ctx => {
+router.get(`/devices/:id.csv`, async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Query device (CSV)",
     context: ctx,
-    id: ctx.params.id
+    id: ctx.params.id,
   };
 
-  const filter = ["=", ["PARAM", RESOURCE_IDS.devices], ctx.params.id];
+  const filter = and(authorizer.getFilter("devices", 2), [
+    "=",
+    ["PARAM", RESOURCE_IDS.devices],
+    ctx.params.id,
+  ]);
 
-  if (!ctx.state.authorizer.hasAccess("devices", 2)) {
+  if (!authorizer.hasAccess("devices", 2)) {
     logUnauthorizedWarning(log);
-    return void (ctx.status = 404);
+    return void (ctx.status = 403);
   }
 
   const res = await db.query("devices", filter);
@@ -92,7 +101,7 @@ router.get(`/devices/:id.csv`, async ctx => {
 
   ctx.body = new stream.PassThrough();
   ctx.body.write(
-    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp\n"
+    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp,Notification,Notification timestamp,Access list,Access list timestamp\n"
   );
 
   for (const k of Object.keys(res[0]).sort()) {
@@ -105,36 +114,43 @@ router.get(`/devices/:id.csv`, async ctx => {
       p.writableTimestamp,
       p.value != null ? `"${p.value[0].toString().replace(/"/g, '""')}"` : "",
       p.value != null ? p.value[1] : "",
-      p.valueTimestamp
+      p.valueTimestamp,
+      p.notification,
+      p.notificationTimestamp,
+      p.accessList ? p.accessList.join(", ") : "",
+      p.accessListTimestamp,
     ];
-    ctx.body.write(row.map(r => (r != null ? r : "")).join(",") + "\n");
+    ctx.body.write(row.map((r) => (r != null ? r : "")).join(",") + "\n");
   }
   ctx.body.end();
   logger.accessInfo(log);
 });
 
 for (const [resource, flags] of Object.entries(resources)) {
-  router.head(`/${resource}`, async (ctx, next) => {
+  router.head(`/${resource}`, async (ctx) => {
-    let filter: Expression = true;
+    const authorizer: Authorizer = ctx.state.authorizer;
-    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+    let filter: Expression = authorizer.getFilter(resource, 1);
+    if (ctx.request.query.filter)
+      filter = and(filter, parse(ctx.request.query.filter));
 
     const log = {
       message: `Count ${resource}`,
       context: ctx,
       filter: ctx.request.query.filter,
-      count: null
+      count: null,
     };
 
-    if (!ctx.state.authorizer.hasAccess(resource, 1)) {
+    if (!authorizer.hasAccess(resource, 1)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
+        "OR",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -147,10 +163,12 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.get(`/${resource}`, async (ctx, next) => {
+  router.get(`/${resource}`, async (ctx) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = {};
-    let filter: Expression = true;
+    let filter: Expression = authorizer.getFilter(resource, 2);
-    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+    if (ctx.request.query.filter)
+      filter = and(filter, parse(ctx.request.query.filter));
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
@@ -168,19 +186,20 @@ for (const [resource, flags] of Object.entries(resources)) {
       limit: options.limit,
       skip: options.skip,
       sort: options.sort,
-      projection: options.projection
+      projection: options.projection,
     };
 
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
+        "OR",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -189,7 +208,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     let c = 0;
     ctx.body.write("[\n");
-    await db.query(resource, filter, options, obj => {
+    await db.query(resource, filter, options, (obj) => {
       ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
     });
     ctx.body.end("]");
@@ -198,10 +217,12 @@ for (const [resource, flags] of Object.entries(resources)) {
   });
 
   // CSV download
-  router.get(`/${resource}.csv`, async (ctx, next) => {
+  router.get(`/${resource}.csv`, async (ctx) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = { projection: {} };
-    let filter: Expression = true;
+    let filter: Expression = authorizer.getFilter(resource, 2);
-    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+    if (ctx.request.query.filter)
+      filter = and(filter, parse(ctx.request.query.filter));
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
@@ -213,15 +234,17 @@ for (const [resource, flags] of Object.entries(resources)) {
       filter: ctx.request.query.filter,
       limit: options.limit,
       skip: options.skip,
-      sort: options.sort
+      sort: options.sort,
     };
 
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
-    const columns = JSON.parse(ctx.request.query.columns);
+    const columns: Record<string, Expression> = JSON.parse(
+      ctx.request.query.columns
+    );
     const now = Date.now();
 
     for (const [k, v] of Object.entries(columns)) {
@@ -234,8 +257,9 @@ for (const [resource, flags] of Object.entries(resources)) {
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
+        "OR",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000]
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -246,18 +270,18 @@ for (const [resource, flags] of Object.entries(resources)) {
     );
 
     ctx.body.write(
-      Object.keys(columns).map(k => `"${k.replace(/"/, '""')}"`) + "\n"
+      Object.keys(columns).map((k) => `"${k.replace(/"/, '""')}"`) + "\n"
     );
-    await db.query(resource, filter, options, obj => {
+    await db.query(resource, filter, options, (obj) => {
-      const arr = Object.values(columns).map(exp => {
+      const arr = Object.values(columns).map((exp) => {
-        const v = evaluate(exp, obj, null, e => {
+        const v = evaluate(exp, obj, null, (e) => {
           if (Array.isArray(e)) {
             if (e[0] === "PARAM") {
               if (resource === "devices") {
                 if (e[1] === "Tags") {
                   const tags = [];
                   for (const p in obj)
-                    if (p.startsWith("Tags.")) tags.push(p.slice(5));
+                    if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
 
                   return tags.join(", ");
                 }
@@ -284,65 +308,78 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.head(`/${resource}/:id`, async (ctx, next) => {
+  router.head(`/${resource}/:id`, async (ctx) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Count ${resource}`,
       context: ctx,
-      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`,
     };
 
-    const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
+    const filter = and(authorizer.getFilter(resource, 2), [
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+      "=",
+      ["PARAM", RESOURCE_IDS[resource]],
+      ctx.params.id,
+    ]);
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void next();
+    if (!res.length) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = "";
   });
 
-  router.get(`/${resource}/:id`, async (ctx, next) => {
+  router.get(`/${resource}/:id`, async (ctx) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Query ${resource}`,
       context: ctx,
-      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`,
     };
 
-    const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
+    const filter = and(authorizer.getFilter(resource, 2), [
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+      "=",
+      ["PARAM", RESOURCE_IDS[resource]],
+      ctx.params.id,
+    ]);
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void next();
+    if (!res.length) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = res[0];
   });
 
-  // TODO add PUT, PATCH routes
   if (flags & RESOURCE_DELETE) {
-    router.delete(`/${resource}/:id`, async (ctx, next) => {
+    router.delete(`/${resource}/:id`, async (ctx) => {
+      const authorizer: Authorizer = ctx.state.authorizer;
       const log = {
         message: `Delete ${resource}`,
         context: ctx,
-        id: ctx.params.id
+        id: ctx.params.id,
       };
 
-      const authorizer = ctx.state.authorizer;
+      const filter = and(authorizer.getFilter(resource, 3), [
-      const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
+        "=",
+        ["PARAM", RESOURCE_IDS[resource]],
+        ctx.params.id,
+      ]);
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
       const res = await db.query(resource, filter);
-      if (!res.length) return void next();
+      if (!res.length) return void (ctx.status = 404);
 
       const validate = authorizer.getValidator(resource, res[0]);
       if (!validate("delete")) {
@@ -359,19 +396,19 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 
   if (flags & RESOURCE_PUT) {
-    router.put(`/${resource}/:id`, async (ctx, next) => {
+    router.put(`/${resource}/:id`, async (ctx) => {
+      const authorizer: Authorizer = ctx.state.authorizer;
       const id = ctx.params.id;
 
       const log = {
         message: `Put ${resource}`,
         context: ctx,
-        id: id
+        id: id,
       };
 
-      const authorizer = ctx.state.authorizer;
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
 
       const obj = ctx.request.body;
@@ -398,7 +435,8 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 }
 
-router.put("/files/:id", async (ctx, next) => {
+router.put("/files/:id", async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const resource = "files";
   const id = ctx.params.id;
 
@@ -406,20 +444,19 @@ router.put("/files/:id", async (ctx, next) => {
     message: `Upload ${resource}`,
     context: ctx,
     id: id,
-    metadata: null
+    metadata: null,
   };
 
-  const authorizer = ctx.state.authorizer;
   if (!authorizer.hasAccess(resource, 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
   const metadata = {
     fileType: ctx.request.headers["metadata.filetype"] || "",
     oui: ctx.request.headers["metadata.oui"] || "",
     productClass: ctx.request.headers["metadata.productclass"] || "",
-    version: ctx.request.headers["metadata.version"] || ""
+    version: ctx.request.headers["metadata.version"] || "",
   };
 
   const validate = authorizer.getValidator(resource, metadata);
@@ -428,6 +465,12 @@ router.put("/files/:id", async (ctx, next) => {
     return void (ctx.status = 403);
   }
 
+  try {
+    await db.deleteFile(id);
+  } catch (err) {
+    // File doesn't exist, ignore
+  }
+
   await db.putFile(id, metadata, ctx.req);
   log.metadata = metadata;
   logger.accessInfo(log);
@@ -435,24 +478,29 @@ router.put("/files/:id", async (ctx, next) => {
   ctx.body = "";
 });
 
-router.post("/devices/:id/tasks", async (ctx, next) => {
+router.post("/devices/:id/tasks", async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Commit tasks",
     context: ctx,
     deviceId: ctx.params.id,
-    tasks: null
+    tasks: null,
   };
 
-  const authorizer = ctx.state.authorizer;
+  const filter = and(authorizer.getFilter("devices", 3), [
-  const filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
+    "=",
+    ["PARAM", "DeviceID.ID"],
+    ctx.params.id,
+  ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
   const devices = await db.query("devices", filter);
-  if (!devices.length) return void next();
+  if (!devices.length) return void (ctx.status = 404);
+  const device = devices[0];
 
-  const validate = authorizer.getValidator("devices", devices[0]);
+  const validate = authorizer.getValidator("devices", device);
   for (const t of ctx.request.body) {
     if (!validate("task", t)) {
       logUnauthorizedWarning(log);
@@ -460,19 +508,37 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
     }
   }
 
-  // TODO Pass context and timestamp
   const onlineThreshold = getConfig(
     ctx.state.configSnapshot,
-    "cwmp.deviceOnlineThreshold"
+    "cwmp.deviceOnlineThreshold",
-  );
+    {},
+    Date.now(),
+    (exp) => {
+      if (!Array.isArray(exp)) return exp;
+      if (exp[0] === "PARAM") {
+        const p = device[exp[1]];
+        if (p && p.value) return p.value[0];
+      } else if (exp[0] === "FUNC") {
+        if (exp[1] === "REMOTE_ADDRESS") {
+          for (const root of ["InternetGatewayDevice", "Device"]) {
+            const p = device[`${root}.ManagementServer.ConnectionRequestURL`];
+            if (p && p.value) return url.parse(p.value[0]).host;
+          }
+          return null;
+        }
+      }
+      return exp;
+    }
+  ) as number;
 
   const res = await apiFunctions.postTasks(
     ctx.params.id,
     ctx.request.body,
-    onlineThreshold
+    onlineThreshold,
+    device
   );
 
-  log.tasks = res.tasks.map(t => t._id).join(",");
+  log.tasks = res.tasks.map((t) => t._id).join(",");
 
   logger.accessInfo(log);
 
@@ -480,22 +546,26 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
   ctx.body = res.tasks;
 });
 
-router.post("/devices/:id/tags", async (ctx, next) => {
+router.post("/devices/:id/tags", async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Update tags",
     context: ctx,
     deviceId: ctx.params.id,
-    tags: ctx.request.body
+    tags: ctx.request.body,
   };
 
-  const authorizer = ctx.state.authorizer;
+  const filter = and(authorizer.getFilter("devices", 3), [
-  const filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
+    "=",
+    ["PARAM", "DeviceID.ID"],
+    ctx.params.id,
+  ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
   const res = await db.query("devices", filter);
-  if (!res.length) return void next();
+  if (!res.length) return void (ctx.status = 404);
 
   const validate = authorizer.getValidator("devices", res[0]);
   if (!validate("tags", ctx.request.body)) {
@@ -517,30 +587,33 @@ router.post("/devices/:id/tags", async (ctx, next) => {
   ctx.body = "";
 });
 
-router.get("/ping/:host", async ctx => {
+router.get("/ping/:host", async (ctx) => {
-  try {
+  return new Promise((resolve) => {
-    ctx.body = await apiFunctions.ping(ctx.params.host);
+    ping(ctx.params.host, (err, parsed) => {
-  } catch (err) {
+      if (parsed) {
-    ctx.status = 404;
+        ctx.body = parsed;
-    ctx.response.headers["Cache-Control"] = "no-cache";
+      } else {
+        ctx.status = 500;
         ctx.body = `${err.name}: ${err.message}`;
       }
+      resolve();
+    });
+  });
 });
 
-router.put("/users/:id/password", async (ctx, next) => {
+router.put("/users/:id/password", async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const username = ctx.params.id;
   const log = {
     message: "Change password",
     context: ctx,
-    username: username
+    username: username,
   };
 
-  const authorizer = ctx.state.authorizer;
-
   if (!ctx.state.user) {
     // User not logged in
     if (
-      !(await apiFunctions.authSimple(
+      !(await apiFunctions.authLocal(
         ctx.state.configSnapshot,
         username,
         ctx.request.body.authPassword
@@ -553,12 +626,16 @@ router.put("/users/:id/password", async (ctx, next) => {
     }
   } else if (!authorizer.hasAccess("users", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
-  const filter = ["=", ["PARAM", RESOURCE_IDS.users], username];
+  const filter = and(authorizer.getFilter("users", 3), [
+    "=",
+    ["PARAM", RESOURCE_IDS.users],
+    username,
+  ]);
   const res = await db.query("users", filter);
-  if (!res.length) return void next();
+  if (!res.length) return void (ctx.status = 404);
 
   const newPassword = ctx.request.body.newPassword;
   if (ctx.state.user) {

lib/ui/db.ts

@@ -17,19 +17,21 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, GridFSBucket } from "mongodb";
+import { MongoClient, GridFSBucket, ObjectID } from "mongodb";
 import { Script } from "vm";
 import * as config from "../config";
 import * as mongodbFunctions from "../mongodb-functions";
 import * as expression from "../common/expression";
-import { QueryOptions } from "../types";
+import { QueryOptions, Expression } from "../types";
+import { Readable } from "stream";
+import { minimize } from "../common/boolean-expression";
 
 const CACHE_TTL = 300000;
 
-let _clientPromise = null;
+let clientPromise: Promise<MongoClient>;
 
 const RESOURCE_COLLECTION = {
-  files: "fs.files"
+  files: "fs.files",
 };
 
 function ensureIndexes(client): void {
@@ -40,12 +42,12 @@ function ensureIndexes(client): void {
 }
 
 function getClient(): Promise<MongoClient> {
-  if (!_clientPromise) {
+  if (!clientPromise) {
-    _clientPromise = new Promise((resolve, reject) => {
+    clientPromise = new Promise((resolve, reject) => {
       const CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
       MongoClient.connect(
         CONNECTION_URL,
-        { useNewUrlParser: true },
+        { useNewUrlParser: true, useUnifiedTopology: true },
         (err, client) => {
           if (err) return void reject(err);
           ensureIndexes(client);
@@ -55,30 +57,34 @@ function getClient(): Promise<MongoClient> {
     });
   }
 
-  return _clientPromise;
+  return clientPromise;
 }
 
-export function cache<T>(key, valueGetter: () => Promise<T>, ttl): Promise<T> {
+export function cache<T>(
+  key: string,
+  valueGetter: () => Promise<T>,
+  ttl: number
+): Promise<T> {
   return new Promise((resolve, reject) => {
     getClient()
-      .then(client => {
+      .then((client) => {
         const collection = client.db().collection("cache");
         collection.findOne({ _id: key }, (err, doc) => {
           if (err) return void reject(err);
           if (doc != null) return void resolve(JSON.parse(doc.value) as T);
           valueGetter()
-            .then(res => {
+            .then((res) => {
               const expire = Date.now() + (ttl || CACHE_TTL);
               const cacheDoc = {
                 _id: key,
                 value: JSON.stringify(res),
-                expire: new Date(expire)
+                expire: new Date(expire),
               };
               collection.updateOne(
                 { _id: key },
                 { $set: cacheDoc },
                 { upsert: true },
-                err => {
+                (err) => {
                   if (err) reject(err);
                   else resolve(res);
                 }
@@ -92,14 +98,15 @@ export function cache<T>(key, valueGetter: () => Promise<T>, ttl): Promise<T> {
 }
 
 export function query(
-  resource,
+  resource: string,
-  filter,
+  filter: Expression,
   options?: QueryOptions,
   callback?: (doc: any) => void
 ): Promise<any[]> {
   options = options || {};
   let q;
   filter = expression.evaluate(filter, null, Date.now());
+  filter = minimize(filter, true);
 
   if (Array.isArray(filter)) {
     if (resource === "devices") {
@@ -121,7 +128,8 @@ export function query(
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then(client => {
+    getClient()
+      .then((client) => {
         const collection = client
           .db()
           .collection(RESOURCE_COLLECTION[resource] || resource);
@@ -160,20 +168,20 @@ export function query(
           cursor.toArray((err, docs) => {
             if (err) return reject(err);
             if (resource === "devices")
-            docs = docs.map(d => mongodbFunctions.flattenDevice(d));
+              docs = docs.map((d) => mongodbFunctions.flattenDevice(d));
             else if (resource === "faults")
-            docs = docs.map(d => mongodbFunctions.flattenFault(d));
+              docs = docs.map((d) => mongodbFunctions.flattenFault(d));
             else if (resource === "tasks")
-            docs = docs.map(d => mongodbFunctions.flattenTask(d));
+              docs = docs.map((d) => mongodbFunctions.flattenTask(d));
             else if (resource === "presets")
-            docs = docs.map(d => mongodbFunctions.flattenPreset(d));
+              docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
             else if (resource === "files")
-            docs = docs.map(d => mongodbFunctions.flattenFile(d));
+              docs = docs.map((d) => mongodbFunctions.flattenFile(d));
             return resolve(docs);
           });
         } else {
           cursor.forEach(
-          doc => {
+            (doc) => {
               if (resource === "devices")
                 doc = mongodbFunctions.flattenDevice(doc);
               else if (resource === "faults")
@@ -186,19 +194,21 @@ export function query(
                 doc = mongodbFunctions.flattenFile(doc);
               callback(doc);
             },
-          err => {
+            (err) => {
               if (err) reject(err);
               else resolve();
             }
           );
         }
-    });
+      })
+      .catch(reject);
   });
 }
 
-export function count(resource, filter): Promise<number> {
+export function count(resource: string, filter: Expression): Promise<number> {
   let q;
   filter = expression.evaluate(filter, null, Date.now());
+  filter = minimize(filter, true);
 
   if (Array.isArray(filter)) {
     if (resource === "devices")
@@ -213,7 +223,8 @@ export function count(resource, filter): Promise<number> {
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then(client => {
+    getClient()
+      .then((client) => {
         const collection = client
           .db()
           .collection(RESOURCE_COLLECTION[resource] || resource);
@@ -221,50 +232,42 @@ export function count(resource, filter): Promise<number> {
           if (err) reject(err);
           else resolve(c);
         });
-    });
-  });
-}
-
-export function updateDeviceTags(deviceId, tags): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const add = [];
-    const pull = [];
-
-    const regex = /^[0-9a-zA-Z_]+$/;
-    for (let [tag, onOff] of Object.entries(tags)) {
-      tag = tag.trim();
-      if (onOff) {
-        if (!tag.match(regex))
-          return void reject(new Error(`Invalid tag '${tag}'`));
-        add.push(tag);
-      } else {
-        pull.push(tag);
-      }
-    }
-    getClient()
-      .then(client => {
-        const collection = client.db().collection("devices");
-
-        const object = {};
-        if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
-        if (pull && pull.length) object["$pullAll"] = { _tags: pull };
-        collection.updateOne({ _id: deviceId }, object, err => {
-          if (err) return void reject(err);
-          resolve();
-        });
       })
       .catch(reject);
   });
 }
 
+export async function updateDeviceTags(
+  deviceId: string,
+  tags: Record<string, boolean>
+): Promise<void> {
+  const add = [];
+  const pull = [];
+
+  for (let [tag, onOff] of Object.entries(tags)) {
+    tag = tag.trim();
+    if (onOff) add.push(tag);
+    else pull.push(tag);
+  }
+
+  const client = await getClient();
+  const collection = client.db().collection("devices");
+  const object = {};
+
+  if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
+  if (pull && pull.length) object["$pullAll"] = { _tags: pull };
+
+  await collection.updateOne({ _id: deviceId }, object);
+}
+
 function putResource(resource, id, object): Promise<void> {
   return new Promise((resolve, reject) => {
     getClient()
-      .then(client => {
+      .then((client) => {
         const collection = client
           .db()
           .collection(RESOURCE_COLLECTION[resource] || resource);
-        collection.replaceOne({ _id: id }, object, { upsert: true }, err => {
+        collection.replaceOne({ _id: id }, object, { upsert: true }, (err) => {
           if (err) return void reject(err);
           resolve();
         });
@@ -273,14 +276,17 @@ function putResource(resource, id, object): Promise<void> {
   });
 }
 
-function deleteResource(resource, id): Promise<void> {
+function deleteResource(
+  resource: string,
+  id: string | ObjectID
+): Promise<void> {
   return new Promise((resolve, reject) => {
     getClient()
-      .then(client => {
+      .then((client) => {
         const collection = client
           .db()
           .collection(RESOURCE_COLLECTION[resource] || resource);
-        collection.deleteOne({ _id: id }, err => {
+        collection.deleteOne({ _id: id }, (err) => {
           if (err) return void reject(err);
           resolve();
         });
@@ -289,16 +295,22 @@ function deleteResource(resource, id): Promise<void> {
   });
 }
 
-export function putPreset(id, object): Promise<void> {
+export function putPreset(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   object = mongodbFunctions.preProcessPreset(object);
   return putResource("presets", id, object);
 }
 
-export function deletePreset(id): Promise<void> {
+export function deletePreset(id: string): Promise<void> {
   return deleteResource("presets", id);
 }
 
-export function putProvision(id, object): Promise<void> {
+export function putProvision(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   if (!object.script) object.script = "";
   try {
     new Script(`"use strict";(function(){\n${object.script}\n})();`);
@@ -308,11 +320,14 @@ export function putProvision(id, object): Promise<void> {
   return putResource("provisions", id, object);
 }
 
-export function deleteProvision(id): Promise<void> {
+export function deleteProvision(id: string): Promise<void> {
   return deleteResource("provisions", id);
 }
 
-export function putVirtualParameter(id, object): Promise<void> {
+export function putVirtualParameter(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   if (!object.script) object.script = "";
   try {
     new Script(`"use strict";(function(){\n${object.script}\n})();`);
@@ -322,37 +337,46 @@ export function putVirtualParameter(id, object): Promise<void> {
   return putResource("virtualParameters", id, object);
 }
 
-export function deleteVirtualParameter(id): Promise<void> {
+export function deleteVirtualParameter(id: string): Promise<void> {
   return deleteResource("virtualParameters", id);
 }
 
-export function putConfig(id, object): Promise<void> {
+export function putConfig(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   return putResource("config", id, object);
 }
 
-export function deleteConfig(id): Promise<void> {
+export function deleteConfig(id: string): Promise<void> {
   return deleteResource("config", id);
 }
 
-export function putPermission(id, object): Promise<void> {
+export function putPermission(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   return putResource("permissions", id, object);
 }
 
-export function deletePermission(id): Promise<void> {
+export function deletePermission(id: string): Promise<void> {
   return deleteResource("permissions", id);
 }
 
-export function putUser(id, object): Promise<void> {
+export function putUser(
+  id: string,
+  object: Record<string, unknown>
+): Promise<void> {
   return new Promise((resolve, reject) => {
     getClient()
-      .then(client => {
+      .then((client) => {
         const collection = client.db().collection("users");
         // update instead of replace to keep the password if not set by user
         collection.updateOne(
           { _id: id },
           { $set: object },
           { upsert: true },
-          err => {
+          (err) => {
             if (err) return void reject(err);
             resolve();
           }
@@ -362,17 +386,21 @@ export function putUser(id, object): Promise<void> {
   });
 }
 
-export function deleteUser(id): Promise<void> {
+export function deleteUser(id: string): Promise<void> {
   return deleteResource("users", id);
 }
 
-export function putFile(filename, metadata, contentStream): Promise<void> {
+export function putFile(
+  filename: string,
+  metadata: Record<string, string>,
+  contentStream: Readable
+): Promise<void> {
   return new Promise((resolve, reject) => {
     getClient()
-      .then(client => {
+      .then((client) => {
         const bucket = new GridFSBucket(client.db());
         const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
-          metadata: metadata
+          metadata: metadata,
         });
         uploadStream.on("error", reject);
         contentStream.on("error", reject);
@@ -383,12 +411,12 @@ export function putFile(filename, metadata, contentStream): Promise<void> {
   });
 }
 
-export function deleteFile(filename): Promise<void> {
+export function deleteFile(filename: string): Promise<void> {
   return new Promise((resolve, reject) => {
     getClient()
-      .then(client => {
+      .then((client) => {
         const bucket = new GridFSBucket(client.db());
-        bucket.delete(filename, err => {
+        bucket.delete(filename as any, (err) => {
           if (err) return void reject(err);
           resolve();
         });
@@ -397,18 +425,14 @@ export function deleteFile(filename): Promise<void> {
   });
 }
 
-export function deleteFault(id): Promise<void> {
+export function deleteFault(id: string): Promise<void> {
   return deleteResource("faults", id);
 }
 
-export function deleteTask(id): Promise<void> {
+export function deleteTask(id: ObjectID): Promise<void> {
   return deleteResource("tasks", id);
 }
 
-export function disconnect(): void {
+export async function disconnect(): Promise<void> {
-  if (_clientPromise) {
+  if (clientPromise) await (await clientPromise).close();
-    _clientPromise.then(client => {
-      client.close();
-    });
-  }
 }

lib/versioned-map.ts

@@ -46,7 +46,7 @@ export default class VersionedMap<K, V> {
     return this._revision;
   }
 
-  public set revision(rev) {
+  public set revision(rev: number) {
     for (let i = this._sizeDiff.length; i <= rev; ++i)
       this._sizeDiff[i] = this._sizeDiff[i - 1];
 
@@ -193,7 +193,7 @@ export default class VersionedMap<K, V> {
       const last = v[l];
       v.splice(revision, l - revision);
 
-      if (last === NONEXISTENT && !v.some(vv => vv !== NONEXISTENT))
+      if (last === NONEXISTENT && !v.some((vv) => vv !== NONEXISTENT))
         this.map.delete(k);
     }
   }

lib/xml-parser.ts

@@ -53,7 +53,8 @@ export interface Element {
 }
 
 export function parseXmlDeclaration(buffer: Buffer): Attribute[] {
-  for (const enc of ["utf16le", "utf8", "latin1", "ascii"]) {
+  const encodings: BufferEncoding[] = ["utf16le", "utf8", "latin1", "ascii"];
+  for (const enc of encodings) {
     let str = buffer.toString(enc, 0, 150);
     if (str.startsWith("<?xml")) {
       str = str.split("\n")[0].trim();
@@ -90,7 +91,7 @@ export function parseAttrs(string: string): Attribute[] {
               name: name,
               namespace: namespace,
               localName: localName,
-              value: value
+              value: value,
             };
             attrs.push(e);
             name = "";
@@ -128,8 +129,8 @@ export function parseAttrs(string: string): Attribute[] {
   return attrs;
 }
 
-export function decodeEntities(string): string {
+export function decodeEntities(string: string): string {
-  return string.replace(/&[0-9a-z#]+;/gi, match => {
+  return string.replace(/&[0-9a-z#]+;/gi, (match) => {
     switch (match) {
       case "&quot;":
         return '"';
@@ -161,15 +162,15 @@ export function decodeEntities(string): string {
   });
 }
 
-export function encodeEntities(string): string {
+export function encodeEntities(string: string): string {
   const entities = {
     "&": "&amp;",
     '"': "&quot;",
     "'": "&apos;",
     "<": "&lt;",
-    ">": "&gt;"
+    ">": "&gt;",
   };
-  return string.replace(/[&"'<>]/g, m => entities[m]);
+  return string.replace(/[&"'<>]/g, (m) => entities[m]);
 }
 
 export function parseXml(string: string): Element {
@@ -186,7 +187,7 @@ export function parseXml(string: string): Element {
     attrs: "",
     text: "",
     bodyIndex: 0,
-    children: []
+    children: [],
   };
 
   const stack: Element[] = [root];
@@ -330,7 +331,7 @@ export function parseXml(string: string): Element {
                   : "",
                 text: "",
                 bodyIndex: i + 1,
-                children: []
+                children: [],
               };
               parent.children.push(e);
               if (!selfClosing) stack.push(e);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.0-beta.0",
+  "version": "1.2.3",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -20,68 +20,69 @@
     "tools": "tools"
   },
   "dependencies": {
+    "espresso-iisojs": "^1.0.3",
+    "iconv-lite": "^0.6.2",
+    "ipaddr.js": "^2.0.0",
     "jsonwebtoken": "^8.5.1",
-    "koa": "^2.7.0",
+    "koa": "^2.13.0",
-    "koa-bodyparser": "^4.2.1",
+    "koa-bodyparser": "^4.3.0",
-    "koa-compress": "^3.0.0",
+    "koa-compress": "^5.0.1",
-    "koa-jwt": "^3.6.0",
+    "koa-jwt": "^4.0.0",
-    "koa-router": "^7.4.0",
+    "koa-router": "^9.4.0",
     "koa-static": "^5.0.0",
     "later": "^1.2.0",
-    "mongodb": "^3.2.6",
+    "mongodb": "^3.6.2",
-    "parsimmon": "^1.13.0",
+    "parsimmon": "^1.16.0",
-    "seedrandom": "^3.0.1",
+    "seedrandom": "^3.0.5",
-    "yaml": "^1.6.0"
+    "yaml": "^1.10.0"
-  },
-  "optionalDependencies": {
-    "libxmljs": "^0.19.5"
   },
   "devDependencies": {
-    "@types/jsonwebtoken": "^8.3.2",
+    "@rollup/plugin-commonjs": "^16.0.0",
-    "@types/koa": "^2.0.49",
+    "@rollup/plugin-json": "^4.1.0",
-    "@types/koa-compress": "^2.0.9",
+    "@rollup/plugin-node-resolve": "^10.0.0",
-    "@types/koa-router": "^7.0.42",
+    "@rollup/plugin-typescript": "^6.0.0",
+    "@types/jsonwebtoken": "^8.5.0",
+    "@types/koa": "^2.11.6",
+    "@types/koa-compress": "^4.0.1",
+    "@types/koa-router": "^7.4.1",
     "@types/koa-static": "^4.0.1",
-    "@types/libxmljs": "^0.18.3",
+    "@types/mithril": "^2.0.5",
-    "@types/mithril": "^2.0.0",
+    "@types/mongodb": "^3.5.31",
-    "@types/mongodb": "^3.1.31",
+    "@types/node": "^14.14.5",
-    "@types/parsimmon": "^1.10.0",
+    "@types/parsimmon": "^1.10.3",
     "@types/seedrandom": "^2.4.28",
-    "@types/svgo": "^1.2.1",
+    "@types/svgo": "^1.3.3",
-    "@typescript-eslint/eslint-plugin": "^1.13.0",
+    "@typescript-eslint/eslint-plugin": "^4.6.0",
-    "@typescript-eslint/parser": "^1.13.0",
+    "@typescript-eslint/parser": "^4.6.0",
-    "ava": "^2.2.0",
+    "ava": "^3.13.0",
-    "codemirror": "^5.48.2",
+    "codemirror": "^5.58.2",
     "cssnano": "^4.1.10",
-    "eslint": "^6.1.0",
+    "eslint": "^7.12.1",
-    "eslint-config-prettier": "^6.0.0",
+    "eslint-config-prettier": "^6.14.0",
     "esm": "^3.2.25",
-    "mithril": "^2.0.3",
+    "jsbi": "^3.1.4",
-    "postcss": "^7.0.17",
+    "mithril": "^2.0.4",
-    "postcss-import": "^12.0.1",
+    "postcss": "^8.1.4",
+    "postcss-import": "^13.0.0",
     "postcss-preset-env": "^6.7.0",
-    "prettier": "^1.18.2",
+    "prettier": "^2.1.2",
-    "rollup": "^1.17.0",
+    "rollup": "^2.32.1",
-    "rollup-plugin-json": "^4.0.0",
+    "rollup-plugin-terser": "^7.0.2",
-    "rollup-plugin-replace": "^2.2.0",
+    "source-map-loader": "^1.1.1",
-    "rollup-plugin-terser": "^5.1.1",
+    "svgo": "^1.3.2",
-    "rollup-plugin-typescript": "^1.0.1",
+    "ts-node": "^9.0.0",
-    "source-map-loader": "^0.2.4",
+    "typescript": "^4.0.5",
-    "svgo": "^1.3.0",
+    "webpack": "^5.2.0"
-    "ts-node": "^8.3.0",
-    "typescript": "^3.5.3",
-    "webpack": "^4.38.0"
   },
   "engines": {
-    "node": ">=8.14.0"
+    "node": ">=12.13.0"
   },
   "scripts": {
     "test": "ava --verbose --color",
-    "lint-ln": "find bin -type f -exec sh -c 'ln -fs \"$(sh -c pwd)/$1\" \"bin/.${1#bin/}.ts\"' _ {} \\;; find tools -type f -exec sh -c 'ln -fs \"$(sh -c pwd)/$1\" \"tools/.${1#tools/}.ts\"' _ {} \\;",
+    "lint-eslint": "eslint --fix \"bin/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/*.ts\"",
-    "lint-eslint": "eslint --fix \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
     "lint-prettier": "prettier --parser typescript --write \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
     "lint-tsc": "tsc --noEmit",
-    "lint": "npm run lint-ln && npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
+    "lint": "npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
     "build": "node -r esm -r ts-node/register/transpile-only build/build"
   }
 }

test/auth.ts

@@ -2,7 +2,7 @@ import ava from "ava";
 import { randomBytes } from "crypto";
 import * as auth from "../lib/auth";
 
-ava("digest", t => {
+ava("digest", (t) => {
   const username = "test";
   const password = "test";
   const uri = "/";
@@ -14,7 +14,7 @@ ava("digest", t => {
   const challenges = [
     `Digest realm="${realm}",nonce="${nonce}"`,
     `Digest realm="${realm}",nonce="${nonce}",qop="auth"`,
-    `Digest realm="${realm}",nonce="${nonce}",qop="auth-int"`
+    `Digest realm="${realm}",nonce="${nonce}",qop="auth-int"`,
   ];
 
   t.plan(challenges.length);

test/common.ts

@@ -1,35 +1,7 @@
 import ava from "ava";
 import * as common from "../lib/common";
 
-ava("typeOf", t => {
+ava("generateDeviceId", (t) => {
-  const cases = [
-    [undefined, common.UNDEFINED_TYPE],
-    [null, common.NULL_TYPE],
-    [true, common.BOOLEAN_TYPE],
-    [false, common.BOOLEAN_TYPE],
-    [0, common.NUMBER_TYPE],
-    [1, common.NUMBER_TYPE],
-    [-1, common.NUMBER_TYPE],
-    [0.0, common.NUMBER_TYPE],
-    [1.0, common.NUMBER_TYPE],
-    [1.1, common.NUMBER_TYPE],
-    [-1.0, common.NUMBER_TYPE],
-    [-1.1, common.NUMBER_TYPE],
-    ["", common.STRING_TYPE],
-    ["TEST", common.STRING_TYPE],
-    [[], common.ARRAY_TYPE],
-    [[1, 2, 3], common.ARRAY_TYPE],
-    [{}, common.OBJECT_TYPE],
-    [{ test: 1 }, common.OBJECT_TYPE],
-    [/[^A-Za-z0-9_]/g, common.REGEXP_TYPE],
-    [new RegExp("//"), common.REGEXP_TYPE],
-    [new Date(), common.DATE_TYPE]
-  ];
-  t.plan(cases.length);
-  for (const c of cases) t.is(common.typeOf(c[0]), c[1]);
-});
-
-ava("generateDeviceId", t => {
   const space = [" ", "%20"];
   const special = [";", "%3B"];
   const cases = [
@@ -37,30 +9,31 @@ ava("generateDeviceId", t => {
       {
         ProductClass: "TestProductClass",
         OUI: "TestOUI",
-        SerialNumber: "TestSerialNumber"
+        SerialNumber: "TestSerialNumber",
       },
-      "TestOUI-TestProductClass-TestSerialNumber"
+      "TestOUI-TestProductClass-TestSerialNumber",
     ],
     [
       {
         OUI: "TestOUI",
-        SerialNumber: "TestSerialNumber"
+        SerialNumber: "TestSerialNumber",
       },
-      "TestOUI-TestSerialNumber"
+      "TestOUI-TestSerialNumber",
     ],
     [
       {
         OUI: `TestOUIWith${space[0]}_${special[0]}2912`,
-        SerialNumber: `TestSerialNumberWith${space[0]}_${special[0]}2912`
+        SerialNumber: `TestSerialNumberWith${space[0]}_${special[0]}2912`,
       },
-      `TestOUIWith${space[1]}_${special[1]}2912-TestSerialNumberWith${space[1]}_${special[1]}2912`
+      `TestOUIWith${space[1]}_${special[1]}2912-TestSerialNumberWith${space[1]}_${special[1]}2912`,
-    ]
+    ],
   ];
   t.plan(cases.length);
-  for (const c of cases) t.is(common.generateDeviceId(c[0]), c[1]);
+  for (const c of cases)
+    t.is(common.generateDeviceId(c[0] as Record<string, string>), c[1]);
 });
 
-ava("escapeRegExp", t => {
+ava("escapeRegExp", (t) => {
   t.is(
     common.escapeRegExp("\\ ^ $ * + ? . ( ) | { } [ ]"),
     "\\\\ \\^ \\$ \\* \\+ \\? \\. \\( \\) \\| \\{ \\} \\[ \\]"

test/device.ts

@@ -5,7 +5,7 @@ import PathSet from "../lib/common/path-set";
 import VersionedMap from "../lib/versioned-map";
 import { Attributes } from "../lib/types";
 
-ava("getAliasDeclarations", t => {
+ava("getAliasDeclarations", (t) => {
   const path = Path.parse("a.[aa:10,bb.[aaa:100].cc:1].b");
   const decs = device.getAliasDeclarations(path, 99);
 
@@ -22,64 +22,64 @@ ava("getAliasDeclarations", t => {
   }
 });
 
-ava("unpack", t => {
+ava("unpack", (t) => {
   const now = Date.now();
   const deviceData = {
     paths: new PathSet(),
     timestamps: new VersionedMap<Path, number>(),
     attributes: new VersionedMap<Path, Attributes>(),
     trackers: new Map<Path, { [name: string]: number }>(),
-    changes: new Set<string>()
+    changes: new Set<string>(),
   };
 
   device.set(deviceData, Path.parse("a.1.b"), now, {
-    value: [now, ["b", "xsd:string"]]
+    value: [now, ["b", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.c"), now, {
-    value: [now, ["c", "xsd:string"]]
+    value: [now, ["c", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.1.a"), now, {
-    value: [now, ["", "xsd:string"]]
+    value: [now, ["", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.1.b"), now, {
-    value: [now, ["b1", "xsd:string"]]
+    value: [now, ["b1", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.1.c"), now, {
-    value: [now, ["c1", "xsd:string"]]
+    value: [now, ["c1", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.2.a"), now, {
-    value: [now, ["", "xsd:string"]]
+    value: [now, ["", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.2.b"), now, {
-    value: [now, ["b2", "xsd:string"]]
+    value: [now, ["b2", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.1.a.2.c"), now, {
-    value: [now, ["c2", "xsd:string"]]
+    value: [now, ["c2", "xsd:string"]],
   });
 
   device.set(deviceData, Path.parse("a.2.b"), now, {
-    value: [now, ["b", "xsd:string"]]
+    value: [now, ["b", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.c"), now, {
-    value: [now, ["c", "xsd:string"]]
+    value: [now, ["c", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.1.a"), now, {
-    value: [now, ["", "xsd:string"]]
+    value: [now, ["", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.1.b"), now, {
-    value: [now, ["b1", "xsd:string"]]
+    value: [now, ["b1", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.1.c"), now, {
-    value: [now, ["c1", "xsd:string"]]
+    value: [now, ["c1", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.2.a"), now, {
-    value: [now, ["", "xsd:string"]]
+    value: [now, ["", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.2.b"), now, {
-    value: [now, ["c1", "xsd:string"]]
+    value: [now, ["c1", "xsd:string"]],
   });
   device.set(deviceData, Path.parse("a.2.a.2.c"), now, {
-    value: [now, ["b1", "xsd:string"]]
+    value: [now, ["b1", "xsd:string"]],
   });
 
   let unpacked: Path[];

test/mongodb-functions.ts

@@ -1,13 +1,13 @@
 import ava from "ava";
 import * as mongodbFunctions from "../lib/mongodb-functions";
-import { stringify } from "../lib/common/expression-parser";
+import { stringify, parse } from "../lib/common/expression-parser";
 
-ava("mongoQueryToFilter", t => {
+ava("convertOldPrecondition", (t) => {
   const tests = [
     [{}, "true"],
     [{ test: "test" }, 'test = "test"'],
     [{ test: { $eq: "test" } }, 'test = "test"'],
-    [{ test: { $ne: "test" } }, 'test <> "test"'],
+    [{ test: { $ne: "test" } }, 'test <> "test" OR test IS NULL'],
     [{ test: { $gte: "test" } }, 'test >= "test"'],
     [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
     [{ "Tags.test": false }, "Tags.test IS NULL"],
@@ -22,17 +22,21 @@ ava("mongoQueryToFilter", t => {
     [{ _tags: { $eq: "test" } }, "Tags.test IS NOT NULL"],
     [
       { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" AND test <> "test"'
+      'test = "test" AND (test <> "test" OR test IS NULL)',
     ],
     [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
     [
       { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" OR test <> "test"'
+      'test = "test" OR (test <> "test" OR test IS NULL)',
     ],
     [
       { test: { $gte: "test1", $ne: "test2" } },
-      'test >= "test1" AND test <> "test2"'
+      'test >= "test1" AND (test <> "test2" OR test IS NULL)',
-    ]
+    ],
+    [
+      { test: "test", test2: { $ne: "test2" } },
+      'test = "test" AND (test2 <> "test2" OR test2 IS NULL)',
+    ],
   ];
 
   const shouldFailTests = [
@@ -41,18 +45,88 @@ ava("mongoQueryToFilter", t => {
     [{ "Tags.test": { $gt: true } }, "Invalid tag query"],
     [{ _tags: [] }, "Invalid type"],
     [{ _tags: { $gt: "test" } }, "Invalid tag query"],
-    [{ $nor: [] }, "Operator $nor not supported"]
+    [{ $nor: [] }, "Operator $nor not supported"],
   ];
 
   t.plan(tests.length + 2 * shouldFailTests.length);
   for (const test of tests)
-    t.is(stringify(mongodbFunctions.mongoQueryToFilter(test[0])), test[1]);
+    t.is(stringify(mongodbFunctions.convertOldPrecondition(test[0])), test[1]);
 
   for (const test of shouldFailTests) {
     const func = (): void => {
-      mongodbFunctions.mongoQueryToFilter(test[0]);
+      mongodbFunctions.convertOldPrecondition(
+        test[0] as Record<string, unknown>
+      );
     };
-    const error = t.throws(func, Error);
+    const error = t.throws(func, { instanceOf: Error });
     t.is(error.message, test[1]);
   }
 });
+
+ava("filterToMongoQuery", async (t) => {
+  const queries: [string, Record<string, unknown>][] = [
+    ["true", {}],
+    ["Tags.tag1 = true", { _tags: "tag1" }],
+    ["Tags.tag1 <> false", { _tags: "tag1" }],
+    ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }],
+    ["Tags.tag1 = 123", { "Tags.tag1": 123 }],
+    ["Param1 = 'value1'", { "Param1._value": "value1" }],
+    ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }],
+    ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
+    ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
+    ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }],
+    ["LOWER(Param1) LIKE 'value'", { "Param1._value": /^value$/i }],
+    [
+      "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          {
+            $and: [
+              { "Param2._value": { $nin: ["value1", null] } },
+              { "Param1._value": { $gte: "value2" } },
+            ],
+          },
+        ],
+      },
+    ],
+    [
+      "Param1 <> 'value2' OR Param1 IS NULL",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          { "Param1._value": null },
+        ],
+      },
+    ],
+  ];
+
+  for (const [expStr, expect] of queries) {
+    const exp = parse(expStr);
+    const query = mongodbFunctions.filterToMongoQuery(
+      mongodbFunctions.processDeviceFilter(exp)
+    );
+    t.deepEqual(query, expect);
+  }
+
+  const failQueries: [any, string][] = [
+    ["Param1 = Param2", "Invalid RHS operand of = clause"],
+    ["Param1 LIKE Param2", "Invalid RHS operand of LIKE clause"],
+    ["1 LIKE '1'", "Invalid LHS operand of LIKE clause"],
+    ["1 = 2", "Invalid LHS operand of = clause"],
+    ["1 IS NULL", "Invalid LHS operand of IS NULL clause"],
+    ["false", "Primitives are not valid queries"],
+    ["UPPER(Param1) LIKE 'value'", "Invalid RHS operand of LIKE clause"],
+  ];
+
+  for (const [expStr, err] of failQueries) {
+    const exp = parse(expStr);
+    t.throws(
+      () =>
+        mongodbFunctions.filterToMongoQuery(
+          mongodbFunctions.processDeviceFilter(exp)
+        ),
+      { message: err }
+    );
+  }
+});

test/path-set.ts

@@ -2,7 +2,7 @@ import ava from "ava";
 import Path from "../lib/common/path";
 import PathSet from "../lib/common/path-set";
 
-ava("depth", t => {
+ava("depth", (t) => {
   const pathSet = new PathSet();
   t.is(pathSet.depth, 0);
   pathSet.add(Path.parse(""));
@@ -11,14 +11,14 @@ ava("depth", t => {
   t.is(pathSet.depth, 2);
 });
 
-ava("add", t => {
+ava("add", (t) => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a"));
   pathSet.add(Path.parse("a"));
   t.is(pathSet.find(Path.parse("a"), true, true, 99).length, 1);
 });
 
-ava("get", t => {
+ava("get", (t) => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a.*"));
   pathSet.add(Path.parse("a.a"));
@@ -28,7 +28,7 @@ ava("get", t => {
   t.is(pathSet.get(Path.parse("*.a")), null);
 });
 
-ava("find", t => {
+ava("find", (t) => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a"));
   pathSet.add(Path.parse("a.*"));
@@ -37,32 +37,32 @@ ava("find", t => {
   pathSet.add(Path.parse("*.*"));
 
   t.deepEqual(
-    pathSet.find(Path.parse(""), true, true, 1).map(p => p.toString()),
+    pathSet.find(Path.parse(""), true, true, 1).map((p) => p.toString()),
     ["a"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse(""), false, false, 2).map(p => p.toString()),
+    pathSet.find(Path.parse(""), false, false, 2).map((p) => p.toString()),
     ["a", "a.*", "a.a", "*.a", "*.*"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), false, false).map(p => p.toString()),
+    pathSet.find(Path.parse("a.*"), false, false).map((p) => p.toString()),
     ["a.*"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), false, true).map(p => p.toString()),
+    pathSet.find(Path.parse("a.*"), false, true).map((p) => p.toString()),
     ["a.*", "a.a"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), true, false).map(p => p.toString()),
+    pathSet.find(Path.parse("a.*"), true, false).map((p) => p.toString()),
     ["a.*", "*.*"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), true, true).map(p => p.toString()),
+    pathSet.find(Path.parse("a.*"), true, true).map((p) => p.toString()),
     ["a.*", "a.a", "*.a", "*.*"]
   );
 });

test/path.ts

@@ -1,13 +1,13 @@
 import ava from "ava";
 import Path from "../lib/common/path";
 
-ava("toString", t => {
+ava("toString", (t) => {
   const path1 = Path.parse('abc.[ abc : 123 , 123 : abc , 123: " abc "].123');
   const path2 = Path.parse('abc.[123:  " abc ",abc:123,123:abc].123');
   t.is(path1.toString(), path2.toString());
 });
 
-ava("slice", t => {
+ava("slice", (t) => {
   const path = Path.parse("a.*.b.[x:y].c");
   const sliced = path.slice(1, -1);
   t.is(sliced.toString(), '*.b.[x:"y"]');
@@ -15,7 +15,7 @@ ava("slice", t => {
   t.is(sliced.wildcard, 0b1);
 });
 
-ava("concat", t => {
+ava("concat", (t) => {
   const path1 = Path.parse("a");
   const path2 = Path.parse("*.[a:b]");
   const concat = path1.concat(path2);

test/xml-parser.ts

@@ -2,10 +2,10 @@ import ava from "ava";
 import {
   parseXmlDeclaration,
   decodeEntities,
-  parseXml
+  parseXml,
 } from "../lib/xml-parser";
 
-ava("parseXmlDeclaration", t => {
+ava("parseXmlDeclaration", (t) => {
   const buf = Buffer.from(
     '<?xml version="1.0" encoding="UTF-8"?>\n<soap-env:Envelope />'
   );
@@ -15,25 +15,25 @@ ava("parseXmlDeclaration", t => {
       name: "version",
       namespace: "",
       localName: "version",
-      value: "1.0"
+      value: "1.0",
     },
     {
       name: "encoding",
       namespace: "",
       localName: "encoding",
-      value: "UTF-8"
+      value: "UTF-8",
-    }
+    },
   ]);
 });
 
-ava("decodeEntities", t => {
+ava("decodeEntities", (t) => {
   t.is(
     decodeEntities("&&amp;&lt;&gt;&quot;&apos;&gt;&#167;&#xd842;&#xDFB7;;"),
     "&&<>\"'>§𠮷;"
   );
 });
 
-ava("parse", t => {
+ava("parse", (t) => {
   const xml =
     '<?xml version="1.0"?>\n<a-b:c><d f="1<g>"/><!-- comment --><h >i</h></a-b:c>';
   const parsed = parseXml(xml);
@@ -60,7 +60,7 @@ ava("parse", t => {
             attrs: 'f="1<g>"',
             text: "",
             bodyIndex: 42,
-            children: []
+            children: [],
           },
           {
             name: "h",
@@ -69,10 +69,10 @@ ava("parse", t => {
             attrs: "",
             text: "i",
             bodyIndex: 62,
-            children: []
+            children: [],
-          }
+          },
-        ]
+        ],
-      }
+      },
-    ]
+    ],
   });
 });

tools/dump-data-model.ts

@@ -1,5 +1,3 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -60,7 +58,7 @@ connect()
   .then(() => {
     devicesCollection.findOne(
       {
-        _id: process.argv[2]
+        _id: process.argv[2],
       },
       {},
       (err, device) => {
@@ -79,6 +77,6 @@ connect()
       }
     );
   })
-  .catch(err => {
+  .catch((err) => {
     throw err;
   });

tsconfig.json

@@ -14,11 +14,11 @@
     "allowJs": true
   },
   "include": [
-      "./bin/.*.ts",
+      "./bin/*.ts",
       "./lib/**/*",
       "./ui/**/*",
       "./test/**/*",
       "./build/**/*",
-      "./tools/.*.ts"
+      "./tools/*.ts"
   ]
 }

ui/admin-menu.ts

@@ -22,7 +22,7 @@ import { ClosureComponent, Component } from "mithril";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const active = { [vnode.attrs["page"]]: "active" };
       const tabs = [];
 
@@ -97,7 +97,7 @@ const component: ClosureComponent = (): Component => {
       }
 
       return m("nav#side-menu", m("ul", tabs));
-    }
+    },
   };
 };
 

ui/app.ts

@@ -37,13 +37,14 @@ import * as usersPage from "./users-page";
 import Authorizer from "../lib//common/authorizer";
 import * as notifications from "./notifications";
 import { contextifyComponent } from "./components";
+import { PermissionSet, UiConfig } from "../lib/types";
 
 declare global {
   interface Window {
     authorizer: Authorizer;
-    permissionSets: any;
+    permissionSets: PermissionSet[];
     username: string;
-    clientConfig: any;
+    clientConfig: { ui: UiConfig };
     configSnapshot: string;
     genieacsVersion: string;
   }
@@ -58,7 +59,7 @@ const adminPages = [
   "files",
   "config",
   "users",
-  "permissions"
+  "permissions",
 ];
 
 let state;
@@ -70,7 +71,7 @@ let fulfillTimeout = null;
 function fulfill(): void {
   clearTimeout(fulfillTimeout);
   fulfillTimeout = setTimeout(() => {
-    store.fulfill(lastRenderTimestamp, pageVisitTimestamp).then(updated => {
+    store.fulfill(lastRenderTimestamp, pageVisitTimestamp, (updated) => {
       if (updated) m.redraw();
     });
   }, 100);
@@ -88,7 +89,7 @@ function pagify(pageName, page): RouteResolver {
       attrs["page"] = pageName;
       return m(layout, attrs, p);
     },
-    onmatch: null
+    onmatch: null,
   };
 
   component.onmatch = (args, requestedPath) => {
@@ -99,16 +100,16 @@ function pagify(pageName, page): RouteResolver {
       return null;
     }
 
-    return new Promise(resolve => {
+    return new Promise((resolve) => {
       page
         .init(args)
-        .then(st => {
+        .then((st) => {
           if (!st) return void m.route.set("/");
           state = st;
           resolve();
           fulfill();
         })
-        .catch(err => {
+        .catch((err) => {
           if (!window.username && err.message.indexOf("authorized") >= 0) {
             notifications.push("error", err.message);
             m.route.set("/login", { continue: requestedPath });
@@ -132,7 +133,7 @@ function redirectAdminPage(): RouteResolver {
         }
       }
       return null;
-    }
+    },
   };
   return component;
 }
@@ -154,5 +155,5 @@ m.route(document.body, "/overview", {
   "/admin/files": pagify("files", filesPage),
   "/admin/config": pagify("config", configPage),
   "/admin/users": pagify("users", usersPage),
-  "/admin/permissions": pagify("permissions", permissionsPage)
+  "/admin/permissions": pagify("permissions", permissionsPage),
 });

ui/autocomplete-compnent.ts

@@ -17,11 +17,13 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-export default class Autocomplete {
+type AutocompleteCallback = (
-  private callback: (
   value: string,
   callback: (suggestions: { value: string; tip?: string }[]) => void
-  ) => void;
+) => void;
+
+export default class Autocomplete {
+  private callback: AutocompleteCallback;
   private element: HTMLInputElement;
   private hideTimeout: NodeJS.Timeout;
   private visible: boolean;
@@ -29,7 +31,7 @@ export default class Autocomplete {
   private selection: number;
   private container: HTMLElement;
 
-  public constructor(className, callback) {
+  public constructor(className: string, callback: AutocompleteCallback) {
     this.callback = callback;
     this.element = null;
     this.hideTimeout = null;
@@ -44,7 +46,7 @@ export default class Autocomplete {
     this.container.className = className;
   }
 
-  public attach(el): void {
+  public attach(el: HTMLInputElement): void {
     el.setAttribute("autocomplete", "off");
 
     el.addEventListener("focus", () => {
@@ -62,7 +64,7 @@ export default class Autocomplete {
       this.hide();
     });
 
-    el.addEventListener("keydown", e => {
+    el.addEventListener("keydown", (e) => {
       if (this.element !== el) return;
       if (e.key === "Escape") {
         if (this.visible) this.hide();
@@ -108,7 +110,7 @@ export default class Autocomplete {
   private update(): void {
     const el = this.element;
 
-    this.callback(el.value, suggestions => {
+    this.callback(el.value, (suggestions) => {
       if (this.element !== el) return;
       this.default = null;
 
@@ -154,7 +156,7 @@ export default class Autocomplete {
 
         const t = document.createTextNode(suggestion.value);
         e.appendChild(t);
-        e.addEventListener("mousedown", ev => {
+        e.addEventListener("mousedown", (ev) => {
           ev.preventDefault();
           el.value = suggestion.value;
           if (this.element === el) this.update();

ui/change-password-component.ts

@@ -24,7 +24,7 @@ import { m } from "./components";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const onPasswordChange = vnode.attrs["onPasswordChange"];
       const enforceAuth = !vnode.attrs["noAuth"];
       const username = vnode.attrs["username"];
@@ -40,19 +40,19 @@ const component: ClosureComponent = (): Component => {
             type: "text",
             value: vnode.state["username"],
             disabled: !!username,
-            oninput: e => {
+            oninput: (e) => {
               vnode.state["username"] = e.target.value;
             },
-            oncreate: _vnode => {
+            oncreate: (_vnode) => {
               (_vnode.dom as HTMLSelectElement).focus();
-            }
+            },
           })
-        )
+        ),
       ];
 
       let fields = {
         newPassword: "New password",
-        confirmPassword: "Confirm password"
+        confirmPassword: "Confirm password",
       };
       if (enforceAuth)
         fields = Object.assign({ authPassword: "Your password" }, fields);
@@ -66,9 +66,9 @@ const component: ClosureComponent = (): Component => {
               name: f,
               type: "password",
               value: vnode.state[f],
-              oninput: e => {
+              oninput: (e) => {
                 vnode.state[f] = e.target.value;
-              }
+              },
             })
           )
         );
@@ -77,7 +77,7 @@ const component: ClosureComponent = (): Component => {
       const submit = m(
         "button.primary",
         {
-          type: "submit"
+          type: "submit",
         },
         "Change password"
       ) as VnodeDOM;
@@ -89,7 +89,7 @@ const component: ClosureComponent = (): Component => {
         m(
           "form",
           {
-            onsubmit: e => {
+            onsubmit: (e) => {
               e.redraw = false;
               e.preventDefault();
               if (
@@ -120,19 +120,19 @@ const component: ClosureComponent = (): Component => {
                     if (onPasswordChange) onPasswordChange();
                     (submit.dom as HTMLFormElement).disabled = false;
                   })
-                  .catch(err => {
+                  .catch((err) => {
                     notifications.push("error", err.message);
                     (submit.dom as HTMLFormElement).disabled = false;
                   });
               }
-            }
+            },
           },
           form
-        )
+        ),
       ];
 
       return m("div.put-form", children);
-    }
+    },
   };
 };
 

ui/code-editor-component.ts

@@ -23,11 +23,11 @@ import { codeMirror } from "./dynamic-loader";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       return m("textarea", {
         name: vnode.attrs["id"],
         value: vnode.attrs["value"],
-        oncreate: _vnode => {
+        oncreate: (_vnode) => {
           const editor = codeMirror.fromTextArea(_vnode.dom, {
             mode: vnode.attrs["mode"],
             lineNumbers: true,
@@ -40,21 +40,21 @@ const component: ClosureComponent = (): Component => {
               "Cmd-Enter": () => {
                 if (vnode.attrs["onSubmit"])
                   vnode.attrs["onSubmit"](_vnode.dom);
-              }
+              },
-            }
+            },
           });
 
           if (vnode.attrs["onChange"]) {
-            editor.on("change", e => {
+            editor.on("change", (e) => {
               vnode.attrs["onChange"](e.getValue());
             });
           }
 
           if (vnode.attrs["focus"]) editor.focus();
           if (vnode.attrs["onReady"]) vnode.attrs["onReady"](editor);
-        }
+        },
       });
-    }
+    },
   };
 };
 

ui/components.ts

@@ -24,7 +24,7 @@ import m, {
   ComponentTypes,
   Lifecycle,
   ClosureComponent,
-  Vnode
+  Vnode,
 } from "mithril";
 import parameter from "./components/parameter";
 import parameterList from "./components/parameter-list";
@@ -53,7 +53,7 @@ const comps = {
   tags,
   ping,
   "device-link": deviceLink,
-  "long-text": longTextComponent
+  "long-text": longTextComponent,
 };
 
 const contextifiedComponents = new WeakMap<ComponentTypes, ComponentTypes>();
@@ -96,7 +96,7 @@ const M = new Proxy(m, {
   get: (target, prop) => {
     if (prop === "context") return contextFn;
     else return Reflect.get(target, prop);
-  }
+  },
 }) as MC;
 
 function contextFn(context, ...argumentsList): Vnode {
@@ -125,17 +125,17 @@ export function contextifyComponent(component: ComponentTypes): ComponentTypes {
     if (typeof component !== "function") {
       c = Object.assign({}, component);
       const view = component.view;
-      c.view = function(vnode) {
+      c.view = function (vnode) {
         const context = vnodeContext.get(vnode) || {};
         const res = Reflect.apply(view, this, [vnode]);
         applyContext(res, context);
         return res;
       };
     } else if (!component.prototype || !component.prototype.view) {
-      c = initialNode => {
+      c = (initialNode) => {
         const state = (component as ClosureComponent)(initialNode);
         const view = state.view;
-        state.view = function(vnode) {
+        state.view = function (vnode) {
           const context = vnodeContext.get(vnode) || {};
           const res = Reflect.apply(view, this, [vnode]);
           applyContext(res, context);

ui/components/all-parameters.ts

@@ -30,40 +30,73 @@ function escapeRegExp(str): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
 
+const keysByDepth: WeakMap<Record<string, unknown>, string[][]> = new WeakMap();
+
+function orderKeysByDepth(device: Record<string, unknown>): string[][] {
+  if (keysByDepth.has(device)) return keysByDepth.get(device);
+  const res: string[][] = [];
+  for (const key of Object.keys(device)) {
+    let count = 0;
+    for (
+      let i = key.lastIndexOf(".", key.length - 2);
+      i >= 0;
+      i = key.lastIndexOf(".", i - 1)
+    )
+      ++count;
+    while (res.length <= count) res.push([]);
+    res[count].push(key);
+  }
+  keysByDepth.set(device, res);
+  return res;
+}
+
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
 
       const search = m("input", {
         type: "text",
         placeholder: "Search parameters",
-        oninput: e => {
+        oninput: (e) => {
           vnode.state["searchString"] = e.target.value;
           e.redraw = false;
           clearTimeout(vnode.state["timeout"]);
           vnode.state["timeout"] = setTimeout(m.redraw, 500);
-        }
+        },
       });
 
       const instanceRegex = /\.[0-9]+$/;
       let re;
       if (vnode.state["searchString"]) {
-        const keywords = vnode.state["searchString"].split(" ").filter(s => s);
+        const keywords = vnode.state["searchString"]
+          .split(" ")
+          .filter((s) => s);
         if (keywords.length)
-          re = new RegExp(keywords.map(s => escapeRegExp(s)).join(".*"), "i");
+          re = new RegExp(keywords.map((s) => escapeRegExp(s)).join(".*"), "i");
       }
 
-      const rows = Object.keys(device)
+      const filteredKeys: string[] = [];
-        .sort()
+      const allKeys = orderKeysByDepth(device);
-        .map(k => {
+      let count = 0;
+      for (const keys of allKeys) {
+        let c = 0;
+        for (const k of keys) {
+          const p = device[k];
+          const str = p.value && p.value[0] ? `${k} ${p.value[0]}` : k;
+          if (re && !re.test(str)) continue;
+          ++c;
+          if (count < 100) filteredKeys.push(k);
+        }
+        count += c;
+      }
+
+      filteredKeys.sort();
+
+      const rows = filteredKeys.map((k) => {
         const p = device[k];
         const val = [];
-          const attrs = {};
+        const attrs = { key: k };
-          if (re) {
-            const str = p.value && p.value[0] ? `${k} ${p.value[0]}` : k;
-            if (!re.test(str)) attrs["style"] = "display: none;";
-          }
 
         if (p.object === false) {
           val.push(
@@ -83,9 +116,9 @@ const component: ClosureComponent = (): Component => {
                     taskQueue.queueTask({
                       name: "deleteObject",
                       device: device["DeviceID.ID"].value[0],
-                        objectName: k
+                      objectName: k,
                     });
-                    }
+                  },
                 },
                 getIcon("delete-instance")
               )
@@ -100,9 +133,9 @@ const component: ClosureComponent = (): Component => {
                     taskQueue.queueTask({
                       name: "addObject",
                       device: device["DeviceID.ID"].value[0],
-                        objectName: k
+                      objectName: k,
                     });
-                    }
+                  },
                 },
                 getIcon("add-instance")
               )
@@ -119,9 +152,9 @@ const component: ClosureComponent = (): Component => {
                 taskQueue.queueTask({
                   name: "getParameterValues",
                   device: device["DeviceID.ID"].value[0],
-                    parameterNames: [k]
+                  parameterNames: [k],
                 });
-                }
+              },
             },
             getIcon("refresh")
           )
@@ -140,16 +173,25 @@ const component: ClosureComponent = (): Component => {
         m(
           "a.download-csv",
           {
-            href: `/api/devices/${device["DeviceID.ID"].value[0]}.csv`,
+            href: `api/devices/${encodeURIComponent(
+              device["DeviceID.ID"].value[0]
+            )}.csv`,
             download: "",
-            style: "float: right;"
+            style: "float: right;",
           },
           "Download"
         ),
         search,
-        m(".parameter-list", m("table", m("tbody", rows)))
+        m(
+          ".parameter-list",
+          m("table", m("tbody", rows)),
+          m(
+            "m",
+            `Displaying ${filteredKeys.length} out of ${count} parameters.`
+          )
+        )
       );
-    }
+    },
   };
 };
 

ui/components/container.ts

@@ -23,7 +23,7 @@ import * as store from "../store";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       if (vnode.attrs["filter"]) {
         if (
           !store.evaluateExpression(
@@ -34,13 +34,13 @@ const component: ClosureComponent = (): Component => {
           return null;
       }
 
-      const children = Object.values(vnode.attrs["components"]).map(c => {
+      const children = Object.values(vnode.attrs["components"]).map((c) => {
         if (typeof c !== "object") return `${c}`;
         return m(c["type"], c);
       });
       if (vnode.attrs["element"]) return m(vnode.attrs["element"], children);
       else return children;
-    }
+    },
   };
 };
 

ui/components/device-actions.ts

@@ -25,7 +25,7 @@ import * as store from "../store";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
 
       const buttons = [];
@@ -38,9 +38,9 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.queueTask({
                 name: "reboot",
-                device: device["DeviceID.ID"].value[0]
+                device: device["DeviceID.ID"].value[0],
               });
-            }
+            },
           },
           "Reboot"
         )
@@ -54,9 +54,9 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.queueTask({
                 name: "factoryReset",
-                device: device["DeviceID.ID"].value[0]
+                device: device["DeviceID.ID"].value[0],
               });
-            }
+            },
           },
           "Reset"
         )
@@ -70,9 +70,9 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageDownload({
                 name: "download",
-                device: device["DeviceID.ID"].value[0]
+                devices: [device["DeviceID.ID"].value[0]],
               });
-            }
+            },
           },
           "Push file"
         )
@@ -93,17 +93,17 @@ const component: ClosureComponent = (): Component => {
                   notifications.push("success", `${deviceId}: Device deleted`);
                   m.route.set("/devices");
                 })
-                .catch(err => {
+                .catch((err) => {
                   notifications.push("error", err.message);
                 });
-            }
+            },
           },
           "Delete"
         )
       );
 
       return m(".actions-bar", buttons);
-    }
+    },
   };
 };
 

ui/components/device-faults.ts

@@ -25,13 +25,13 @@ import { getIcon } from "../icons";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
       const deviceId = device["DeviceID.ID"].value[0];
       const faults = store.fetch("faults", [
         "AND",
         [">", ["PARAM", "_id"], `${deviceId}:`],
-        ["<", ["PARAM", "_id"], `${deviceId}:zzzz`]
+        ["<", ["PARAM", "_id"], `${deviceId}:zzzz`],
       ]);
 
       const headers = [
@@ -39,8 +39,8 @@ const component: ClosureComponent = (): Component => {
         "Code",
         "Message",
         "Retries",
-        "Timestamp"
+        "Timestamp",
-      ].map(l => m("th", l));
+      ].map((l) => m("th", l));
       const thead = m("thead", m("tr", headers));
 
       const rows = [];
@@ -57,7 +57,7 @@ const component: ClosureComponent = (): Component => {
               "button",
               {
                 title: "Delete fault",
-                onclick: e => {
+                onclick: (e) => {
                   e.redraw = false;
                   store
                     .deleteResource("faults", f["_id"])
@@ -66,21 +66,24 @@ const component: ClosureComponent = (): Component => {
                       store.fulfill(Date.now(), Date.now());
                       m.redraw();
                     })
-                    .catch(err => {
+                    .catch((err) => {
                       notifications.push("error", err.message);
                       store.fulfill(Date.now(), Date.now());
                     });
-                }
+                },
               },
               getIcon("remove")
             )
-          )
+          ),
         ]);
       }
 
       let tbody;
       if (rows.length) {
-        tbody = m("tbody", rows.map(r => m("tr", r)));
+        tbody = m(
+          "tbody",
+          rows.map((r) => m("tr", r))
+        );
       } else {
         tbody = m(
           "tbody",
@@ -89,7 +92,7 @@ const component: ClosureComponent = (): Component => {
       }
 
       return m("table.table", thead, tbody);
-    }
+    },
   };
 };
 

ui/components/device-link.ts

@@ -22,12 +22,12 @@ import { m } from "../components";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       let deviceId;
       if (vnode.attrs["device"])
         deviceId = vnode.attrs["device"]["DeviceID.ID"].value[0];
 
-      const children = Object.values(vnode.attrs["components"]).map(c => {
+      const children = Object.values(vnode.attrs["components"]).map((c) => {
         if (typeof c !== "object") return `${c}`;
         const attrs = Object.assign({}, vnode.attrs, c);
         return m(attrs["type"], attrs);
@@ -41,7 +41,7 @@ const component: ClosureComponent = (): Component => {
       } else {
         return children;
       }
-    }
+    },
   };
 };
 

ui/components/overview-dot.ts

@@ -26,9 +26,9 @@ const CHARTS = config.ui.overview.charts;
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
-      const chart = CHARTS[vnode.attrs["chart"]];
+      const chart = CHARTS[vnode.attrs["chart"]] as Record<string, unknown>;
       if (!chart) return null;
       for (const slice of Object.values(chart.slices)) {
         const filter = slice["filter"];
@@ -39,20 +39,20 @@ const component: ClosureComponent = (): Component => {
               width: "1em",
               height: "1em",
               xmlns: "http://www.w3.org/2000/svg",
-              "xmlns:xlink": "http://www.w3.org/1999/xlink"
+              "xmlns:xlink": "http://www.w3.org/1999/xlink",
             },
             m("circle", {
               cx: "0.5em",
               cy: "0.5em",
               r: "0.4em",
-              fill: slice["color"]
+              fill: slice["color"],
             })
           );
           return m("span.overview-dot", dot, `${slice["label"]}`);
         }
       }
       return null;
-    }
+    },
   };
 };
 

ui/components/parameter-list.ts

@@ -22,14 +22,14 @@ import { m } from "../components";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
 
-      const rows = Object.values(vnode.attrs["parameters"]).map(parameter => {
+      const rows = Object.values(vnode.attrs["parameters"]).map((parameter) => {
         const p = m.context(
           {
             device: device,
-            parameter: parameter["parameter"]
+            parameter: parameter["parameter"],
           },
           parameter["type"] || "parameter",
           parameter
@@ -38,11 +38,11 @@ const component: ClosureComponent = (): Component => {
         return m(
           "tr",
           {
-            onupdate: vn => {
+            onupdate: (vn) => {
               (vn.dom as HTMLElement).style.display = (p as VnodeDOM).dom
                 ? ""
                 : "none";
-            }
+            },
           },
           m("th", parameter["label"]),
           m("td", p)
@@ -50,7 +50,7 @@ const component: ClosureComponent = (): Component => {
       });
 
       return m("table.parameter-list", rows);
-    }
+    },
   };
 };
 

ui/components/parameter-table.ts

@@ -26,21 +26,21 @@ import { getIcon } from "../icons";
 
 const component: ClosureComponent = (): Component => {
   return {
-    oninit: vnode => {
+    oninit: (vnode) => {
       const obj = vnode.attrs["parameter"];
       if (!Array.isArray(obj) || obj[0] !== "PARAM")
         throw new Error("Object must be a parameter path");
       vnode.state["object"] = obj[1];
       vnode.state["parameters"] = Object.values(vnode.attrs["childParameters"]);
     },
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
       const object = store.evaluateExpression(vnode.state["object"], device);
       const parameters = vnode.state["parameters"];
 
-      if (!device[object]) return null;
+      if (typeof object !== "string" || !device[object]) return null;
 
-      const instances = new Set();
+      const instances: Set<string> = new Set();
       const prefix = `${object}.`;
       for (const p in device) {
         if (p.startsWith(prefix)) {
@@ -50,7 +50,7 @@ const component: ClosureComponent = (): Component => {
         }
       }
 
-      const headers = Object.values(parameters).map(p => m("th", p["label"]));
+      const headers = Object.values(parameters).map((p) => m("th", p["label"]));
 
       const thead = m("thead", m("tr", headers));
 
@@ -59,7 +59,7 @@ const component: ClosureComponent = (): Component => {
         let filter =
           vnode.attrs["filter"] != null ? vnode.attrs["filter"] : true;
 
-        filter = expressionParser.map(filter, e => {
+        filter = expressionParser.map(filter, (e) => {
           if (Array.isArray(e) && e[0] === "PARAM")
             return ["PARAM", ["||", i, ".", e[1]]];
           return e;
@@ -67,8 +67,8 @@ const component: ClosureComponent = (): Component => {
 
         if (!store.evaluateExpression(filter, device)) continue;
 
-        const row = parameters.map(p => {
+        const row = parameters.map((p) => {
-          const param = expressionParser.map(p.parameter, e => {
+          const param = expressionParser.map(p.parameter, (e) => {
             if (Array.isArray(e) && e[0] === "PARAM")
               return ["PARAM", ["||", i, ".", e[1]]];
             return e;
@@ -78,12 +78,13 @@ const component: ClosureComponent = (): Component => {
             m.context(
               {
                 device: device,
-                parameter: param
+                parameter: param,
               },
               p.type || "parameter",
               Object.assign({}, p, {
                 device: device,
-                parameter: param
+                parameter: param,
+                label: null,
               })
             )
           );
@@ -101,9 +102,9 @@ const component: ClosureComponent = (): Component => {
                     taskQueue.queueTask({
                       name: "deleteObject",
                       device: device["DeviceID.ID"].value[0],
-                      objectName: i
+                      objectName: i,
                     });
-                  }
+                  },
                 },
                 getIcon("delete-instance")
               )
@@ -134,9 +135,9 @@ const component: ClosureComponent = (): Component => {
                     taskQueue.queueTask({
                       name: "addObject",
                       device: device["DeviceID.ID"].value[0],
-                      objectName: object
+                      objectName: object,
                     });
-                  }
+                  },
                 },
                 getIcon("add-instance")
               )
@@ -149,7 +150,7 @@ const component: ClosureComponent = (): Component => {
       if (vnode.attrs["label"]) label = m("h2", vnode.attrs["label"]);
 
       return [label, m("table.table", thead, m("tbody", rows))];
-    }
+    },
   };
 };
 

ui/components/parameter.ts

@@ -28,31 +28,49 @@ import { getIcon } from "../icons";
 
 const evaluateParam = memoize((exp, obj, now: number) => {
   let timestamp = now;
-  const params = new Set();
+  exp = expression.evaluate(exp, null, now, (e) => {
-  const value = expression.evaluate(exp, obj, now, e => {
+    if (!Array.isArray(e)) return e;
-    if (Array.isArray(e)) {
+    for (let i = 1; i < e.length; ++i) {
-      if (e[0] === "PARAM") {
+      if (
-        params.add(e[1]);
+        Array.isArray(e[i]) &&
-        if (!Array.isArray(e[1])) {
+        e[i][0] === "PARAM" &&
-          const p = obj[e[1]];
+        !Array.isArray(e[i][1])
-          if (p && p.valueTimestamp)
+      ) {
+        let v = null;
+        const p = obj[e[i][1]];
+        if (p && p.value) {
+          v = p.value[0];
           timestamp = Math.min(timestamp, p.valueTimestamp);
         }
+        e = e.slice();
+        e[i] = v;
+      }
     }
     if (e[0] === "FUNC" && e[1] === "DATE_STRING" && !Array.isArray(e[2]))
       return new Date(e[2]).toLocaleString();
-    }
     return e;
   });
 
-  let parameter = params.size === 1 ? params.values().next().value : null;
+  let parameter = null;
-  if (Array.isArray(parameter)) parameter = null;
+  let value = null;
+
+  if (!Array.isArray(exp)) {
+    value = exp;
+  } else if (exp[0] === "PARAM") {
+    const p = obj[exp[1]];
+    if (p && p.value) {
+      timestamp = p.valueTimestamp;
+      value = p.value[0];
+      parameter = exp[1];
+    }
+  }
+
   return { value, timestamp, parameter };
 });
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
 
       const { value, timestamp, parameter } = evaluateParam(
@@ -72,16 +90,16 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageSpv({
                 name: "setParameterValues",
-                device: device["DeviceID.ID"].value[0],
+                devices: [device["DeviceID.ID"].value[0]],
                 parameterValues: [
                   [
                     parameter,
                     device[parameter].value[0],
-                    device[parameter].value[1]
+                    device[parameter].value[1],
-                  ]
+                  ],
-                ]
+                ],
               });
-            }
+            },
           },
           getIcon("edit")
         );
@@ -93,7 +111,7 @@ const component: ClosureComponent = (): Component => {
         "span",
         {
           class: "parameter-value",
-          onmouseover: e => {
+          onmouseover: (e) => {
             e.redraw = false;
             // Don't update any child element
             if (e.target === (el as VnodeDOM).dom) {
@@ -101,12 +119,12 @@ const component: ClosureComponent = (): Component => {
               const localeString = new Date(timestamp).toLocaleString();
               e.target.title = `${localeString} (${timeAgo(now - timestamp)})`;
             }
-          }
+          },
         },
         el,
         edit
       );
-    }
+    },
   };
 };
 

ui/components/ping.ts

@@ -17,59 +17,68 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ClosureComponent, Component } from "mithril";
+import { ClosureComponent, Component, VnodeDOM } from "mithril";
 import { m } from "../components";
 import * as store from "../store";
 
 const REFRESH_INTERVAL = 3000;
 
-const component: ClosureComponent = (): Component => {
+const component: ClosureComponent = (vn): Component => {
-  return {
+  let interval: ReturnType<typeof setInterval>;
-    oninit: vnode => {
+  let host: string;
-      vnode.state["timestamp"] = 0;
+
-      vnode.state["ping"] = null;
-      vnode.state["host"] = null;
-      vnode.state["timeout"] = null;
-    },
-    view: vnode => {
   const refresh = (): void => {
+    if (!host) {
+      const dom = (vn as VnodeDOM).dom;
+      if (dom) dom.innerHTML = "";
+      return;
+    }
+
+    let status = "";
+    store
+      .ping(host)
+      .then((res) => {
+        if (res["avg"] != null) status = `${Math.trunc(res["avg"])} ms`;
+        else status = "Unreachable";
+      })
+      .catch(() => {
+        status = "Error!";
+        clearInterval(interval);
+      })
+      .finally(() => {
+        const dom = (vn as VnodeDOM).dom;
+        if (dom) dom.innerHTML = `Pinging ${host}: ${status}`;
+      });
+  };
+
+  return {
+    onremove: () => {
+      clearInterval(interval);
+    },
+    view: (vnode) => {
       const device = vnode.attrs["device"];
       let param =
         device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"];
       if (!param)
         param = device["Device.ManagementServer.ConnectionRequestURL"];
-        if (!param || !param.value) return;
 
+      let h;
+      if (param && param.value) {
         const url = new URL(param.value[0]);
-        vnode.state["host"] = url.hostname;
+        h = url.hostname;
-        vnode.state["timestamp"] = Date.now();
-        store
-          .ping(vnode.state["host"])
-          .then(res => {
-            vnode.state["ping"] = res;
-          })
-          .catch(() => {
-            // Do nothing
-          });
-      };
-
-      const t = vnode.state["timestamp"] + REFRESH_INTERVAL - Date.now();
-      clearTimeout(vnode.state["timeout"]);
-      if (t <= 0) refresh();
-      else vnode.state["timeout"] = setTimeout(refresh, t);
-
-      if (vnode.state["host"]) {
-        let status = "";
-        if (vnode.state["ping"]) {
-          if (vnode.state["ping"].avg != null)
-            status = `${Math.trunc(vnode.state["ping"].avg)} ms`;
-          else status = "Unreachable";
       }
 
-        return m("div", `Pinging ${vnode.state["host"]}: ${status}`);
+      if (host !== h) {
+        host = h;
+        clearInterval(interval);
+        if (host) {
+          refresh();
+          interval = setInterval(refresh, REFRESH_INTERVAL);
         }
-      return null;
       }
+
+      return m("div", `Pinging ${host}:`);
+    },
   };
 };
 

ui/components/summon-button.ts

@@ -25,26 +25,26 @@ import * as notifications from "../notifications";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
 
       return m(
         "button.primary",
         {
           title: "Initiate session and refresh basic parameters",
-          onclick: e => {
+          onclick: (e) => {
             e.target.disabled = true;
             const params = Object.values(vnode.attrs["parameters"])
-              .map(exp => {
+              .map((exp) => {
                 if (!Array.isArray(exp) || exp[0] !== "PARAM") return null;
-                return store.evaluateExpression(exp[1], device);
+                return store.evaluateExpression(exp[1], device) as string;
               })
-              .filter(exp => !!exp);
+              .filter((exp) => !!exp);
 
             const task = {
               name: "getParameterValues",
               parameterNames: params,
-              device: device["DeviceID.ID"].value[0]
+              device: device["DeviceID.ID"].value[0],
             };
 
             taskQueue
@@ -79,12 +79,16 @@ const component: ClosureComponent = (): Component => {
               .then(() => {
                 e.target.disabled = false;
                 store.fulfill(0, Date.now());
+              })
+              .catch((err) => {
+                e.target.disabled = false;
+                notifications.push("error", err.message);
               });
-          }
+          },
         },
         "Summon"
       );
-    }
+    },
   };
 };
 

ui/components/tags.ts

@@ -22,32 +22,38 @@ import { m } from "../components";
 import * as notifications from "../notifications";
 import * as store from "../store";
 import { getIcon } from "../icons";
+import { decodeTag } from "../../lib/common";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       const device = vnode.attrs["device"];
       let writable = true;
       if (vnode.attrs["writable"] != null) writable = vnode.attrs["writable"];
 
       const tags = [];
       for (const p of Object.keys(device))
-        if (p.startsWith("Tags.")) tags.push(p.slice(5));
+        if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
 
       tags.sort();
 
-      if (!writable) return m(".tags", tags.map(t => m("span.tag", t)));
+      if (!writable) {
+        return m(
+          ".tags",
+          tags.map((t) => m("span.tag", t))
+        );
+      }
 
       return m(
         ".tags",
-        tags.map(tag =>
+        tags.map((tag) =>
           m(
             "span.tag",
             tag,
             m(
               "button",
               {
-                onclick: e => {
+                onclick: (e) => {
                   e.target.disabled = true;
                   const deviceId = device["DeviceID.ID"].value[0];
                   store
@@ -60,14 +66,14 @@ const component: ClosureComponent = (): Component => {
                       );
                       store.fulfill(0, Date.now());
                     })
-                    .catch(err => {
+                    .catch((err) => {
                       e.target.disabled = false;
                       notifications.push(
                         "error",
                         `${deviceId}: ${err.message}`
                       );
                     });
-                }
+                },
               },
               getIcon("remove")
             )
@@ -79,7 +85,7 @@ const component: ClosureComponent = (): Component => {
           m(
             "button",
             {
-              onclick: e => {
+              onclick: (e) => {
                 e.target.disabled = true;
                 const deviceId = device["DeviceID.ID"].value[0];
                 const tag = prompt(`Enter tag to assign to device:`);
@@ -95,17 +101,17 @@ const component: ClosureComponent = (): Component => {
                     notifications.push("success", `${deviceId}: Tags updated`);
                     store.fulfill(0, Date.now());
                   })
-                  .catch(err => {
+                  .catch((err) => {
                     e.target.disabled = false;
                     notifications.push("error", `${deviceId}: ${err.message}`);
                   });
-              }
+              },
             },
             getIcon("add")
           )
         )
       );
-    }
+    },
   };
 };
 

ui/config-functions.ts

@@ -27,7 +27,7 @@ interface Diff {
   remove: string[];
 }
 
-export function flattenConfig(config): any {
+export function flattenConfig(config: Record<string, unknown>): any {
   const flatten = {};
   const recuresive = (obj: any, root: string): void => {
     for (const [k, v] of Object.entries(obj)) {
@@ -42,6 +42,35 @@ export function flattenConfig(config): any {
   return flatten;
 }
 
+// Order keys such that nested objects come last
+function orderKeys(config: any): number {
+  let res = 1;
+  if (typeof config !== "object") return res;
+  if (Array.isArray(config)) {
+    for (const c of config) res += orderKeys(c);
+    return res;
+  }
+
+  const weights: [string, number][] = Object.entries(config).map(([k, v]) => [
+    k,
+    orderKeys(v),
+  ]);
+
+  weights.sort((a, b) => {
+    if (a[1] !== b[1]) return a[1] - b[1];
+    if (b[0] > a[0]) return -1;
+    else return 1;
+  });
+
+  for (const [k, w] of weights) {
+    res += w;
+    const v = config[k];
+    delete config[k];
+    config[k] = v;
+  }
+  return res;
+}
+
 export function structureConfig(config: Config[]): any {
   config.sort((a, b) => (a._id > b._id ? 1 : a._id < b._id ? -1 : 0));
   const _config = {};
@@ -56,7 +85,7 @@ export function structureConfig(config: Config[]): any {
     ref[keys[0]] = c.value;
   }
 
-  const toArray = function(object): any {
+  const toArray = function (object): any {
     const MAX_BITS = 30;
     const MAX_ARRAY_SIZE = MAX_BITS * 10;
 
@@ -93,13 +122,18 @@ export function structureConfig(config: Config[]): any {
     return object;
   };
 
-  return toArray(_config);
+  const res = toArray(_config);
+  orderKeys(res);
+  return res;
 }
 
-export function diffConfig(current, target): Diff {
+export function diffConfig(
+  current: Record<string, unknown>,
+  target: Record<string, unknown>
+): Diff {
   const diff = {
     add: [],
-    remove: []
+    remove: [],
   };
 
   for (const [k, v] of Object.entries(target))

ui/config-page.ts

@@ -30,7 +30,7 @@ import { getIcon } from "./icons";
 
 const attributes = [
   { id: "_id", label: "Key" },
-  { id: "value", label: "Value", type: "textarea" }
+  { id: "value", label: "Value", type: "textarea" },
 ];
 
 interface ValidationErrors {
@@ -52,13 +52,13 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
         object.value = stringify(parse(object.value || ""));
       } catch (err) {
         return void resolve({
-          value: "Config value must be valid expression"
+          value: "Config value must be valid expression",
         });
       }
 
       store
         .resourceExists("config", id)
-        .then(exists => {
+        .then((exists) => {
           if (exists && isNew) {
             store.fulfill(0, Date.now());
             return void resolve({ _id: "Config already exists" });
@@ -90,7 +90,7 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
           store.fulfill(0, Date.now());
           resolve();
         })
-        .catch(err => {
+        .catch((err) => {
           store.fulfill(0, Date.now());
           reject(err);
         });
@@ -102,14 +102,14 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
 
 const formData = {
   resource: "config",
-  attributes: attributes
+  attributes: attributes,
 };
 
 function escapeRegExp(str): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
 
-export function init(): Promise<{}> {
+export function init(): Promise<Record<string, unknown>> {
   if (!window.authorizer.hasAccess("config", 2)) {
     return Promise.reject(
       new Error("You are not authorized to view this page")
@@ -131,9 +131,9 @@ function renderTable(confsResponse, searchString): Children {
 
   let regex;
   if (searchString) {
-    const keywords = searchString.split(" ").filter(s => s);
+    const keywords = searchString.split(" ").filter((s) => s);
     if (keywords.length)
-      regex = new RegExp(keywords.map(s => escapeRegExp(s)).join(".*"), "i");
+      regex = new RegExp(keywords.map((s) => escapeRegExp(s)).join(".*"), "i");
   }
 
   const rows = [];
@@ -147,16 +147,16 @@ function renderTable(confsResponse, searchString): Children {
       {
         title: "Edit config value",
         onclick: () => {
-          const cb = (): Children => {
+          let cb: () => Children = null;
-            return m(
+          const comp = m(
             putFormComponent,
             Object.assign(
               {
                 base: conf,
                 actionHandler: (action, object) => {
-                    return new Promise(resolve => {
+                  return new Promise((resolve) => {
                     putActionHandler(action, object, false)
-                        .then(errors => {
+                      .then((errors) => {
                         const ErrorList = errors ? Object.values(errors) : [];
                         if (ErrorList.length) {
                           for (const err of ErrorList)
@@ -166,19 +166,24 @@ function renderTable(confsResponse, searchString): Children {
                         }
                         resolve();
                       })
-                        .catch(err => {
+                      .catch((err) => {
                         notifications.push("error", err.message);
                         resolve();
                       });
                   });
-                  }
+                },
               },
               formData
             )
           );
-          };
+          cb = () => comp;
-          overlay.open(cb);
+          overlay.open(
-        }
+            cb,
+            () =>
+              !comp.state["current"]["modified"] ||
+              confirm("You have unsaved changes. Close anyway?")
+          );
+        },
       },
       getIcon("edit")
     );
@@ -190,10 +195,10 @@ function renderTable(confsResponse, searchString): Children {
         onclick: () => {
           if (!confirm(`Deleting ${conf._id} config. Are you sure?`)) return;
 
-          putActionHandler("delete", conf).catch(err => {
+          putActionHandler("delete", conf).catch((err) => {
             throw err;
           });
-        }
+        },
       },
       getIcon("remove")
     );
@@ -219,18 +224,18 @@ function renderTable(confsResponse, searchString): Children {
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       document.title = "Config - GenieACS";
 
       const search = m("input", {
         type: "text",
         placeholder: "Search config",
-        oninput: e => {
+        oninput: (e) => {
           vnode.state["searchString"] = e.target.value;
           e.redraw = false;
           clearTimeout(vnode.state["timeout"]);
           vnode.state["timeout"] = setTimeout(m.redraw, 250);
-        }
+        },
       });
 
       const confs = store.fetch("config", true);
@@ -244,15 +249,15 @@ export const component: ClosureComponent = (): Component => {
           {
             title: "Create new config",
             onclick: () => {
-              const cb = (): Children => {
+              let cb: () => Children = null;
-                return m(
+              const comp = m(
                 putFormComponent,
                 Object.assign(
                   {
                     actionHandler: (action, object) => {
-                        return new Promise(resolve => {
+                      return new Promise((resolve) => {
                         putActionHandler(action, object, true)
-                            .then(errors => {
+                          .then((errors) => {
                             const errorList = errors
                               ? Object.values(errors)
                               : [];
@@ -264,19 +269,24 @@ export const component: ClosureComponent = (): Component => {
                             }
                             resolve();
                           })
-                            .catch(err => {
+                          .catch((err) => {
                             notifications.push("error", err.message);
                             resolve();
                           });
                       });
-                      }
+                    },
                   },
                   formData
                 )
               );
-              };
+              cb = () => comp;
-              overlay.open(cb);
+              overlay.open(
-            }
+                cb,
+                () =>
+                  !comp.state["current"]["modified"] ||
+                  confirm("You have unsaved changes. Close anyway?")
+              );
+            },
           },
           "New config"
         );
@@ -286,7 +296,7 @@ export const component: ClosureComponent = (): Component => {
           { name: "charts", prefix: "ui.overview.charts.", data: [] },
           { name: "filters", prefix: "ui.filters.", data: [] },
           { name: "index page", prefix: "ui.index.", data: [] },
-          { name: "device page", prefix: "ui.device.", data: [] }
+          { name: "device page", prefix: "ui.device.", data: [] },
         ];
 
         if (confs.fulfilled) {
@@ -307,12 +317,12 @@ export const component: ClosureComponent = (): Component => {
               "button",
               {
                 onclick: () => {
-                  const cb = (): Children => {
+                  let cb: () => Children = null;
-                    return m(
+                  const comp = m(
                     uiConfigComponent,
                     Object.assign(
                       {
-                          onUpdate: errs => {
+                        onUpdate: (errs: string[]) => {
                           const errors = errs ? Object.values(errs) : [];
                           if (errors.length) {
                             for (const err of errors)
@@ -329,18 +339,23 @@ export const component: ClosureComponent = (): Component => {
                           }
                           store.fulfill(0, Date.now());
                         },
-                          onError: err => {
+                        onError: (err) => {
                           notifications.push("error", err.message);
                           store.fulfill(0, Date.now());
                           overlay.close(cb);
-                          }
+                        },
                       },
                       attrs
                     )
                   );
-                  };
+                  cb = () => comp;
-                  overlay.open(cb);
+                  overlay.open(
-                }
+                    cb,
+                    () =>
+                      !comp.state["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
+                },
               },
               `Edit ${sub.name}`
             )
@@ -359,8 +374,8 @@ export const component: ClosureComponent = (): Component => {
             renderTable(confs, vnode.state["searchString"])
           ),
           m(".actions-bar", [newConfig].concat(subs))
-        )
+        ),
       ];
-    }
+    },
   };
 };

ui/css/app.css

@@ -178,6 +178,8 @@ body {
       }
 
       & > .queue {
+        max-height: 400px;
+        overflow-y: auto;
         & > .pending {
           animation: 0.5s linear 0s infinite alternate pending-blink;
           border-radius: 0.4em;
@@ -460,7 +462,7 @@ input {
   margin-bottom: 0.5em;
   & > input {
     font-size: 90%;
-    width: 30em;
+    width: 35em;
     margin: 2px;
     padding: 2px;
   }
@@ -562,7 +564,7 @@ span.na {
     font-family: monospace;
     margin-bottom: 2px;
     padding: 4px;
-    width: 600px;
+    width: 700px;
   }
 
   & > .parameter-list {

ui/device-page.ts

@@ -22,7 +22,9 @@ import { m } from "./components";
 import config from "./config";
 import * as store from "./store";
 
-export function init(args): Promise<{}> {
+export function init(
+  args: Record<string, unknown>
+): Promise<Record<string, unknown>> {
   if (!window.authorizer.hasAccess("devices", 2)) {
     return Promise.reject(
       new Error("You are not authorized to view this page")
@@ -31,13 +33,13 @@ export function init(args): Promise<{}> {
 
   return Promise.resolve({
     deviceId: args.id,
-    deviceFilter: ["=", ["PARAM", "DeviceID.ID"], args.id]
+    deviceFilter: ["=", ["PARAM", "DeviceID.ID"], args.id],
   });
 }
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       document.title = `${vnode.attrs["deviceId"]} - Devices - GenieACS`;
 
       const dev = store.fetch("devices", vnode.attrs["deviceFilter"]).value;
@@ -49,6 +51,6 @@ export const component: ClosureComponent = (): Component => {
         cmps.push(m.context({ device: dev[0] }, c["type"], c));
 
       return [m("h1", vnode.attrs["deviceId"]), cmps];
-    }
+    },
   };
 };

ui/devices-page.ts

@@ -23,7 +23,7 @@ import config from "./config";
 import indexTableComponent from "./index-table-component";
 import filterComponent from "./filter-component";
 import * as store from "./store";
-import * as taskQueue from "./task-queue";
+import { queueTask, stageDownload } from "./task-queue";
 import * as notifications from "./notifications";
 import { parse, stringify } from "../lib/common/expression-parser";
 import { evaluate, extractParams } from "../lib/common/expression";
@@ -35,7 +35,7 @@ const PAGE_SIZE = config.ui.pageSize || 10;
 
 const memoizedParse = memoize(parse);
 const memoizedJsonParse = memoize(JSON.parse);
-const memoizedGetSortable = memoize(p => {
+const memoizedGetSortable = memoize((p) => {
   const expressionParams = extractParams(p);
   if (expressionParams.length === 1) {
     const param = evaluate(expressionParams[0]);
@@ -47,21 +47,23 @@ const memoizedGetSortable = memoize(p => {
 const getDownloadUrl = memoize((filter, indexParameters) => {
   const columns = {};
   for (const p of indexParameters) columns[p.label] = stringify(p.parameter);
-  return `/api/devices.csv?${m.buildQueryString({
+  return `api/devices.csv?${m.buildQueryString({
     filter: stringify(filter),
-    columns: JSON.stringify(columns)
+    columns: JSON.stringify(columns),
   })}`;
 });
 
-const unpackSmartQuery = memoize(query => {
+const unpackSmartQuery = memoize((query) => {
-  return expressionParser.map(query, e => {
+  return expressionParser.map(query, (e) => {
     if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "Q")
       return smartQuery.unpack("devices", e[2], e[3]);
     return e;
   });
 });
 
-export function init(args): Promise<{}> {
+export function init(
+  args: Record<string, unknown>
+): Promise<Record<string, unknown>> {
   return new Promise((resolve, reject) => {
     if (!window.authorizer.hasAccess("devices", 2))
       return void reject(new Error("You are not authorized to view this page"));
@@ -72,14 +74,14 @@ export function init(args): Promise<{}> {
     if (!indexParameters.length) {
       indexParameters.push({
         label: "ID",
-        parameter: ["PARAM", "DeviceID.ID"]
+        parameter: ["PARAM", "DeviceID.ID"],
       });
     }
     resolve({ filter, indexParameters, sort });
   });
 }
 
-function renderActions(selected): Children {
+function renderActions(selected: Set<string>): Children {
   const buttons = [];
 
   buttons.push(
@@ -89,13 +91,12 @@ function renderActions(selected): Children {
         title: "Reboot selected devices",
         disabled: !selected.size,
         onclick: () => {
-          for (const d of selected) {
+          const tasks = [...selected].map((s) => ({
-            taskQueue.queueTask({
             name: "reboot",
-              device: d
+            device: s,
-            });
+          }));
-          }
+          queueTask(...tasks);
-        }
+        },
       },
       "Reboot"
     )
@@ -108,18 +109,34 @@ function renderActions(selected): Children {
         title: "Factory reset selected devices",
         disabled: !selected.size,
         onclick: () => {
-          for (const d of selected) {
+          const tasks = [...selected].map((s) => ({
-            taskQueue.queueTask({
             name: "factoryReset",
-              device: d
+            device: s,
-            });
+          }));
-          }
+          queueTask(...tasks);
-        }
+        },
       },
       "Reset"
     )
   );
 
+  buttons.push(
+    m(
+      "button.critical",
+      {
+        title: "Push a firmware or a config file",
+        disabled: !selected.size,
+        onclick: () => {
+          stageDownload({
+            name: "download",
+            devices: [...selected],
+          });
+        },
+      },
+      "Push file"
+    )
+  );
+
   buttons.push(
     m(
       "button.primary",
@@ -139,13 +156,13 @@ function renderActions(selected): Children {
                 notifications.push("success", `${id}: Deleted`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               })
-              .catch(err => {
+              .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
           if (--counter === 0) store.fulfill(0, Date.now());
-        }
+        },
       },
       "Delete"
     )
@@ -171,13 +188,13 @@ function renderActions(selected): Children {
                 notifications.push("success", `${id}: Tags updated`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               })
-              .catch(err => {
+              .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
           if (--counter === 0) store.fulfill(0, Date.now());
-        }
+        },
       },
       "Tag"
     )
@@ -205,13 +222,13 @@ function renderActions(selected): Children {
                 notifications.push("success", `${id}: Tags updated`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               })
-              .catch(err => {
+              .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
           if (--counter === 0) store.fulfill(0, Date.now());
-        }
+        },
       },
       "Untag"
     )
@@ -222,7 +239,7 @@ function renderActions(selected): Children {
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
       document.title = "Devices - GenieACS";
       const attributes = vnode.attrs["indexParameters"];
 
@@ -273,7 +290,7 @@ export const component: ClosureComponent = (): Component => {
 
       const devs = store.fetch("devices", filter, {
         limit: vnode.state["showCount"] || PAGE_SIZE,
-        sort: sort
+        sort: sort,
       });
       const count = store.count("devices", filter);
 
@@ -302,7 +319,7 @@ export const component: ClosureComponent = (): Component => {
           {
             href: `#!/devices/${encodeURIComponent(
               device["DeviceID.ID"].value[0]
-            )}`
+            )}`,
           },
           "Show"
         );
@@ -319,8 +336,8 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing devices"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs)
+        m(indexTableComponent, attrs),
       ];
-    }
+    },
   };
 };

ui/drawer-component.ts

@@ -22,12 +22,23 @@ import m, {
   Child,
   ClosureComponent,
   Component,
-  VnodeDOM
+  VnodeDOM,
 } from "mithril";
-import * as taskQueue from "./task-queue";
 import * as store from "./store";
 import * as notifications from "./notifications";
 import { getIcon } from "./icons";
+import {
+  clear,
+  commit,
+  deleteTask,
+  getQueue,
+  getStaging,
+  QueueTask,
+  queueTask,
+  StageTask,
+} from "./task-queue";
+
+const invalid: WeakSet<StageTask> = new WeakSet();
 
 function mparam(param): Children {
   return m("span.parameter", { title: param }, `${param}`);
@@ -37,11 +48,11 @@ function mval(val): Children {
   return m("span.value", { title: val }, `${val}`);
 }
 
-function renderStagingSpv(task, queueFunc, cancelFunc): Children {
+function renderStagingSpv(task: StageTask, queueFunc, cancelFunc): Children {
-  function keydown(e): void {
+  function keydown(e: KeyboardEvent): void {
-    if (e.keyCode === 13) queueFunc();
+    if (e.key === "Enter") queueFunc();
-    else if (e.keyCode === 27) cancelFunc();
+    else if (e.key === "Escape") cancelFunc();
-    else e.redraw = false;
+    else e["redraw"] = false;
   }
 
   let input;
@@ -50,18 +61,18 @@ function renderStagingSpv(task, queueFunc, cancelFunc): Children {
       "select",
       {
         value: task.parameterValues[0][1].toString(),
-        onchange: e => {
+        onchange: (e) => {
           e.redraw = false;
           task.parameterValues[0][1] = input.dom.value;
         },
         onkeydown: keydown,
-        oncreate: vnode => {
+        oncreate: (vnode) => {
           (vnode.dom as HTMLSelectElement).focus();
-        }
+        },
       },
       [
         m("option", { value: "true" }, "true"),
-        m("option", { value: "false" }, "false")
+        m("option", { value: "false" }, "false"),
       ]
     );
   } else {
@@ -72,31 +83,42 @@ function renderStagingSpv(task, queueFunc, cancelFunc): Children {
         ? "number"
         : "text",
       value: task.parameterValues[0][1],
-      oninput: e => {
+      oninput: (e) => {
         e.redraw = false;
         task.parameterValues[0][1] = input.dom.value;
       },
       onkeydown: keydown,
-      oncreate: vnode => {
+      oncreate: (vnode) => {
         (vnode.dom as HTMLInputElement).focus();
         (vnode.dom as HTMLInputElement).select();
         // Need to prevent scrolling on focus because
         // we're animating height and using overflow: hidden
         (vnode.dom.parentNode.parentNode as Element).scrollTop = 0;
-      }
+      },
     });
   }
 
   return [m("span", "Editing ", mparam(task.parameterValues[0][0])), input];
 }
 
-function renderStagingDownload(task): Children {
+function renderStagingDownload(task: StageTask): Children {
-  task.invalid = !task.fileName || !task.fileType;
+  if (!task.fileName || !task.fileType) invalid.add(task);
+  else invalid.delete(task);
   const files = store.fetch("files", true);
-  const idParts = task.device.split("-");
+  let oui = "";
-  const oui = decodeURIComponent(idParts[0]);
+  let productClass = "";
-  const productClass =
+  for (const d of task.devices) {
-    idParts.length === 3 ? decodeURIComponent(idParts[1]) : "";
+    const parts = d.split("-");
+    if (oui === "") oui = parts[0];
+    else if (oui !== parts[0]) oui = null;
+    if (parts.length === 3) {
+      if (productClass === "") productClass = parts[1];
+      else if (productClass !== parts[1]) productClass = null;
+    }
+  }
+
+  if (oui) oui = decodeURIComponent(oui);
+  if (productClass) productClass = decodeURIComponent(productClass);
 
   const typesList = [
     "",
@@ -104,8 +126,8 @@ function renderStagingDownload(task): Children {
     "2 Web Content",
     "3 Vendor Configuration File",
     "4 Tone File",
-    "5 Ringer File"
+    "5 Ringer File",
-  ].map(t =>
+  ].map((t) =>
     m(
       "option",
       { disabled: !t, value: t, selected: (task.fileType || "") === t },
@@ -117,14 +139,14 @@ function renderStagingDownload(task): Children {
     .concat(
       files.value
         .filter(
-          f =>
+          (f) =>
             (!f["metadata.oui"] || f["metadata.oui"] === oui) &&
             (!f["metadata.productClass"] ||
               f["metadata.productClass"] === productClass)
         )
-        .map(f => f._id)
+        .map((f) => f._id)
     )
-    .map(f =>
+    .map((f) =>
       m(
         "option",
         { disabled: !f, value: f, selected: (task.fileName || "") === f },
@@ -137,7 +159,7 @@ function renderStagingDownload(task): Children {
     m(
       "select",
       {
-        onchange: e => {
+        onchange: (e) => {
           const f = e.target.value;
           task.fileName = f;
           task.fileType = "";
@@ -145,7 +167,7 @@ function renderStagingDownload(task): Children {
             if (file._id === f) task.fileType = file["metadata.fileType"];
         },
         disabled: files.fulfilling,
-        style: "width: 350px"
+        style: "width: 350px",
       },
       filesList
     ),
@@ -153,22 +175,26 @@ function renderStagingDownload(task): Children {
     m(
       "select",
       {
-        onchange: e => {
+        onchange: (e) => {
           task.fileType = e.target.value;
-        }
+        },
       },
       typesList
-    )
+    ),
   ];
 }
 
-function renderStaging(staging): Child[] {
+function renderStaging(staging: Set<StageTask>): Child[] {
   const elements: Child[] = [];
 
   for (const s of staging) {
     const queueFunc = (): void => {
       staging.delete(s);
-      taskQueue.queueTask(s);
+      for (const d of s.devices) {
+        const t = Object.assign({ device: d }, s);
+        delete t.devices;
+        queueTask(t);
+      }
     };
     const cancelFunc = (): void => {
       staging.delete(s);
@@ -181,7 +207,7 @@ function renderStaging(staging): Child[] {
 
     const queue = m(
       "button.primary",
-      { title: "Queue task", onclick: queueFunc, disabled: s.invalid },
+      { title: "Queue task", onclick: queueFunc, disabled: invalid.has(s) },
       "Queue"
     );
     const cancel = m(
@@ -195,7 +221,7 @@ function renderStaging(staging): Child[] {
   return elements;
 }
 
-function renderQueue(queue): Child[] {
+function renderQueue(queue: Set<QueueTask>): Child[] {
   const details: Child[] = [];
   const devices: { [deviceId: string]: any[] } = {};
   for (const t of queue) {
@@ -215,8 +241,8 @@ function renderQueue(queue): Child[] {
             {
               title: "Retry this task",
               onclick: () => {
-                taskQueue.queueTask(t);
+                queueTask(t);
-              }
+              },
             },
             getIcon("retry")
           )
@@ -229,8 +255,8 @@ function renderQueue(queue): Child[] {
           {
             title: "Remove this task",
             onclick: () => {
-              taskQueue.deleteTask(t);
+              deleteTask(t);
-            }
+            },
           },
           getIcon("remove")
         )
@@ -324,18 +350,18 @@ function renderNotifications(notifs): Child[] {
         {
           class: n.type,
           style: "position: absolute;opacity: 0",
-          oncreate: vnode => {
+          oncreate: (vnode) => {
             (vnode.dom as HTMLDivElement).style.opacity = "1";
           },
-          onbeforeremove: vnode => {
+          onbeforeremove: (vnode) => {
             (vnode.dom as HTMLDivElement).style.opacity = "0";
-            return new Promise(resolve => {
+            return new Promise((resolve) => {
               setTimeout(() => {
                 resolve();
               }, 500);
             });
           },
-          key: n.timestamp
+          key: n.timestamp,
         },
         m("div", buttons, n.message)
       )
@@ -346,9 +372,9 @@ function renderNotifications(notifs): Child[] {
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: vnode => {
+    view: (vnode) => {
-      const queue = taskQueue.getQueue();
+      const queue = getQueue();
-      const staging = taskQueue.getStaging();
+      const staging = getStaging();
       const notifs = notifications.getNotifications();
 
       let drawerElement, statusElement;
@@ -394,11 +420,10 @@ const component: ClosureComponent = (): Component => {
               title: "Commit queued tasks",
               disabled: !statusCount.queued,
               onclick: () => {
-                const tasks = Array.from(taskQueue.getQueue()).filter(
+                const tasks = Array.from(getQueue()).filter(
-                  t => t["status"] === "queued"
+                  (t) => t["status"] === "queued"
                 );
-                taskQueue
+                commit(
-                  .commit(
                   tasks,
                   (deviceId, err, connectionRequestStatus, tasks2) => {
                     if (err) {
@@ -441,8 +466,11 @@ const component: ClosureComponent = (): Component => {
                 )
                   .then(() => {
                     store.fulfill(0, Date.now());
+                  })
+                  .catch((err) => {
+                    notifications.push("error", err.message);
                   });
-              }
+              },
             },
             "Commit"
           ),
@@ -450,8 +478,8 @@ const component: ClosureComponent = (): Component => {
             "button",
             {
               title: "Clear tasks",
-              onclick: taskQueue.clear,
+              onclick: clear,
-              disabled: !queueElements.length
+              disabled: !queueElements.length,
             },
             "Clear"
           )
@@ -487,31 +515,31 @@ const component: ClosureComponent = (): Component => {
           {
             key: "drawer",
             style: "opacity: 0;height: 0;",
-            oncreate: vnode2 => {
+            oncreate: (vnode2) => {
               vnode.state["mouseIn"] = false;
               (vnode2.dom as HTMLDivElement).style.opacity = "1";
               resizeDrawer();
             },
-            onmouseover: e => {
+            onmouseover: (e) => {
               vnode.state["mouseIn"] = true;
               resizeDrawer();
               e.redraw = false;
             },
-            onmouseleave: e => {
+            onmouseleave: (e) => {
               vnode.state["mouseIn"] = false;
               resizeDrawer();
               e.redraw = false;
             },
             onupdate: resizeDrawer,
-            onbeforeremove: vnode2 => {
+            onbeforeremove: (vnode2) => {
               (vnode2.dom as HTMLDivElement).onmouseover = null;
               (vnode2.dom as HTMLDivElement).onmouseleave = null;
               (vnode2.dom as HTMLDivElement).style.opacity = "0";
               (vnode2.dom as HTMLDivElement).style.height = "0";
-              return new Promise(resolve => {
+              return new Promise((resolve) => {
                 setTimeout(resolve, 500);
               });
-            }
+            },
           },
           statusElement,
           stagingElements.length ? stagingElements : m(".queue", queueElements)
@@ -527,12 +555,12 @@ const component: ClosureComponent = (): Component => {
             key: "notifications",
             style: "position: relative;",
             onupdate: repositionNotifications,
-            oncreate: repositionNotifications
+            oncreate: repositionNotifications,
           },
           notificationElements
         )
       );
-    }
+    },
   };
 };