Include notification and accessList in CSV

.eslintrc.json

@@ -21,7 +21,6 @@
     "@typescript-eslint/no-explicit-any": 0,
     "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
     "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
-    "@typescript-eslint/no-floating-promises": "error",
     "require-atomic-updates": 0,
     "no-prototype-builtins": 0
   }

CHANGELOG.md

@@ -1,61 +1,5 @@
 # Change Log
 
-## 1.2.3 (2020-10-26)
-
-- New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
-report writable parameters as non-writable. When set to true, the scripts will
-no longer respect the 'writable' attribute of the CPE parameters and will send
-a SetParamteerValues, AddObject, or DeleteObject request anyway.
-- Tags no longer restrict what characters are allowed. Any character other than
-alphanumeric characters, hyphen, or underscore is now encoded in the data
-model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
-- Ask for a confirmation before closing a pop-up dialog with unsaved changes.
-- Better XML validation to avoid crashes caused by invalid CPE requests.
-- Fix confusing 404 error message when the user attempts to modify a resource
-when they don't have the necessary permissions.
-- Fix a rare issue where genieacs-cwmp stops accepting new connections after
-running for a few weeks.
-- Fix exception when IS NULL operator is used in certain situations.
-
-## 1.2.2 (2020-10-03)
-
-- Added button to push files to selected devices from device listing page.
-- A few minor UI improvements.
-- Fix exception that can happen and persist after a Download request.
-- Fix validation bug preventing running refreshObject task on data model root.
-- Fix invalid arguments fault in refresh preset configuration when upgrading
-from v1.1.
-
-## 1.2.1 (2020-09-08)
-
-- Fix bug causing faults to not be displayed in the UI.
-- Fix bug where deleting objects does not get reflected immediately in the UI.
-- Improve conversion between filters written in the expression format and
-MongoDB queries. There should now be fewer edge cases where the two are not
-equisatisfiable.
-
-## 1.2.0 (2020-09-01)
-
-- Support GetParameterAttributes and SetParameterAttributes TR-069 methods.
-- Support CASE statement and COALESCE function in expressions.
-- Provision arguments can now be a list of expressions that are dynamically
-evaluated.
-- Support Forwarded HTTP header to display in the logs the correct IP of CPEs
-behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
-- Config expressions can now access all available device parameters, not only
-serial number, product class, and OUI.
-- Use relative URLs throughout the UI to allow serving from a subdirectory
-using a reverse proxy.
-- Make Date.parse() and Date.UTC() available to provision scripts.
-- libxmljs has been entirely removed in favor of our bespoke XML parser.
-- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now
-used to determine the TCP connection timeout.
-- The all-parameters component now limits the number of parameters displayed
-for better performance.
-- The process genieacs-cwmp is now much less likely to throw exceptions as a
-result of invalid requests from CPE.
-- A large number of bug fixes and stability improvements.
-
 ## 1.2.0-beta.0 (2019-07-30)
 
 - A brand new UI superseding genieacs-gui.

CONTRIBUTING.md

@@ -1,21 +1,22 @@
 # Have a question?
 
-Please ask in the [forum] instead of opening a Github Issue. The Github issue
-tracker is strictly for bug reports.
+Please ask in the [Users mailing list][mailing-lists] instead of opening a
+Github Issue. The Github issue tracker is strictly for bug reports.
 
 # Issues
 
 We use GitHub issues to track bugs. Please ensure your bug description is
 clear and has sufficient instructions to be able to reproduce the issue.
 
-If you face interoperability issues with your CPE, more often than not it's a
-device related issue. Please consult the [forum] before submitting an Issue.
+If you face interoperability issues with your device, more often than not it's
+a device related issue. Please consult the [Users mailing list][mailing-lists]
+before submitting an Issue.
 
 # Pull Requests
 
 Pull requests are welcome. If you have a fix for a bug, go ahead and create a
-pull request. For other kinds of enhancements, please discuss in the [forum]
-first.
+pull request. For other kinds of enhancements, please discuss in the [Users
+mailing list][mailing-lists] first.
 
 Please follow the guidelines [here][commit-guidelines] when committing your
 changes.
@@ -35,4 +36,4 @@ source code contained therein, and that your content does not infringe upon
 the intellectual property rights of a third party.
 
 [commit-guidelines]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
-[forum]: https://forum.genieacs.com
+[mailing-lists]: http://lists.genieacs.com

README.md

@@ -8,31 +8,36 @@ millions of concurrent devices.
 
 ## Quick Start
 
+*Important: This is a pre-release branch. Use
+[v1.1](https://github.com/genieacs/genieacs/tree/v1.1) for production
+deployments.*
+
 Install [Node.js](http://nodejs.org/) and [MongoDB](http://www.mongodb.org/).
 Refer to their corresponding documentation for installation instructions. The
 supported versions are:
 
-- Node.js: 12.3+
-- MongoDB: 3.6+
+- Node.js: 10.x through 12.x
+- MongoDB: 2.6 through 4.1
 
-Install GenieACS from NPM:
-
-    sudo npm install -g --unsafe-perm genieacs
-
-To build from source instead, clone this repo or download the source archive
-then *cd* into the source directory then run:
+Clone this repo or download the source archive then *cd* into the source
+directory and install the required dependencies:
 
     npm install
+
+Now build:
+
     npm run build
 
-Finally, run the following services (found under `./dist/bin/` if building from
-source):
+Finally, run the following services from the build that's generated under
+'dist' directory:
 
 ### genieacs-cwmp
 
 This is the service that the CPEs will communicate with. It listens on port
 7547 by default. Configure the ACS URL in your devices accordingly.
 
+    ./dist/bin/genieacs-cwmp
+
 You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
 as a dummy TR-069 simulator if you don't have a CPE at hand.
 
@@ -42,36 +47,40 @@ This is the northbound interface module. It exposes a REST API on port 7557 by
 default. This one is only required if you have an external system integrating
 with GenieACS using this API.
 
+    ./dist/bin/genieacs-nbi
+
 ### genieacs-fs
 
 This is the file server from which the CPEs will download firmware images and
-such. It listens on port 7567 by default.
+such.
+
+    ./dist/bin/genieacs-fs
 
 ### genieacs-ui
 
 This serves the web based user interface. It listens on port 3000 by default.
-You must pass *--ui-jwt-secret* argument to supply the secret key used for
-signing browser cookies:
 
-    genieacs-ui --ui-jwt-secret secret
+    ./dist/bin/genieacs-ui --ui-jwt-secret secret
+
+The argument *--ui-jwt-secret* supplies the key used for signing browser
+cookies.
 
 The UI has plenty of configuration options. When you open GenieACS's UI in a
 browser you'll be greeted with a database initialization wizard to help you
 populate some initial configuration.
 
-Visit [docs.genieacs.com](https://docs.genieacs.com) for more documentation and
-a complete installation guide for production deployments.
-
 ## Support
 
 The [forum](https://forum.genieacs.com) is a good place to get guidance and
-help from the community. Head on over and join the conversation!
+help from the community. Head on over and join the conversation! In addition,
+the [wiki](https://github.com/genieacs/genieacs/wiki) provides useful
+documentation and tips from GenieACS users.
 
-For commercial support options, please visit
+For commercial support options and professional services, please visit
 [genieacs.com](https://genieacs.com/support/).
 
 ## License
 
-Copyright 2013-2020 GenieACS Inc. GenieACS is released under the [AGPLv3
+Copyright 2013-2019 GenieACS Inc. GenieACS is released under the [AGPLv3
 license
 terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

ava.config.js

@@ -1,4 +1,5 @@
 export default {
+  "compileEnhancements": false,
   "extensions": ["ts"],
   "require": ["esm", "ts-node/register/transpile-only"]
 };

bin/genieacs-cwmp

@@ -1,3 +1,5 @@
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
+
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -24,35 +26,37 @@ import * as server from "../lib/server";
 import * as cwmp from "../lib/cwmp";
 import * as db from "../lib/db";
 import * as extensions from "../lib/extensions";
+import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("cwmp", VERSION);
 
-const SERVICE_ADDRESS = config.get("CWMP_INTERFACE") as string;
-const SERVICE_PORT = config.get("CWMP_PORT") as number;
+const SERVICE_ADDRESS = config.get("CWMP_INTERFACE");
+const SERVICE_PORT = config.get("CWMP_PORT");
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
+    cache.disconnect(),
     extensions.killAll(),
-    cluster.worker.disconnect(),
+    cluster.worker.disconnect()
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
+  extensions.killAll().then(() => {
     process.exit(1);
   });
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES") as number;
+  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES");
 
   logger.info({
     message: `genieacs-cwmp starting`,
     pid: process.pid,
-    version: VERSION,
+    version: VERSION
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -60,7 +64,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid,
+      pid: process.pid
     });
 
     cluster.stop();
@@ -69,40 +73,42 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid,
+      pid: process.pid
     });
 
     cluster.stop();
   });
 } else {
-  const key = config.get("CWMP_SSL_KEY") as string;
-  const cert = config.get("CWMP_SSL_CERT") as string;
-
-  const options = {
-    port: SERVICE_PORT,
-    host: SERVICE_ADDRESS,
-    ssl: key && cert ? { key, cert } : null,
-    onConnection: cwmp.onConnection,
-    timeout: 30000,
-    keepAliveTimeout: 0,
+  const ssl = {
+    key: config.get("CWMP_SSL_KEY"),
+    cert: config.get("CWMP_SSL_CERT")
   };
 
-  process.on("uncaughtException", (err) => {
+  process.on("uncaughtException", err => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid,
+      pid: process.pid
     });
-    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
   });
 
-  const initPromise = db
-    .connect()
+  const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
-      server.start(options, cwmp.listener);
+      server.start(
+        SERVICE_PORT,
+        SERVICE_ADDRESS,
+        ssl,
+        cwmp.listener,
+        cwmp.onConnection,
+        0
+      );
     })
-    .catch((err) => {
+    .catch(err => {
       setTimeout(() => {
         throw err;
       });
@@ -110,13 +116,19 @@ if (!cluster.worker) {
 
   process.on("SIGINT", () => {
     initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
     initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 }

bin/genieacs-ext

@@ -1,3 +1,5 @@
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
+
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -33,8 +35,8 @@ function errorToFault(err: Error): Fault {
     message: err.message,
     detail: {
       name: err.name,
-      message: err.message,
-    },
+      message: err.message
+    }
   };
 
   if (err.stack) {
@@ -54,16 +56,16 @@ function errorToFault(err: Error): Fault {
   return fault;
 }
 
-process.on("uncaughtException", (err) => {
+process.on("uncaughtException", err => {
   const fault = errorToFault(err);
-  jobs.forEach((jobId) => {
+  jobs.forEach(jobId => {
     process.send([jobId, fault, null]);
   });
   jobs.clear();
   process.disconnect();
 });
 
-process.on("message", (message) => {
+process.on("message", message => {
   jobs.add(message[0]);
 
   if (!script) {
@@ -77,7 +79,7 @@ process.on("message", (message) => {
   if (!script[funcName]) {
     const fault = {
       code: "ext",
-      message: `No such function '${funcName}' in extension '${fileName}'`,
+      message: `No such function '${funcName}' in extension '${fileName}'`
     };
     process.send([message[0], fault, null]);
     return;
@@ -91,6 +93,4 @@ process.on("message", (message) => {
 });
 
 // Ignore SIGINT
-process.on("SIGINT", () => {
-  // Ignore
-});
+process.on("SIGINT", () => {});

bin/genieacs-fs

@@ -1,3 +1,5 @@
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
+
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -23,18 +25,21 @@ import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import { listener } from "../lib/fs";
 import * as db from "../lib/db";
+import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("fs", VERSION);
 
-const SERVICE_ADDRESS = config.get("FS_INTERFACE") as string;
-const SERVICE_PORT = config.get("FS_PORT") as number;
+const SERVICE_ADDRESS = config.get("FS_INTERFACE");
+const SERVICE_PORT = config.get("FS_PORT");
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
-  Promise.all([db.disconnect(), cluster.worker.disconnect()]).catch(
-    exitWorkerUngracefully
-  );
+  Promise.all([
+    db.disconnect(),
+    cache.disconnect(),
+    cluster.worker.disconnect()
+  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
@@ -42,12 +47,12 @@ function exitWorkerUngracefully(): void {
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES") as number;
+  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES");
 
   logger.info({
     message: `genieacs-fs starting`,
     pid: process.pid,
-    version: VERSION,
+    version: VERSION
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -55,7 +60,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid,
+      pid: process.pid
     });
 
     cluster.stop();
@@ -64,63 +69,55 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid,
+      pid: process.pid
     });
 
     cluster.stop();
   });
 } else {
-  const key = config.get("FS_SSL_KEY") as string;
-  const cert = config.get("FS_SSL_CERT") as string;
-  const options = {
-    port: SERVICE_PORT,
-    host: SERVICE_ADDRESS,
-    ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
+  const ssl = {
+    key: config.get("FS_SSL_KEY"),
+    cert: config.get("FS_SSL_CERT")
   };
 
-  let stopping = false;
-
-  process.on("uncaughtException", (err) => {
+  process.on("uncaughtException", err => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid,
+      pid: process.pid
     });
-    stopping = true;
-    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
   });
 
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    listener(req, res).catch((err) => {
-      throw err;
-    });
-  };
-
-  const initPromise = db
-    .connect()
+  const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
-      server.start(options, _listener);
+      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
     })
-    .catch((err) => {
+    .catch(err => {
       setTimeout(() => {
         throw err;
       });
     });
 
   process.on("SIGINT", () => {
-    stopping = true;
     initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
-    stopping = true;
     initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 }

bin/genieacs-nbi

@@ -1,3 +1,5 @@
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
+
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -24,35 +26,37 @@ import * as server from "../lib/server";
 import { listener } from "../lib/nbi";
 import * as db from "../lib/db";
 import * as extensions from "../lib/extensions";
+import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("nbi", VERSION);
 
-const SERVICE_ADDRESS = config.get("NBI_INTERFACE") as string;
-const SERVICE_PORT = config.get("NBI_PORT") as number;
+const SERVICE_ADDRESS = config.get("NBI_INTERFACE");
+const SERVICE_PORT = config.get("NBI_PORT");
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
+    cache.disconnect(),
     extensions.killAll(),
-    cluster.worker.disconnect(),
+    cluster.worker.disconnect()
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
+  extensions.killAll().then(() => {
     process.exit(1);
   });
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES") as number;
+  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES");
 
   logger.info({
     message: `genieacs-nbi starting`,
     pid: process.pid,
-    version: VERSION,
+    version: VERSION
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -60,7 +64,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid,
+      pid: process.pid
     });
 
     cluster.stop();
@@ -69,61 +73,55 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid,
+      pid: process.pid
     });
 
     cluster.stop();
   });
 } else {
-  const key = config.get("NBI_SSL_KEY") as string;
-  const cert = config.get("NBI_SSL_CERT") as string;
-  const options = {
-    port: SERVICE_PORT,
-    host: SERVICE_ADDRESS,
-    ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
+  const ssl = {
+    key: config.get("NBI_SSL_KEY"),
+    cert: config.get("NBI_SSL_CERT")
   };
 
-  let stopping = false;
-
-  process.on("uncaughtException", (err) => {
+  process.on("uncaughtException", err => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid,
+      pid: process.pid
     });
-    stopping = true;
-    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
   });
 
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
-  };
-
-  const initPromise = db
-    .connect()
+  const initPromise = Promise.all([db.connect(), cache.connect()])
     .then(() => {
-      server.start(options, _listener);
+      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
     })
-    .catch((err) => {
+    .catch(err => {
       setTimeout(() => {
         throw err;
       });
     });
 
   process.on("SIGINT", () => {
-    stopping = true;
     initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
-    stopping = true;
     initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 }

bin/genieacs-ui

@@ -1,3 +1,5 @@
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
+
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -23,36 +25,40 @@ import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import { listener } from "../lib/ui";
 import * as extensions from "../lib/extensions";
-import * as db from "../lib/db";
+import * as db from "../lib/ui/db";
+import * as db2 from "../lib/db";
+import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("ui", VERSION);
 
-const SERVICE_ADDRESS = config.get("UI_INTERFACE") as string;
-const SERVICE_PORT = config.get("UI_PORT") as number;
+const SERVICE_ADDRESS = config.get("UI_INTERFACE");
+const SERVICE_PORT = config.get("UI_PORT");
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
+    db2.disconnect(),
+    cache.disconnect(),
     extensions.killAll(),
-    cluster.worker.disconnect(),
+    cluster.worker.disconnect()
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
+  extensions.killAll().then(() => {
     process.exit(1);
   });
 }
 
 if (!cluster.worker) {
-  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES") as number;
+  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES");
 
   logger.info({
     message: `genieacs-ui starting`,
     pid: process.pid,
-    version: VERSION,
+    version: VERSION
   });
 
   cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
@@ -60,7 +66,7 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     logger.info({
       message: "Received signal SIGINT, exiting",
-      pid: process.pid,
+      pid: process.pid
     });
 
     cluster.stop();
@@ -69,32 +75,31 @@ if (!cluster.worker) {
   process.on("SIGTERM", () => {
     logger.info({
       message: "Received signal SIGTERM, exiting",
-      pid: process.pid,
+      pid: process.pid
     });
 
     cluster.stop();
   });
 } else {
-  const key = config.get("UI_SSL_KEY") as string;
-  const cert = config.get("UI_SSL_CERT") as string;
-  const options = {
-    port: SERVICE_PORT,
-    host: SERVICE_ADDRESS,
-    ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
+  const ssl = {
+    key: config.get("UI_SSL_KEY"),
+    cert: config.get("UI_SSL_CERT")
   };
 
   let stopping = false;
 
-  process.on("uncaughtException", (err) => {
+  process.on("uncaughtException", err => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
       message: "Uncaught exception",
       exception: err,
-      pid: process.pid,
+      pid: process.pid
     });
     stopping = true;
-    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    server
+      .stop()
+      .then(exitWorkerGracefully)
+      .catch(exitWorkerUngracefully);
   });
 
   const _listener = (req, res): void => {
@@ -102,12 +107,11 @@ if (!cluster.worker) {
     listener(req, res);
   };
 
-  const initPromise = db
-    .connect()
+  const initPromise = Promise.all([db2.connect(), cache.connect()])
     .then(() => {
-      server.start(options, _listener);
+      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
     })
-    .catch((err) => {
+    .catch(err => {
       setTimeout(() => {
         throw err;
       });
@@ -116,14 +120,20 @@ if (!cluster.worker) {
   process.on("SIGINT", () => {
     stopping = true;
     initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
     stopping = true;
     initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+      server
+        .stop()
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 }

build/build.ts

@@ -19,13 +19,13 @@
 
 import * as path from "path";
 import * as fs from "fs";
+import { promisify } from "util";
 import { rollup, WarningHandler } from "rollup";
-import rollupJson from "@rollup/plugin-json";
-import typescript from "@rollup/plugin-typescript";
+import rollupReplace from "rollup-plugin-replace";
+import rollupJson from "rollup-plugin-json";
+import typescript from "rollup-plugin-typescript";
 import { terser } from "rollup-plugin-terser";
-import nodeResolve from "@rollup/plugin-node-resolve";
-import commonjs from "@rollup/plugin-commonjs";
-import webpack, { Configuration } from "webpack";
+import webpack from "webpack";
 import postcss from "postcss";
 import postcssImport from "postcss-import";
 import postcssPresetEnv from "postcss-preset-env";
@@ -55,6 +55,7 @@ const externals = [
   "crypto",
   "util",
   "mongodb",
+  "libxmljs",
   "vm",
   "later",
   "parsimmon",
@@ -77,10 +78,7 @@ const externals = [
   "yaml",
   "codemirror",
   "codemirror/mode/javascript/javascript",
-  "codemirror/mode/yaml/yaml",
-  "ipaddr.js",
-  "jsbi",
-  "espresso-iisojs",
+  "codemirror/mode/yaml/yaml"
 ];
 
 function rmDirSync(dirPath): void {
@@ -125,7 +123,9 @@ function generateSymbol(id: string, svgStr: string): string {
     }
   }
   const symbolBody = xml.children[0].children
-    .map((c) => xmlTostring(c))
+    .map(c => {
+      return xmlTostring(c);
+    })
     .join("");
   return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
 }
@@ -147,7 +147,7 @@ async function init(): Promise<void> {
   delete packageJson["devDependencies"];
   packageJson["scripts"] = {
     install: packageJson["scripts"].install,
-    configure: packageJson["scripts"].configure,
+    configure: packageJson["scripts"].configure
   };
   packageJson["version"] = `${packageJson["version"]}+${BUILD_METADATA}`;
   fs.writeFileSync(
@@ -173,7 +173,7 @@ async function copyStatic(): Promise<void> {
     "README.md",
     "CHANGELOG.md",
     "public/logo.svg",
-    "public/favicon.png",
+    "public/favicon.png"
   ];
 
   for (const file of files) {
@@ -194,31 +194,35 @@ async function generateCss(): Promise<void> {
       stage: 3,
       features: {
         "nesting-rules": true,
-        "color-mod-function": true,
-      },
+        "color-mod-function": true
+      }
     }),
-    cssnano,
+    cssnano
   ]).process(cssIn, { from: cssInPath, to: cssOutPath });
   fs.writeFileSync(cssOutPath, cssOut.css);
 }
 
 async function generateToolsJs(): Promise<void> {
   for (const bin of ["dump-data-model"]) {
-    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}.ts`);
+    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}`);
     const outputFile = path.resolve(OUTPUT_DIR, `tools/${bin}`);
     const bundle = await rollup({
       input: inputFile,
       external: externals,
       acorn: {
-        allowHashBang: true,
+        allowHashBang: true
       },
       plugins: [
+        rollupReplace({
+          delimiters: ["", ""],
+          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
+        }),
         typescript({
           tsconfig: "./tsconfig.json",
-          include: [`tools/${bin}.ts`, "lib/**/*.ts"],
+          include: [`tools/${bin}`, "lib/**/*.ts"]
         }),
-        MODE === "production" ? terser() : null,
-      ],
+        MODE === "production" ? terser() : null
+      ]
     });
 
     await bundle.write({
@@ -227,7 +231,7 @@ async function generateToolsJs(): Promise<void> {
       sourcemap: "inline",
       sourcemapExcludeSources: true,
       banner: "#!/usr/bin/env node",
-      file: outputFile,
+      file: outputFile
     });
 
     // Mark as executable
@@ -242,24 +246,27 @@ async function generateBackendJs(): Promise<void> {
     "genieacs-ext",
     "genieacs-nbi",
     "genieacs-fs",
-    "genieacs-ui",
+    "genieacs-ui"
   ]) {
-    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}.ts`);
+    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}`);
     const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
     const bundle = await rollup({
       input: inputFile,
       external: externals,
       acorn: {
-        allowHashBang: true,
+        allowHashBang: true
       },
       treeshake: {
         propertyReadSideEffects: false,
-        moduleSideEffects: false,
+        pureExternalModules: true
       },
       plugins: [
+        rollupReplace({
+          delimiters: ["", ""],
+          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
+        }),
         rollupJson({ preferConst: true }),
         {
-          name: "",
           resolveId: (importee, importer) => {
             if (importee.endsWith("/package.json")) {
               const p = path.resolve(path.dirname(importer), importee);
@@ -267,14 +274,14 @@ async function generateBackendJs(): Promise<void> {
                 return path.resolve(OUTPUT_DIR, "package.json");
             }
             return null;
-          },
+          }
         },
         typescript({
           tsconfig: "./tsconfig.json",
-          include: [`bin/${bin}.ts`, "lib/**/*.ts"],
+          include: [`bin/${bin}`, "lib/**/*.ts"]
         }),
-        MODE === "production" ? terser() : null,
-      ],
+        MODE === "production" ? terser() : null
+      ]
     });
 
     await bundle.write({
@@ -283,7 +290,7 @@ async function generateBackendJs(): Promise<void> {
       sourcemap: "inline",
       sourcemapExcludeSources: true,
       banner: "#!/usr/bin/env node",
-      file: outputFile,
+      file: outputFile
     });
 
     // Mark as executable
@@ -296,54 +303,37 @@ async function generateFrontendJs(): Promise<void> {
   const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
   const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
 
-  const inlineDeps = ["mithril", "parsimmon", "jsbi", "espresso-iisojs"];
   const bundle = await rollup({
     input: inputFile,
-    external: externals.filter((e) => !inlineDeps.includes(e)),
+    external: externals,
     plugins: [
       rollupJson({ preferConst: true }),
-      {
-        name: "",
-        resolveId: function (importee, importer) {
-          if (importee.endsWith("/bigint")) {
-            return this.resolve(importee + "-jsbi", importer);
-          } else if (importee === "espresso-iisojs") {
-            return this.resolve(
-              "espresso-iisojs/dist/espresso-iisojs-jsbi.mjs",
-              importer
-            );
-          }
-          return null;
-        },
-      },
-      typescript({ tsconfig: "./tsconfig.json" }),
-      nodeResolve(),
-      commonjs(),
+      typescript({ tsconfig: "./tsconfig.json" })
     ],
     inlineDynamicImports: true,
     treeshake: {
       propertyReadSideEffects: false,
-      moduleSideEffects: false,
+      pureExternalModules: true
     },
     onwarn: ((warning, warn) => {
       // Ignore circular dependency warnings
       if (warning.code !== "CIRCULAR_DEPENDENCY") warn(warning);
-    }) as WarningHandler,
+    }) as WarningHandler
   });
 
   await bundle.write({
     preferConst: true,
-    format: "es",
+    format: "esm",
     sourcemap: "inline",
     sourcemapExcludeSources: true,
-    file: outputFile,
+    file: outputFile
   });
 
-  const webpackConf: Configuration = {
-    mode: MODE as "development" | "production",
+  const webpackConf = {
+    mode: MODE,
     entry: outputFile,
     resolve: {
-      aliasFields: ["module"],
+      aliasFields: ["module"]
     },
     devtool: "nosources-source-map",
     module: {
@@ -351,23 +341,18 @@ async function generateFrontendJs(): Promise<void> {
         {
           test: /\.js$/,
           use: ["source-map-loader"],
-          enforce: "pre",
-        },
-      ],
+          enforce: "pre"
+        }
+      ]
     },
     output: {
       path: path.resolve(OUTPUT_DIR, "public"),
-      filename: "app.js",
-    },
+      filename: "app.js"
+    }
   };
 
-  return new Promise((resolve, reject) => {
-    webpack(webpackConf, (err, stats) => {
-      if (err) return reject(err);
+  const stats = await promisify(webpack)(webpackConf);
   process.stdout.write(stats.toString({ colors: true }) + "\n");
-      resolve();
-    });
-  });
 }
 
 async function generateIconsSprite(): Promise<void> {
@@ -394,15 +379,13 @@ init()
       generateIconsSprite(),
       generateToolsJs(),
       generateBackendJs(),
-      generateFrontendJs(),
+      generateFrontendJs()
     ])
-      .then(() => {
-        // Ignore
-      })
-      .catch((err) => {
+      .then(() => {})
+      .catch(err => {
         process.stderr.write(err.stack + "\n");
       });
   })
-  .catch((err) => {
+  .catch(err => {
     process.stderr.write(err.stack + "\n");
   });

docs/environment-variables.rst

@@ -164,13 +164,12 @@ FS_ACCESS_LOG_FILE
 
   Default: unset
 
-FS_URL_PREFIX
-  The URL prefix (e.g. 'https://example.com:7657/') to use when generating the
-  file URL for TR-069 Download requests. Set this if genieacs-fs and
-  genieacs-cwmp are behind a proxy or running on different servers.
+FS_HOSTNAME
+  The hostname to use when generating the file URL for TR-069 Download
+  requests. Set this if the hostname where genieacs-fs is listening differs
+  from the hostname in the ACS URL.
 
-  Default: auto generated based on the hostname from the ACS URL, FS_PORT
-  config, and whether or not SSL is enabled for genieacs-fs.
+  Default: the hostname from the ACS URL
 
 UI_WORKER_PROCESSES
   The number of worker processes to spawn for genieacs-ui. A value of 0 means

docs/virtual-parameters.rst

@@ -18,8 +18,8 @@ differences between the scripts of provisions and virtual parameters are:
 
 .. code:: javascript
 
-  // [<declared attr timestamps, declared attr values>, <current attr timestamps>, <current attr values>]
-  [{path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}, {path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}]
+  // [<current attr timestamps>, <current attr values>, <declared attr timestamps, declared attr values>]
+  [{path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}, {path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}]
 
 - Virtual parameter scripts must return an object containing the attributes of
   this parameter.
@@ -69,9 +69,9 @@ Expose an external value as a virtual parameter
 
   // Example: Expose an external value as a virtual parameter
   let serial = declare("DeviceID.SerialNumber", {value: 1});
-  if (args[1].value) {
-    ext("example-ext", "set", serial.value[0], args[1].value[0]);
-    return {writable: true, value: [args[1].value[0], "xsd:string"]};
+  if (values.value) {
+    ext("example-ext", "set", serial.value[0], values.value[0]);
+    return {writable: true, value: [values.value[0], "xsd:string"]};
   }
   else {
     let v = ext("example-ext", "get", serial.value[0]);
@@ -85,8 +85,8 @@ Create an editable virtual parameter for WPA passphrase
 
   // Example: Create an editable virtual parameter for WPA passphrase
   let m = "";
-  if (args[1].value) {
-    m = args[1].value[0];
+  if (VALUES && VALUES.value) {
+    m = VALUES.value[0];
     declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", null, {value: m});
     declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", null, {value: m});
   }

lib/api-functions.ts

@@ -24,84 +24,64 @@ import * as cache from "./cache";
 import {
   getCurrentSnapshot,
   getConfig,
-  getConfigExpression,
+  getConfigExpression
 } from "./local-cache";
 import {
   httpConnectionRequest,
-  udpConnectionRequest,
+  udpConnectionRequest
 } from "./connection-request";
 import { Expression, Task } from "./types";
-import { flattenDevice } from "./mongodb-functions";
-import { evaluate } from "./common/expression";
 
-export async function connectionRequest(
-  deviceId: string,
-  device?: Record<string, { value?: [boolean | number | string, string] }>
-): Promise<string> {
-  if (!device) {
-    const res = await db.devicesCollection.findOne({ _id: deviceId });
-    if (!res) throw new Error("No such device");
-    device = flattenDevice(res);
+export async function connectionRequest(deviceId): Promise<void> {
+  const options = {
+    projection: {
+      _deviceId: 1,
+      "Device.ManagementServer.ConnectionRequestURL._value": 1,
+      "Device.ManagementServer.UDPConnectionRequestAddress._value": 1,
+      "Device.ManagementServer.ConnectionRequestUsername._value": 1,
+      "Device.ManagementServer.ConnectionRequestPassword._value": 1,
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value": 1,
+      "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress._value": 1,
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername._value": 1,
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword._value": 1
     }
+  };
 
-  let connectionRequestUrl, udpConnectionRequestAddress, username, password;
+  const device = await db.devicesCollection.findOne({ _id: deviceId }, options);
+  if (!device) throw new Error("No such device");
 
-  if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
-    connectionRequestUrl = (device[
-      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL"
-    ].value || [""])[0];
-    udpConnectionRequestAddress = ((
-      device[
-        "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
-      ] || {}
-    ).value || [""])[0];
-    username = ((
-      device[
-        "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
-      ] || {}
-    ).value || [""])[0];
-    password = ((
-      device[
-        "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword"
-      ] || {}
-    ).value || [""])[0];
-  } else {
-    connectionRequestUrl = (device[
-      "Device.ManagementServer.ConnectionRequestURL"
-    ].value || [""])[0];
-    udpConnectionRequestAddress = ((
-      device["Device.ManagementServer.UDPConnectionRequestAddress"] || {}
-    ).value || [""])[0];
-    username = ((
-      device["Device.ManagementServer.ConnectionRequestUsername"] || {}
-    ).value || [""])[0];
-    password = ((
-      device["Device.ManagementServer.ConnectionRequestPassword"] || {}
-    ).value || [""])[0];
+  let managementServer,
+    connectionRequestUrl,
+    udpConnectionRequestAddress,
+    username,
+    password;
+  if (device.Device)
+    // TR-181 data model
+    managementServer = device.Device.ManagementServer;
+  // TR-098 data model
+  else managementServer = device.InternetGatewayDevice.ManagementServer;
+
+  if (managementServer.ConnectionRequestURL)
+    connectionRequestUrl = managementServer.ConnectionRequestURL._value;
+  if (managementServer.UDPConnectionRequestAddress) {
+    udpConnectionRequestAddress =
+      managementServer.UDPConnectionRequestAddress._value;
   }
+  if (managementServer.ConnectionRequestUsername)
+    username = managementServer.ConnectionRequestUsername._value;
+  if (managementServer.ConnectionRequestPassword)
+    password = managementServer.ConnectionRequestPassword._value;
 
-  const remoteAddress = parse(connectionRequestUrl).host;
-
-  const evalCallback = (exp): Expression => {
-    if (!Array.isArray(exp)) return exp;
-    if (exp[0] === "PARAM" && typeof exp[1] === "string") {
-      let name = exp[1];
-      if (name === "id") name = "DeviceID.ID";
-      else if (name === "serialNumber") name = "DeviceID.SerialNumber";
-      else if (name === "productClass") name = "DeviceID.ProductClass";
-      else if (name === "oui") name = "DeviceID.OUI";
-      else if (name === "remoteAddress") return remoteAddress;
-      else if (name === "username") return username;
-      else if (name === "password") return password;
-
-      const p = device[name];
-      if (p && p.value) return p.value[0];
-    } else if (exp[0] === "FUNC") {
-      if (exp[1] === "REMOTE_ADDRESS") return remoteAddress;
-      else if (exp[1] === "USERNAME") return username;
-      else if (exp[1] === "PASSWORD") return password;
-    }
-    return exp;
+  const context = {
+    id: device["_id"],
+    serialNumber: device["_deviceId"]["SerialNumber"],
+    productClass: device["_deviceId"]["ProductClass"],
+    oui: device["_deviceId"]["OUI"],
+    remoteAddress: connectionRequestUrl
+      ? parse(connectionRequestUrl).host
+      : null,
+    username: username || "",
+    password: password || ""
   };
 
   const snapshot = await getCurrentSnapshot();
@@ -109,23 +89,20 @@ export async function connectionRequest(
   const UDP_CONNECTION_REQUEST_PORT = +getConfig(
     snapshot,
     "cwmp.udpConnectionRequestPort",
-    {},
-    now,
-    evalCallback
+    context,
+    now
   );
   const CONNECTION_REQUEST_TIMEOUT = +getConfig(
     snapshot,
     "cwmp.connectionRequestTimeout",
-    {},
-    now,
-    evalCallback
+    context,
+    now
   );
   const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = !!getConfig(
     snapshot,
     "cwmp.connectionRequestAllowBasicAuth",
-    {},
-    now,
-    evalCallback
+    context,
+    now
   );
   let authExp: Expression = getConfigExpression(
     snapshot,
@@ -137,79 +114,64 @@ export async function connectionRequest(
       "FUNC",
       "AUTH",
       ["PARAM", "username"],
-      ["PARAM", "password"],
+      ["PARAM", "password"]
     ] as Expression;
   }
 
-  authExp = evaluate(authExp, {}, now, evalCallback);
-
-  const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
+  const debug = !!getConfig(snapshot, "cwmp.debug", context, now);
 
   let udpProm;
   if (udpConnectionRequestAddress) {
     udpProm = udpConnectionRequest(
       udpConnectionRequestAddress,
       authExp,
+      context,
       UDP_CONNECTION_REQUEST_PORT,
       debug,
       deviceId
     );
   }
 
-  const status = await httpConnectionRequest(
+  try {
+    await httpConnectionRequest(
       connectionRequestUrl,
       authExp,
+      context,
       CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
       CONNECTION_REQUEST_TIMEOUT,
       debug,
       deviceId
     );
-
-  if (udpProm) {
+  } catch (err) {
+    if (!udpProm) throw err;
     await udpProm;
-    return "";
+  }
 }
 
-  return status;
-}
+export async function watchTask(deviceId, taskId, timeout): Promise<string> {
+  await new Promise(resolve => setTimeout(resolve, 500));
 
-export async function awaitSessionStart(
-  deviceId: string,
-  lastInform: number,
-  timeout: number
-): Promise<boolean> {
-  const now = Date.now();
-  const device = await db.devicesCollection.findOne(
-    { _id: deviceId },
-    { projection: { _lastInform: 1 } }
+  const task = await db.tasksCollection.findOne(
+    { _id: taskId },
+    { projection: { _id: 1 } }
   );
-  const li = (device["_lastInform"] as Date).getTime();
-  if (li > lastInform) return true;
-  const token = await cache.get(`cwmp_session_${deviceId}`);
-  if (token) return true;
-  if (timeout < 500) return false;
-  await new Promise((resolve) => setTimeout(resolve, 500));
-  timeout -= Date.now() - now;
-  return awaitSessionStart(deviceId, lastInform, timeout);
-}
+  if (!task) return "completed";
 
-export async function awaitSessionEnd(
-  deviceId: string,
-  timeout: number
-): Promise<boolean> {
-  const now = Date.now();
-  const token = await cache.get(`cwmp_session_${deviceId}`);
-  if (!token) return true;
-  if (timeout < 500) return false;
-  await new Promise((resolve) => setTimeout(resolve, 500));
-  timeout -= Date.now() - now;
-  return awaitSessionEnd(deviceId, timeout);
+  const q = { _id: `${deviceId}:task_${taskId}` };
+  const fault = await db.faultsCollection.findOne(q, {
+    projection: { _id: 1 }
+  });
+  if (fault) return "fault";
+
+  if ((timeout -= 500) <= 0) return "timeout";
+
+  return watchTask(deviceId, taskId, timeout);
 }
 
 function sanitizeTask(task): void {
   task.timestamp = new Date(task.timestamp || Date.now());
   if (task.expiry) {
-    if (task.expiry instanceof Date || isNaN(task.expiry))
+    if (common.typeOf(task.expiry) === common.DATE_TYPE || isNaN(task.expiry))
       task.expiry = new Date(task.expiry);
     else task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000);
   }
@@ -247,10 +209,6 @@ function sanitizeTask(task): void {
       break;
 
     case "refreshObject":
-      if (typeof task.objectName !== "string")
-        throw new Error("Missing 'objectName' property");
-      break;
-
     case "deleteObject":
       if (typeof task.objectName !== "string" || !task.objectName.length)
         throw new Error("Missing 'objectName' property");
@@ -287,10 +245,9 @@ function sanitizeTask(task): void {
     case "provisions":
       if (
         !Array.isArray(task.provisions) ||
-        !task.provisions.every((arr) =>
+        !task.provisions.every(arr =>
           arr.every(
-            (s) =>
-              s == null || ["boolean", "number", "string"].includes(typeof s)
+            s => s == null || ["boolean", "number", "string"].includes(typeof s)
           )
         )
       )
@@ -301,8 +258,8 @@ function sanitizeTask(task): void {
   return task;
 }
 
-export async function insertTasks(tasks: any[]): Promise<Task[]> {
-  if (tasks && !Array.isArray(tasks)) tasks = [tasks];
+export async function insertTasks(tasks): Promise<Task[]> {
+  if (tasks && common.typeOf(tasks) !== common.ARRAY_TYPE) tasks = [tasks];
   else if (!tasks || tasks.length === 0) return tasks || [];
 
   for (const task of tasks) {
@@ -310,7 +267,7 @@ export async function insertTasks(tasks: any[]): Promise<Task[]> {
     if (task.uniqueKey) {
       await db.tasksCollection.deleteOne({
         device: task.device,
-        uniqueKey: task.uniqueKey,
+        uniqueKey: task.uniqueKey
       });
     }
   }
@@ -318,21 +275,20 @@ export async function insertTasks(tasks: any[]): Promise<Task[]> {
   return tasks;
 }
 
-export async function deleteDevice(deviceId: string): Promise<void> {
+export async function deleteDevice(deviceId): Promise<void> {
   await Promise.all([
     db.tasksCollection.deleteMany({ device: deviceId }),
     db.devicesCollection.deleteOne({ _id: deviceId }),
     db.faultsCollection.deleteMany({
       _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
-      },
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`
+      }
     }),
     db.operationsCollection.deleteMany({
       _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
-      },
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`
+      }
     }),
-    cache.del(`${deviceId}_tasks_faults_operations`),
-    db.deleteDeviceUploads(deviceId),
+    cache.del(`${deviceId}_tasks_faults_operations`)
   ]);
 }

lib/auth.ts

@@ -19,17 +19,15 @@
 
 import { createHash, randomBytes, pbkdf2 } from "crypto";
 
-function parseHeaderFeilds(str: string): Record<string, string> {
+function parseHeaderFeilds(str: string): {} {
   const res = {};
   const parts = str.split(",");
 
-  let part: string;
+  let part;
   while ((part = parts.shift()) != null) {
     const name = part.split("=", 1)[0];
-    if (name.length === part.length) {
-      if (!part.trim()) continue;
+    if (name.length === part.length)
       throw new Error("Unable to parse auth header");
-    }
 
     let value = part.slice(name.length + 1);
     if (!/^\s*"/.test(value)) {
@@ -52,9 +50,7 @@ function parseHeaderFeilds(str: string): Record<string, string> {
   return res;
 }
 
-export function parseAuthorizationHeader(
-  authHeader: string
-): { method: string } {
+export function parseAuthorizationHeader(authHeader): { method: string } {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };
@@ -76,9 +72,7 @@ export function parseAuthorizationHeader(
   return res;
 }
 
-export function parseWwwAuthenticateHeader(
-  authHeader: string
-): Record<string, string> {
+export function parseWwwAuthenticateHeader(authHeader): {} {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };
@@ -86,7 +80,7 @@ export function parseWwwAuthenticateHeader(
   return res;
 }
 
-export function basic(username: string, password: string): string {
+export function basic(username, password): string {
   return "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
 }
 
@@ -103,12 +97,20 @@ export function digest(
   nc?: string | Buffer
 ): string {
   const ha1 = createHash("md5");
-  ha1.update(username).update(":").update(realm).update(":").update(password);
+  ha1
+    .update(username)
+    .update(":")
+    .update(realm)
+    .update(":")
+    .update(password);
   // TODO support "MD5-sess" algorithm directive
   const ha1d = ha1.digest("hex");
 
   const ha2 = createHash("md5");
-  ha2.update(httpMethod).update(":").update(uri);
+  ha2
+    .update(httpMethod)
+    .update(":")
+    .update(uri);
 
   if (qop === "auth-int") {
     const bodyHash = createHash("md5")
@@ -120,7 +122,10 @@ export function digest(
   const ha2d = ha2.digest("hex");
 
   const hash = createHash("md5");
-  hash.update(ha1d).update(":").update(nonce);
+  hash
+    .update(ha1d)
+    .update(":")
+    .update(nonce);
   if (qop) {
     hash
       .update(":")
@@ -141,7 +146,7 @@ export function solveDigest(
   uri: string | Buffer,
   httpMethod: string | Buffer,
   body: string | Buffer,
-  authHeader: Record<string, string>
+  authHeader
 ): string {
   const cnonce = randomBytes(8).toString("hex");
   const nc = "00000001";
@@ -178,7 +183,7 @@ export function solveDigest(
   return authString;
 }
 
-export function generateSalt(length: number): Promise<string> {
+export function generateSalt(length): Promise<string> {
   return new Promise((resolve, reject) => {
     randomBytes(length, (err, rand) => {
       if (err) return void reject(err);
@@ -187,7 +192,7 @@ export function generateSalt(length: number): Promise<string> {
   });
 }
 
-export function hashPassword(pass: string, salt: string): Promise<string> {
+export function hashPassword(pass, salt): Promise<string> {
   return new Promise((resolve, reject) => {
     pbkdf2(pass, salt, 10000, 128, "sha512", (err, hash) => {
       if (err) return void reject(err);

lib/cache.ts

@@ -17,102 +17,124 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Collection } from "mongodb";
-import { onConnect } from "./db";
+import { MongoClient, Collection } from "mongodb";
 import * as config from "./config";
 
-const CLOCK_SKEW_TOLERANCE = 30000;
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let cacheCollection: Collection;
+let clientPromise: Promise<MongoClient>;
+let mongoCollection: Collection;
+let mongoTimeOffset = 0;
 
-onConnect(async (db) => {
-  cacheCollection = db.collection("cache");
-  await cacheCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+export async function connect(): Promise<void> {
+  const MONGODB_CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
+  clientPromise = MongoClient.connect(MONGODB_CONNECTION_URL, {
+    useNewUrlParser: true
   });
-
-export async function get(key: string): Promise<string> {
-  const res = await cacheCollection.findOne({ _id: key });
-  if (res) return res["value"];
-  return null;
+  const db = (await clientPromise).db();
+  mongoCollection = db.collection("cache");
+  await mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+  const now = Date.now();
+  const res = await db.command({ hostInfo: 1 });
+  mongoTimeOffset = res.system.currentTime.getTime() - now;
 }
 
-export async function del(key: string): Promise<void> {
-  await cacheCollection.deleteOne({ _id: key });
+export async function disconnect(): Promise<void> {
+  if (clientPromise) await (await clientPromise).close();
+}
+
+export async function get(key): Promise<any> {
+  const expire = new Date(Date.now() - mongoTimeOffset);
+  if (Array.isArray(key)) {
+    const res = await mongoCollection.find({ _id: { $in: key } }).toArray();
+
+    const indices = {};
+    key.forEach((v, i) => {
+      indices[v] = i;
+    });
+
+    const values = [];
+    res.forEach(r => {
+      if (r["expire"] > expire) values[indices[r["_id"]]] = r["value"];
+    });
+
+    return values;
+  } else {
+    const res = await mongoCollection.findOne({ _id: { $in: [key] } });
+    if (res && res["expire"] > expire) return res["value"];
+    return null;
+  }
+}
+
+export async function del(key): Promise<void> {
+  if (Array.isArray(key))
+    await mongoCollection.deleteMany({ _id: { $in: key } });
+  else await mongoCollection.deleteOne({ _id: key });
 }
 
 export async function set(
   key: string,
-  value: string,
+  value: string | number,
   ttl: number = MAX_CACHE_TTL
 ): Promise<void> {
-  const timestamp = new Date();
-  const expire = new Date(
-    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000
-  );
-  await cacheCollection.replaceOne(
+  const expire = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
+  await mongoCollection.replaceOne(
     { _id: key },
-    { _id: key, value, expire, timestamp },
+    { _id: key, value: value, expire: expire },
     { upsert: true }
   );
 }
 
-export async function pop(key: string): Promise<any> {
-  const res = await cacheCollection.findOneAndDelete({ _id: key });
-  if (res && res["value"]) return res["value"]["value"];
+export async function pop(key): Promise<any> {
+  const res = await mongoCollection.findOneAndDelete({ _id: key });
+
+  if (
+    res &&
+    res["value"] &&
+    +res["value"]["expire"] - (Date.now() - mongoTimeOffset)
+  )
+    return res["value"]["value"];
+
   return null;
 }
 
-export async function acquireLock(
-  lockName: string,
-  ttl: number,
-  timeout = 0,
-  token = Math.random().toString(36).slice(2)
-): Promise<string> {
-  try {
-    const now = Date.now();
-    const r = await cacheCollection.findOneAndUpdate(
-      { _id: lockName, token },
-      {
-        $set: {
-          value: token,
-          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
-        },
-        $currentDate: { timestamp: true },
-      },
-      { upsert: true, returnOriginal: false }
-    );
-    const v = r.value;
-    if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
-      throw new Error("Database clock skew too great");
-  } catch (err) {
-    if (err.code === 11000) {
-      if (timeout > 0) {
-        return new Promise((resolve, reject) => {
-          const w = 50 + Math.random() * 50;
-          setTimeout(() => {
-            acquireLock(lockName, ttl, timeout - w, token).then(
-              resolve,
-              reject
-            );
-          }, w);
-        });
-      }
-      throw new Error("Failed to acquire lock");
-    }
-    throw err;
-  }
+export async function lock(lockName, ttl): Promise<Function> {
+  const token = Math.random()
+    .toString(36)
+    .slice(2);
 
-  return token;
-}
-
-export async function releaseLock(
-  lockName: string,
-  token: string
-): Promise<void> {
-  const res = await cacheCollection.deleteOne({
+  async function unlockOrExtend(extendTtl): Promise<void> {
+    if (!extendTtl) {
+      const res = await mongoCollection.deleteOne({
         _id: lockName,
-    value: token,
+        value: token
       });
       if (res["result"]["n"] !== 1) throw new Error("Lock expired");
+    } else {
+      const expire = new Date(Date.now() - mongoTimeOffset + extendTtl * 1000);
+      const res = await mongoCollection.updateOne(
+        { _id: lockName, value: token },
+        { expire: expire }
+      );
+      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
+    }
+  }
+
+  const expireTest = new Date(Date.now() - mongoTimeOffset);
+  const expireSet = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
+
+  try {
+    await mongoCollection.updateOne(
+      { _id: lockName, expire: { $lte: expireTest } },
+      { $set: { value: token, expire: expireSet } },
+      { upsert: true }
+    );
+  } catch (err) {
+    if (err && err.code === 11000) {
+      await new Promise(resolve => setTimeout(resolve, 200));
+      return lock(lockName, ttl);
+    }
+  }
+
+  return unlockOrExtend;
 }

lib/cluster.ts

@@ -41,7 +41,7 @@ function restartWorker(worker, code, signal): void {
     message: "Worker died",
     pid: worker.process.pid,
     exitCode: null,
-    signal: null,
+    signal: null
   };
 
   if (code != null) msg.exitCode = code;
@@ -57,7 +57,7 @@ function restartWorker(worker, code, signal): void {
     min2 = 0,
     min3 = 0;
 
-  crashes = crashes.filter((n) => {
+  crashes = crashes.filter(n => {
     if (n > now - 60000) ++min1;
     else if (n > now - 120000) ++min2;
     else if (n > now - 180000) ++min3;
@@ -72,7 +72,7 @@ function restartWorker(worker, code, signal): void {
 
     logger.error({
       message: "Too many crashes, exiting",
-      pid: process.pid,
+      pid: process.pid
     });
     return;
   }
@@ -89,11 +89,7 @@ function restartWorker(worker, code, signal): void {
   }, respawnTimestamp - now);
 }
 
-export function start(
-  workerCount: number,
-  servicePort: number,
-  serviceAddress: string
-): void {
+export function start(workerCount, servicePort, serviceAddress): void {
   cluster.on("listening", (worker, address) => {
     if (
       (address.addressType === 4 || address.addressType === 6) &&
@@ -104,7 +100,7 @@ export function start(
         message: "Worker listening",
         pid: worker.process.pid,
         address: address.address,
-        port: address.port,
+        port: address.port
       });
     }
   });

lib/common.ts

@@ -17,12 +17,22 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-export function generateDeviceId(
-  deviceIdStruct: Record<string, string>
-): string {
+export const UNDEFINED_TYPE = "[object Undefined]";
+export const NULL_TYPE = "[object Null]";
+export const BOOLEAN_TYPE = "[object Boolean]";
+export const NUMBER_TYPE = "[object Number]";
+export const STRING_TYPE = "[object String]";
+export const ARRAY_TYPE = "[object Array]";
+export const OBJECT_TYPE = "[object Object]";
+export const REGEXP_TYPE = "[object RegExp]";
+export const DATE_TYPE = "[object Date]";
+
+export const typeOf = (obj): string => Object.prototype.toString.call(obj);
+
+export function generateDeviceId(deviceIdStruct): string {
   // Escapes everything except alphanumerics and underscore
   function esc(str): string {
-    return str.replace(/[^A-Za-z0-9_]/g, (chr) => {
+    return str.replace(/[^A-Za-z0-9_]/g, chr => {
       const buf = Buffer.from(chr, "utf8");
       let rep = "";
       for (const b of buf) rep += "%" + b.toString(16).toUpperCase();
@@ -44,20 +54,6 @@ export function generateDeviceId(
 }
 
 // Source: http://stackoverflow.com/a/6969486
-export function escapeRegExp(str: string): string {
+export function escapeRegExp(str): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
-
-export function encodeTag(tag: string): string {
-  return encodeURIComponent(tag)
-    .replace(
-      /[!~*'()]/g,
-      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
-    )
-    .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")
-    .replace(/%/g, "0x");
-}
-
-export function decodeTag(tag: string): string {
-  return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%"));
-}

lib/common/authorizer.ts

@@ -23,13 +23,13 @@ import { evaluate, or } from "./expression";
 export default class Authorizer {
   private permissionSets: PermissionSet[];
   private validatorCache: WeakMap<
-    any,
+    object,
     (mutationType, mutation, any) => boolean
   >;
   private hasAccessCache: Map<string, boolean>;
   private getFilterCache: Map<string, Expression>;
 
-  public constructor(permissionSets: PermissionSet[]) {
+  public constructor(permissionSets) {
     this.permissionSets = permissionSets;
     this.validatorCache = new WeakMap();
     this.hasAccessCache = new Map();
@@ -77,8 +77,8 @@ export default class Authorizer {
   }
 
   public getValidator(
-    resourceType: string,
-    resource: unknown
+    resourceType,
+    resource
   ): (mutationType: string, mutation?: any, args?: any) => boolean {
     if (this.validatorCache.has(resource))
       return this.validatorCache.get(resource);
@@ -108,7 +108,7 @@ export default class Authorizer {
         mutation,
         resourceType,
         object: resource,
-        options: any,
+        options: any
       };
 
       const valueFunction = (paramName): any => {
@@ -124,8 +124,7 @@ export default class Authorizer {
             if (value == null) break;
           }
         } else if (object[entry]) {
-          if (paramName) value = object[entry][paramName];
-          else value = object[entry];
+          value = object[entry][paramName];
         }
 
         return value;

lib/common/bigint-jsbi.ts

@@ -1,108 +0,0 @@
-/**
- * Copyright 2013-2020  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import JSBI from "jsbi";
-
-export function and(a: JSBI, b: JSBI): JSBI {
-  return JSBI.bitwiseAnd(a, b);
-}
-
-export function or(a: JSBI, b: JSBI): JSBI {
-  return JSBI.bitwiseOr(a, b);
-}
-
-export function xor(a: JSBI, b: JSBI): JSBI {
-  return JSBI.bitwiseXor(a, b);
-}
-
-export function not(a: JSBI): JSBI {
-  return JSBI.bitwiseNot(a);
-}
-
-export function lshift(a: JSBI, b: JSBI): JSBI {
-  return JSBI.leftShift(a, b);
-}
-
-export function rshift(a: JSBI, b: JSBI): JSBI {
-  return JSBI.signedRightShift(a, b);
-}
-
-export function add(a: JSBI, b: JSBI): JSBI {
-  return JSBI.add(a, b);
-}
-
-export function sub(a: JSBI, b: JSBI): JSBI {
-  return JSBI.subtract(a, b);
-}
-
-export function mul(a: JSBI, b: JSBI): JSBI {
-  return JSBI.multiply(a, b);
-}
-
-export function div(a: JSBI, b: JSBI): JSBI {
-  return JSBI.divide(a, b);
-}
-
-export function exp(a: JSBI, b: JSBI): JSBI {
-  return JSBI.exponentiate(a, b);
-}
-
-export function rem(a: JSBI, b: JSBI): JSBI {
-  return JSBI.remainder(a, b);
-}
-
-export function toNumber(a: JSBI): number {
-  return JSBI.toNumber(a);
-}
-
-export function eq(a: JSBI, b: JSBI): boolean {
-  return JSBI.equal(a, b);
-}
-
-export function ne(a: JSBI, b: JSBI): boolean {
-  return JSBI.notEqual(a, b);
-}
-
-export function lt(a: JSBI, b: JSBI): boolean {
-  return JSBI.lessThan(a, b);
-}
-
-export function lte(a: JSBI, b: JSBI): boolean {
-  return JSBI.lessThanOrEqual(a, b);
-}
-
-export function gt(a: JSBI, b: JSBI): boolean {
-  return JSBI.greaterThan(a, b);
-}
-
-export function gte(a: JSBI, b: JSBI): boolean {
-  return JSBI.greaterThanOrEqual(a, b);
-}
-
-export function asUintN(a: number, b: JSBI): JSBI {
-  return JSBI.asUintN(a, b);
-}
-
-export function asIntN(a: number, b: JSBI): JSBI {
-  return JSBI.asIntN(a, b);
-}
-
-const _BigInt = JSBI.BigInt;
-type _bigint = JSBI;
-export { _BigInt as BigInt, _bigint as bigint };

lib/common/bigint.ts

@@ -1,106 +0,0 @@
-/**
- * Copyright 2013-2020  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-export function and(a: bigint, b: bigint): bigint {
-  return a & b;
-}
-
-export function or(a: bigint, b: bigint): bigint {
-  return a | b;
-}
-
-export function xor(a: bigint, b: bigint): bigint {
-  return a ^ b;
-}
-
-export function not(a: bigint): bigint {
-  return ~a;
-}
-
-export function lshift(a: bigint, b: bigint): bigint {
-  return a << b;
-}
-
-export function rshift(a: bigint, b: bigint): bigint {
-  return a >> b;
-}
-
-export function add(a: bigint, b: bigint): bigint {
-  return a + b;
-}
-
-export function sub(a: bigint, b: bigint): bigint {
-  return a - b;
-}
-
-export function mul(a: bigint, b: bigint): bigint {
-  return a * b;
-}
-
-export function div(a: bigint, b: bigint): bigint {
-  return a / b;
-}
-
-export function exp(a: bigint, b: bigint): bigint {
-  return a ** b;
-}
-
-export function rem(a: bigint, b: bigint): bigint {
-  return a % b;
-}
-
-export function toNumber(a: bigint): number {
-  return Number(a);
-}
-
-export function eq(a: bigint, b: bigint): boolean {
-  return a === b;
-}
-
-export function ne(a: bigint, b: bigint): boolean {
-  return a !== b;
-}
-
-export function lt(a: bigint, b: bigint): boolean {
-  return a < b;
-}
-
-export function lte(a: bigint, b: bigint): boolean {
-  return a <= b;
-}
-
-export function gt(a: bigint, b: bigint): boolean {
-  return a > b;
-}
-
-export function gte(a: bigint, b: bigint): boolean {
-  return a >= b;
-}
-
-export function asUintN(a: number, b: bigint): bigint {
-  return BigInt.asUintN(a, b);
-}
-
-export function asIntN(a: number, b: bigint): bigint {
-  return BigInt.asIntN(a, b);
-}
-
-const _BigInt = BigInt;
-type _bigint = bigint;
-export { _BigInt as BigInt, _bigint as bigint };

lib/common/expression-cnf.ts

@@ -0,0 +1,285 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { map, parseLikePattern } from "./expression-parser";
+import { Expression } from "../types";
+
+type CNF = number[][];
+
+function or(cnf1: CNF, cnf2: CNF): CNF {
+  if (cnf1.length === 0) return cnf2;
+  else if (cnf2.length === 0) return cnf1;
+
+  if (cnf1.length > cnf2.length) {
+    const c = cnf1;
+    cnf1 = cnf2;
+    cnf2 = c;
+  }
+
+  const res = [];
+  for (let i = 0; i < cnf1.length; ++i)
+    for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j]));
+
+  return res;
+}
+
+function likePatternIncludes(pat1, pat2, idx1 = 0, idx2 = 0): boolean {
+  while (idx1 < pat1.length && idx2 < pat2.length) {
+    if (pat1[idx1] === "\\%") {
+      for (let i = idx2; i <= pat2.length; ++i)
+        if (likePatternIncludes(pat1, pat2, idx1 + 1, i)) return true;
+      return false;
+    } else if (pat2[idx2] === "\\%") {
+      if (pat1[idx1] !== "\\%") return false;
+    } else if (pat1[idx1] === "\\_") {
+      // Ignore
+    } else if (pat2[idx2] === "\\_") {
+      return false;
+    } else if (pat2[idx2] !== pat1[idx1]) {
+      return false;
+    }
+    ++idx1;
+    ++idx2;
+  }
+
+  if (idx1 === pat1.length && idx2 === pat2.length) return true;
+  return false;
+}
+
+export function booleanCnf(exp: Expression): { vars: number; clauses: CNF } {
+  const ranges = new Map();
+  const likePatterns = new Map();
+  map(exp, e => {
+    if (!Array.isArray(e)) return e;
+    if (e[0] === "LIKE" || e[0] === "NOT LIKE") {
+      if (
+        typeof e[2] === "string" &&
+        (e.length >= 4 || (e[3] != null && !Array.isArray(e[3])))
+      ) {
+        const key = JSON.stringify(e[1]);
+        let s = likePatterns.get(key);
+        if (!s) likePatterns.set(key, (s = new Map()));
+        s.set(`${e[3] || ""}:${e[2]}`, parseLikePattern(e[2], e[3]));
+      }
+    } else if ([">", ">=", "<", "<=", "=", "<>"].includes(e[0])) {
+      if (!Array.isArray(e[2])) {
+        const key = `${JSON.stringify(e[1])}:${typeof e[2]}`;
+        let s = ranges.get(key);
+        if (!s) ranges.set(key, (s = new Set()));
+        s.add(e[2]);
+      } else if (!Array.isArray(e[1])) {
+        const key = `${JSON.stringify(e[2])}:${typeof e[1]}`;
+        let s = ranges.get(key);
+        if (!s) ranges.set(key, (s = new Set()));
+        s.add(e[2]);
+      }
+    }
+    return e;
+  });
+
+  for (const [k, v] of ranges) {
+    if (k.endsWith(":number")) {
+      ranges.set(
+        k,
+        Array.from(v as number[]).sort((a: number, b: number) => a - b)
+      );
+    } else {
+      ranges.set(k, Array.from(v).sort());
+    }
+  }
+
+  const expressions: Map<string, number> = new Map();
+  const mutuallyExclusive: Map<string, Set<number>> = new Map();
+
+  function getVariable(lhs, op, rhs = ""): number {
+    const key = `${lhs}:${op}:${rhs}`;
+    let f = expressions.get(key);
+    if (!f) {
+      expressions.set(key, (f = expressions.size + 1));
+      let m = mutuallyExclusive.get(lhs);
+      if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
+      m.add(f);
+      if (rhs || op !== "null") m.add(getVariable(lhs, "null"));
+    }
+    return f;
+  }
+
+  function getComparisonVariables(
+    lhs,
+    from,
+    fromInclusive,
+    to,
+    toInclusive
+  ): number[] {
+    const type = from != null ? typeof from : typeof to;
+    const range = ranges.get(`${lhs}:${type}`);
+    const vars = [];
+
+    if (from == null) vars.push(`${lhs}:--${range[0]}`);
+
+    for (let i = 0; i < range.length; ++i) {
+      if (
+        i > 0 &&
+        (from == null || range[i] > from) &&
+        (to == null || range[i] <= to)
+      )
+        vars.push(`${lhs}:${range[i - 1]}--${range[i]}`);
+
+      if (
+        (range[i] > from && range[i] < to) ||
+        (range[i] === from && fromInclusive) ||
+        (range[i] === to && toInclusive)
+      )
+        vars.push(`${lhs}:${range[i]}`);
+    }
+
+    if (to == null) vars.push(`${lhs}:${range[range.length - 1]}--`);
+
+    return vars.map(key => {
+      let f = expressions.get(key);
+      if (!f) {
+        expressions.set(key, (f = expressions.size + 1));
+        let m = mutuallyExclusive.get(lhs);
+        if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
+        m.add(f);
+      }
+      return f;
+    });
+  }
+
+  function getLikeVariables(lhs, pat, esc): number[] {
+    const vars = [];
+    const pats = likePatterns.get(lhs);
+    pat = pats.get(`${esc || ""}:${pat}`);
+    for (const p of pats.values()) {
+      if (pat === p || likePatternIncludes(pat, p))
+        vars.push(`${lhs}:like:${JSON.stringify(p)}`);
+    }
+
+    return vars.map(key => {
+      let f = expressions.get(key);
+      if (!f) expressions.set(key, (f = expressions.size + 1));
+      return f;
+    });
+  }
+
+  interface ComparisonOperators {
+    [name: string]: (v: string) => [string, boolean, string, boolean];
+  }
+
+  const comparisonOperators: ComparisonOperators = {
+    "=": v => [v, true, v, true],
+    ">": v => [v, false, null, null],
+    ">=": v => [v, true, null, null],
+    "<": v => [null, null, v, false],
+    "<=": v => [null, null, v, true]
+  };
+
+  function recursive(clause, negate): CNF {
+    let op = clause[0];
+
+    if ((op === "AND" && !negate) || (op === "OR" && negate)) {
+      let res = [];
+      for (let i = 1; i < clause.length; ++i)
+        res = res.concat(recursive(clause[i], negate));
+      return res;
+    } else if ((op === "OR" && !negate) || (op === "AND" && negate)) {
+      let res = [];
+      for (let i = 1; i < clause.length; ++i)
+        res = or(res, recursive(clause[i], negate));
+      return res;
+    } else if (op === "NOT") {
+      return recursive(clause[1], !negate);
+    } else if (op === "<>") {
+      const vars = getComparisonVariables(
+        JSON.stringify(clause[1]),
+        ...comparisonOperators["="](clause[2])
+      );
+      if (!negate) return vars.map(x => [0 - x]);
+      else return [vars];
+    } else if (op === "IS NULL") {
+      const v = getVariable(JSON.stringify(clause[1]), "null");
+      if (negate) return [[0 - v]];
+      else return [[v]];
+    } else if (op === "IS NOT NULL") {
+      const v = getVariable(JSON.stringify(clause[1]), "null");
+      if (!negate) return [[0 - v]];
+      else return [[v]];
+    } else if (op in comparisonOperators) {
+      if (!Array.isArray(clause[2])) {
+        const vars = getComparisonVariables(
+          JSON.stringify(clause[1]),
+          ...comparisonOperators[op](clause[2])
+        );
+        if (negate) return vars.map(x => [0 - x]);
+        else return [vars];
+      } else if (!Array.isArray(clause[1])) {
+        op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
+        const vars = getComparisonVariables(
+          JSON.stringify(clause[2]),
+          ...comparisonOperators[op](clause[1])
+        );
+        if (negate) return vars.map(x => [0 - x]);
+        else return [vars];
+      } else {
+        let lhs = JSON.stringify(clause[1]);
+        let rhs = JSON.stringify(clause[2]);
+        // For consistency
+        if (rhs > lhs) {
+          const t = rhs;
+          rhs = lhs;
+          lhs = t;
+          op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
+        }
+
+        const v = getVariable(lhs, op, rhs);
+        if (negate) return [[0 - v]];
+        else return [[v]];
+      }
+    } else if (
+      (op === "LIKE" || op === "NOT LIKE") &&
+      !Array.isArray(clause[2]) &&
+      clause[2] != null &&
+      (clause.length >= 4 || clause[3] != null)
+    ) {
+      if (op === "NOT LIKE") negate = !negate;
+      const vars = getLikeVariables(
+        JSON.stringify(clause[1]),
+        clause[2],
+        clause[3]
+      );
+      if (negate) return vars.map(x => [0 - x]);
+      else return [vars];
+    } else {
+      const v = getVariable(JSON.stringify(clause), "");
+      if (negate) return [[0 - v]];
+      else return [[v]];
+    }
+  }
+
+  const cnf = recursive(exp, false);
+
+  for (const m of mutuallyExclusive.values()) {
+    const ar: number[] = Array.from(m);
+    for (let i = 0; i < ar.length; ++i)
+      for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]);
+  }
+
+  return { vars: expressions.size, clauses: cnf };
+}

lib/common/expression-parser.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import parsimmon from "parsimmon";
+import * as parsimmon from "parsimmon";
 import { Expression } from "../types";
 
 // Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
@@ -27,7 +27,7 @@ function interpretEscapes(str): string {
     f: "\f",
     n: "\n",
     r: "\r",
-    t: "\t",
+    t: "\t"
   };
   return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
     const type = escape.charAt(0);
@@ -40,10 +40,7 @@ function interpretEscapes(str): string {
   });
 }
 
-export function map(
-  exp: Expression,
-  callback: (e: Expression) => Expression
-): Expression {
+export function map(exp, callback): Expression {
   if (!Array.isArray(exp)) return callback(exp);
 
   let clone;
@@ -58,10 +55,7 @@ export function map(
   return callback(clone || exp);
 }
 
-export async function mapAsync(
-  exp: Expression,
-  callback: (e: Expression) => Promise<Expression>
-): Promise<Expression> {
+export async function mapAsync(exp, callback): Promise<Expression> {
   if (!Array.isArray(exp)) return callback(exp);
 
   let clone;
@@ -76,10 +70,7 @@ export async function mapAsync(
   return callback(clone || exp);
 }
 
-function binaryLeft(
-  operatorsParser: parsimmon.Parser<string>,
-  nextParser: parsimmon.Parser<Expression>
-): parsimmon.Parser<Expression> {
+function binaryLeft(operatorsParser, nextParser): parsimmon.Parser<{}> {
   return parsimmon.seqMap(
     nextParser,
     parsimmon.seq(operatorsParser, nextParser).many(),
@@ -110,7 +101,10 @@ const lang = parsimmon.createLanguage({
   LikeOperator: function() {
     return parsimmon
       .alt(
-        parsimmon.regexp(/like/i).result("LIKE").desc("LIKE"),
+        parsimmon
+          .regexp(/like/i)
+          .result("LIKE")
+          .desc("LIKE"),
         parsimmon
           .regexp(/not\s+like/i)
           .result("NOT LIKE")
@@ -168,7 +162,7 @@ const lang = parsimmon.createLanguage({
         )
       )
       .atLeast(1)
-      .map((x) => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
+      .map(x => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
       .skip(parsimmon.optWhitespace)
       .desc("parameter");
   },
@@ -177,7 +171,7 @@ const lang = parsimmon.createLanguage({
       .regexp(/'([^']*)'/, 1)
       .atLeast(1)
       .skip(parsimmon.optWhitespace)
-      .map((s) => s.join("'"))
+      .map(s => s.join("'"))
       .desc("string");
   },
   StringValueJs: function() {
@@ -190,7 +184,6 @@ const lang = parsimmon.createLanguage({
   NumberValue: function() {
     return parsimmon
       .regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/)
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace)
       .map(Number)
       .desc("number");
@@ -198,8 +191,14 @@ const lang = parsimmon.createLanguage({
   BooleanValue: function() {
     return parsimmon
       .alt(
-        parsimmon.regexp(/true/i).result(true).desc("TRUE"),
-        parsimmon.regexp(/false/i).result(false).desc("FALSE")
+        parsimmon
+          .regexp(/true/i)
+          .result(true)
+          .desc("TRUE"),
+        parsimmon
+          .regexp(/false/i)
+          .result(false)
+          .desc("FALSE")
       )
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
@@ -218,55 +217,15 @@ const lang = parsimmon.createLanguage({
         .regexp(/([a-zA-Z0-9_]+)/, 1)
         .skip(parsimmon.optWhitespace)
         .desc("function"),
-      r.ExpressionList.wrap(
+      r.ValueExpression.sepBy(
+        parsimmon.string(",").skip(parsimmon.optWhitespace)
+      ).wrap(
         parsimmon.string("(").skip(parsimmon.optWhitespace),
         parsimmon.string(")").skip(parsimmon.optWhitespace)
       ),
       (f, args) => ["FUNC", f.toUpperCase()].concat(args)
     );
   },
-  WhenPair: function (r) {
-    return parsimmon.seq(
-      parsimmon
-        .regexp(/when/i)
-        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-        .skip(parsimmon.optWhitespace)
-        .desc("WHEN")
-        .then(r.Expression),
-      parsimmon
-        .regexp(/then/i)
-        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-        .skip(parsimmon.optWhitespace)
-        .desc("THEN")
-        .then(r.Expression)
-    );
-  },
-  CaseStatement: function (r) {
-    return parsimmon.seqMap(
-      parsimmon
-        .regexp(/case/i)
-        .result("CASE")
-        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-        .skip(parsimmon.optWhitespace)
-        .desc("CASE"),
-      r.WhenPair.many(),
-      parsimmon
-        .regexp(/else/i)
-        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-        .skip(parsimmon.optWhitespace)
-        .desc("ELSE")
-        .then(r.Expression)
-        .map((e) => [[true, e]])
-        .fallback(null)
-        .skip(
-          parsimmon
-            .regex(/end/i)
-            .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-        )
-        .skip(parsimmon.optWhitespace),
-      (...arr) => arr.flat(2)
-    );
-  },
   Value: function(r) {
     return parsimmon.alt(
       r.NullValue,
@@ -274,8 +233,7 @@ const lang = parsimmon.createLanguage({
       r.NumberValue,
       r.StringValueSql,
       r.StringValueJs,
-      r.FuncValue,
-      r.CaseStatement
+      r.FuncValue
     );
   },
   ValueExpression: function(r) {
@@ -331,16 +289,8 @@ const lang = parsimmon.createLanguage({
       )
     );
   },
-  ExpressionList: function (r) {
-    return r.Expression.sepBy(
-      parsimmon.string(",").skip(parsimmon.optWhitespace)
-    );
-  },
   Expression: function(r) {
-    function unary(
-      operatorsParser: parsimmon.Parser<string>,
-      nextParser: parsimmon.Parser<Expression>
-    ): parsimmon.Parser<Expression> {
+    function unary(operatorsParser, nextParser): parsimmon.Parser<{}> {
       return parsimmon.seq(operatorsParser, nextParser).or(nextParser);
     }
 
@@ -351,7 +301,7 @@ const lang = parsimmon.createLanguage({
         unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
       )
     ).trim(parsimmon.optWhitespace);
-  },
+  }
 });
 
 export function parse(str: string): Expression {
@@ -359,12 +309,7 @@ export function parse(str: string): Expression {
   return lang.Expression.tryParse(str);
 }
 
-export function parseList(str: string): Expression[] {
-  if (!str) return [];
-  return lang.ExpressionList.tryParse(str);
-}
-
-export function stringify(exp: Expression, level = 0): string {
+export function stringify(exp, level = 0): string {
   if (!Array.isArray(exp)) return JSON.stringify(exp);
 
   const opLevels = {
@@ -385,7 +330,7 @@ export function stringify(exp: Expression, level = 0): string {
     "+": 31,
     "-": 31,
     "*": 32,
-    "/": 32,
+    "/": 32
   };
 
   const op = exp[0].toUpperCase();
@@ -399,7 +344,7 @@ export function stringify(exp: Expression, level = 0): string {
     return wrap(
       `${exp[1]}(${exp
         .slice(2)
-        .map((e) => stringify(e))
+        .map(e => stringify(e))
         .join(", ")})`
     );
   } else if (op === "PARAM") {
@@ -409,7 +354,7 @@ export function stringify(exp: Expression, level = 0): string {
       return wrap(
         exp[1]
           .slice(1)
-          .map((p) => {
+          .map(p => {
             if (typeof p === "string") return p;
             else return `{${stringify(p)}}`;
           })
@@ -436,17 +381,6 @@ export function stringify(exp: Expression, level = 0): string {
         )}`
       );
     }
-  } else if (op === "CASE") {
-    const parts: string[] = ["CASE"];
-    for (let i = 1; i < exp.length - 1; i += 2) {
-      if (!Array.isArray(exp[i]) && exp[i]) {
-        if (exp[i + 1] != null) parts.push("ELSE", stringify(exp[i + 1]));
-        break;
-      }
-      parts.push("WHEN", stringify(exp[i]), "THEN", stringify(exp[i + 1]));
-    }
-    parts.push("END");
-    return parts.join(" ");
   } else if (op in opLevels) {
     const parts = exp.slice(1).map((e, i) => {
       return stringify(e, opLevels[exp[0]] + Math.min(i - 1, 0));
@@ -459,7 +393,7 @@ export function stringify(exp: Expression, level = 0): string {
   }
 }
 
-export function parseLikePattern(pat: string, esc: string): string[] {
+export function parseLikePattern(pat, esc): string[] {
   const chars = pat.split("");
 
   for (let i = 0; i < chars.length; ++i) {
@@ -474,5 +408,5 @@ export function parseLikePattern(pat: string, esc: string): string[] {
       while (chars[i + 1] === "%") chars[++i] = "";
     }
   }
-  return chars.filter((c) => c);
+  return chars.filter(c => c);
 }

lib/common/expression.ts

@@ -18,11 +18,13 @@
  */
 
 import { map, mapAsync, parseLikePattern } from "./expression-parser";
+import { booleanCnf } from "./expression-cnf";
+import { naiveDpll } from "./sat-solver";
 import { Expression } from "../types";
 
 const isArray = Array.isArray;
 
-const regExpCache = new WeakMap<any, RegExp>();
+const regExpCache = new WeakMap<object, RegExp>();
 
 const REDUCE_SKIP = {};
 function reduce(exp, callback): Expression {
@@ -42,7 +44,7 @@ function reduce(exp, callback): Expression {
   return exp;
 }
 
-export function likePatternToRegExp(pat: string, esc = "", flags = ""): RegExp {
+export function likePatternToRegExp(pat, esc = "", flags = ""): RegExp {
   const convChars = {
     "-": "\\-",
     "/": "\\/",
@@ -61,231 +63,159 @@ export function likePatternToRegExp(pat: string, esc = "", flags = ""): RegExp {
     "{": "\\{",
     "}": "\\}",
     "\\%": ".*",
-    "\\_": ".",
+    "\\_": "."
   };
   let chars = parseLikePattern(pat, esc);
   if (!chars.length) return new RegExp("^$", flags);
-  chars = chars.map((c) => convChars[c] || c);
+  chars = chars.map(c => convChars[c] || c);
   chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
   const l = chars.length - 1;
   chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
   return new RegExp(chars.join(""), flags);
 }
 
-function compare(
-  a: boolean | number | string,
-  b: boolean | number | string
-): number {
-  if (typeof a === "boolean") a = +a;
-  if (typeof b === "boolean") b = +b;
-  if (typeof a !== typeof b) return typeof a === "string" ? 1 : -1;
-  return a > b ? 1 : a < b ? -1 : 0;
-}
-
-function toNumber(a: boolean | number | string): number {
-  switch (typeof a) {
-    case "number":
-      return a;
-    case "boolean":
-      return +a;
-    case "string":
-      return parseFloat(a) || 0;
-  }
-}
-
-function toString(a: boolean | number | string): string {
-  switch (typeof a) {
-    case "string":
-      return a;
-    case "number":
-      return a.toString();
-    case "boolean":
-      return (+a).toString();
-  }
-}
-
-export function evaluateCallback(exp: Expression): Expression {
-  if (!Array.isArray(exp)) return exp;
-  if (exp[0] === "CASE") {
-    for (let i = 1; i < exp.length; i += 2) {
-      if (Array.isArray(exp[i])) return exp;
-      if (exp[i]) return exp[i + 1];
-    }
-    return null;
-  } else if (exp[0] === "FUNC") {
-    if (exp[1] === "COALESCE") {
-      const args: Expression[] = [];
-      for (let i = 2; i < exp.length; ++i) {
-        const e = exp[i];
-        if (e == null) continue;
-        args.push(e);
-        if (!Array.isArray(e)) break;
-      }
-      if (!args.length) return null;
-      if (args.length === 1) return args[0];
-      return ["FUNC", "COALESCE", ...args];
-    } else if (exp[1] === "UPPER") {
-      if (exp[2] == null) return null;
-      if (!isArray(exp[2])) return toString(exp[2]).toUpperCase();
-    } else if (exp[1] === "LOWER") {
-      if (exp[2] == null) return null;
-      if (!isArray(exp[2])) return toString(exp[2]).toLowerCase();
-    }
-  } else if (exp[0] === "PARAM") {
-    if (exp[1] == null) return null;
-  } else if (exp[0] === "AND") {
-    for (let i = 1; i < exp.length; ++i)
-      if (!Array.isArray(exp[i]) && exp[i] != null && !exp[i]) return false;
-    const args: Expression[] = [];
-    for (let i = 1; i < exp.length; ++i) {
-      const ee = exp[i];
-      if (ee == null) return null;
-      if (Array.isArray(ee)) {
-        if (ee[0] === "AND") args.push(...ee.slice(1));
-        else args.push(ee);
-      }
-    }
-    if (!args.length) return true;
-    if (args.length === 1) args.push(true);
-    return ["AND", ...args];
-  } else if (exp[0] === "OR") {
-    const args: Expression[] = [];
-    for (let i = 1; i < exp.length; ++i) {
-      const ee = exp[i];
-      if (Array.isArray(ee)) {
-        if (ee[0] === "OR") args.push(...ee.slice(1));
-        else args.push(ee);
-      } else if (ee) {
-        return true;
-      }
-    }
-    if (!args.length) return exp.some((ee) => ee == null) ? null : false;
-    if (args.length === 1) args.push(false);
-    return ["OR", ...args];
-  } else if (exp[0] === "NOT") {
-    if (exp[1] == null) return null;
-    if (!isArray(exp[1])) return !exp[1];
-    else if (exp[1][0] === "NOT") return exp[1][1];
-  } else if (exp[0] === "IS NULL") {
-    if (isArray(exp[1])) return exp;
-    else if (exp[1] == null) return true;
+function evalExp(e: Expression): Expression {
+  if (e[0] === "AND") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a)) return a ? b : a;
+      if (!isArray(b)) return b ? a : b;
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "OR") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a)) return a ? a : b;
+      if (!isArray(b)) return b ? b : a;
+      return REDUCE_SKIP;
+    });
+  } else if (e[0] === "NOT") {
+    if (!isArray(e[1])) return !e[1];
+    else if (e[1][0] === "NOT") return e[1][1];
+  } else if (e[0] === "IS NULL") {
+    if (isArray(e[1])) return e;
+    else if (e[1] == null) return true;
     else return null;
-  } else if (exp[0] === "IS NOT NULL") {
-    if (isArray(exp[1])) return exp;
-    else if (exp[1] != null) return true;
+  } else if (e[0] === "IS NOT NULL") {
+    if (isArray(e[1])) return e;
+    else if (e[1] != null) return true;
     else return null;
-  } else if (exp[0] === "LIKE") {
-    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
+  } else if (e[0] === "LIKE") {
+    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
     else if (
-      exp[1] == null ||
-      exp[2] == null ||
-      ((exp as any[]).length >= 4 && exp[3] == null)
+      e[1] == null ||
+      e[2] == null ||
+      ((e as any[]).length >= 4 && e[3] == null)
     )
       return null;
-    let r = regExpCache.get(exp as any[]);
+    let r = regExpCache.get(e as any[]);
     if (!r) {
-      r = likePatternToRegExp(exp[2], exp[3]);
-      regExpCache.set(exp as any[], r);
+      r = likePatternToRegExp(e[2], e[3]);
+      regExpCache.set(e as any[], r);
     }
-    return r.test(exp[1]);
-  } else if (exp[0] === "NOT LIKE") {
-    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
+    return r.test(e[1]);
+  } else if (e[0] === "NOT LIKE") {
+    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
     else if (
-      exp[1] == null ||
-      exp[2] == null ||
-      ((exp as any[]).length >= 4 && exp[3] == null)
+      e[1] == null ||
+      e[2] == null ||
+      ((e as any[]).length >= 4 && e[3] == null)
     )
       return null;
-    let r = regExpCache.get(exp as any[]);
+    let r = regExpCache.get(e as any[]);
     if (!r) {
-      r = likePatternToRegExp(exp[2], exp[3]);
-      regExpCache.set(exp as any[], r);
+      r = likePatternToRegExp(e[2], e[3]);
+      regExpCache.set(e as any[], r);
     }
-    return !r.test(exp[1]);
-  } else if (exp[0] === "=") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) === 0;
-  } else if (exp[0] === "<>") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) !== 0;
-  } else if (exp[0] === ">") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) > 0;
-  } else if (exp[0] === ">=") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) >= 0;
-  } else if (exp[0] === "<") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) < 0;
-  } else if (exp[0] === "<=") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) <= 0;
-  } else if (exp[0] === "*") {
-    return reduce(exp, (a, b) => {
+    return !r.test(e[1]);
+  } else if (e[0] === "=") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] === e[2];
+  } else if (e[0] === "<>") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] !== e[2];
+  } else if (e[0] === ">") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] > e[2];
+  } else if (e[0] === ">=") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] >= e[2];
+  } else if (e[0] === "<") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] < e[2];
+  } else if (e[0] === "<=") {
+    if (isArray(e[1]) || isArray(e[2])) return e;
+    if (e[1] == null || e[2] == null) return null;
+    return e[1] <= e[2];
+  } else if (e[0] === "*") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a) && !isArray(b)) {
         if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b)) return toNumber(a) * toNumber(b);
+        return a * b;
+      }
       return REDUCE_SKIP;
     });
-  } else if (exp[0] === "/") {
-    return reduce(exp, (a, b, i) => {
+  } else if (e[0] === "/") {
+    return reduce(e, (a, b, i) => {
+      if (!isArray(a) && !isArray(b)) {
         if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b))
-        return i === 0 ? toNumber(a) / toNumber(b) : toNumber(a) * toNumber(b);
+        return i === 0 ? a / b : a * b;
+      }
       return REDUCE_SKIP;
     });
-  } else if (exp[0] === "+") {
-    return reduce(exp, (a, b) => {
+  } else if (e[0] === "+") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a) && !isArray(b)) {
         if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b)) return toNumber(a) + toNumber(b);
+        return a + b;
+      }
       return REDUCE_SKIP;
     });
-  } else if (exp[0] === "-") {
-    return reduce(exp, (a, b, i) => {
+  } else if (e[0] === "-") {
+    return reduce(e, (a, b, i) => {
+      if (!isArray(a) && !isArray(b)) {
         if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b))
-        return i === 0 ? toNumber(a) - toNumber(b) : toNumber(a) + toNumber(b);
+        return i === 0 ? a - b : a + b;
+      }
       return REDUCE_SKIP;
     });
-  } else if (exp[0] === "||") {
-    return reduce(exp, (a, b) => {
+  } else if (e[0] === "||") {
+    return reduce(e, (a, b) => {
+      if (!isArray(a) && !isArray(b)) {
         if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b)) return toString(a) + toString(b);
+        return `${a}${b}`;
+      }
       return REDUCE_SKIP;
     });
   }
-  return exp;
+  return e;
 }
 
 export function evaluate(
-  exp: Expression,
-  obj: Record<string, unknown> | ((e: string) => Expression),
+  exp,
+  obj,
   now: number,
-  cb?: (e: Expression) => Expression
+  cb?: Function
 ): string | number | boolean | null;
-export function evaluate(
-  exp: Expression,
-  obj?: Record<string, unknown> | ((e: string) => Expression),
-  now?: number,
-  cb?: (e: Expression) => Expression
-): Expression;
-export function evaluate(
-  exp: Expression,
-  obj?: Record<string, unknown> | ((e: string) => Expression),
-  now?: number,
-  cb?: (e: Expression) => Expression
-): Expression {
-  return map(exp, (e) => {
+export function evaluate(exp, obj?, now?: number, cb?: Function): Expression;
+export function evaluate(exp, obj?, now?: number, cb?: Function): Expression {
+  return map(exp, e => {
     if (cb) e = cb(e);
     if (!isArray(e)) return e;
 
-    if (e[0] === "FUNC" && e[1] === "NOW") {
+    if (e[0] === "FUNC") {
+      if (e[1] === "NOW") {
         if (now) return now;
+      } else if (e[1] === "UPPER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
+      } else if (e[1] === "LOWER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
+      }
     } else if (e[0] === "PARAM") {
       if (e[1] == null) return null;
       if (obj && !isArray(e[1])) {
@@ -298,29 +228,29 @@ export function evaluate(
         return v;
       }
     }
-    return evaluateCallback(e);
+    return evalExp(e);
   });
 }
 
 export async function evaluateAsync(
-  exp: Expression,
-  obj: Record<string, unknown>,
+  exp,
+  obj,
   now: number,
-  cb?: (e: Expression) => Promise<Expression>
+  cb?: Function
 ): Promise<string | number | boolean | null>;
 export async function evaluateAsync(
-  exp: Expression,
-  obj?: Record<string, unknown>,
+  exp,
+  obj?,
   now?: number,
-  cb?: (e: Expression) => Promise<Expression>
+  cb?: Function
 ): Promise<Expression>;
 export async function evaluateAsync(
-  exp: Expression,
-  obj?: Record<string, unknown>,
+  exp,
+  obj?,
   now?: number,
-  cb?: (e: Expression) => Promise<Expression>
+  cb?: Function
 ): Promise<Expression> {
-  return mapAsync(exp, async (e) => {
+  return mapAsync(exp, async e => {
     if (cb) e = await cb(e);
     if (!isArray(e)) return e;
 
@@ -339,59 +269,59 @@ export async function evaluateAsync(
       if (obj && !isArray(e[1])) {
         let v = obj[e[1]];
         if (v == null) return null;
-        if (typeof v === "object") v = v["value"] ? v["value"][0] : null;
-        return v as Expression;
+        if (typeof v === "object") v = v.value ? v.value[0] : null;
+        return v;
       }
     }
-    return evaluateCallback(e);
+    return evalExp(e);
   });
 }
 
 export function and(exp1: Expression, exp2: Expression): Expression {
-  if (exp1 != null && !exp1) return false;
-  if (exp2 != null && !exp2) return false;
-  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
-    if (exp1 == null || exp2 == null) return null;
-    return true;
-  }
-  if (!Array.isArray(exp2) && exp1[0] === "AND") return exp1;
-  if (!Array.isArray(exp1) && exp2[0] === "AND") return exp2;
+  if (!isArray(exp1)) return exp1 ? exp2 : exp1;
+  if (!isArray(exp2)) return exp2 ? exp1 : exp2;
 
-  const res: Expression = ["AND"];
+  let res: Expression = ["AND"];
 
-  if (Array.isArray(exp1) && exp1[0] === "AND") res.push(...exp1.slice(1));
+  if (exp1[0] === "AND") res = res.concat(exp1.slice(1));
   else res.push(exp1);
 
-  if (Array.isArray(exp2) && exp2[0] === "AND") res.push(...exp2.slice(1));
+  if (exp2[0] === "AND") res = res.concat(exp2.slice(1));
   else res.push(exp2);
 
   return res;
 }
 
 export function or(exp1: Expression, exp2: Expression): Expression {
-  if (!Array.isArray(exp1) && exp1) return true;
-  if (!Array.isArray(exp2) && exp2) return true;
-  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
-    if (exp1 == null || exp2 == null) return null;
-    return false;
-  }
-  if (!Array.isArray(exp2) && exp1[0] === "OR") return exp1;
-  if (!Array.isArray(exp1) && exp2[0] === "OR") return exp2;
+  if (!isArray(exp1)) return exp1 ? exp1 : exp2;
+  if (!isArray(exp2)) return exp2 ? exp2 : exp1;
 
-  const res: Expression = ["OR"];
+  let res: Expression = ["OR"];
 
-  if (Array.isArray(exp1) && exp1[0] === "OR") res.push(...exp1.slice(1));
+  if (exp1[0] === "OR") res = res.concat(exp1.slice(1));
   else res.push(exp1);
 
-  if (Array.isArray(exp2) && exp2[0] === "OR") res.push(...exp2.slice(1));
+  if (exp2[0] === "OR") res = res.concat(exp2.slice(1));
   else res.push(exp2);
 
   return res;
 }
 
-export function extractParams(exp: Expression): Expression[] {
+export function not(exp): Expression {
+  if (isArray(exp) && exp[0] === "NOT") return exp[1];
+  return ["NOT", exp];
+}
+
+export function subset(exp1, exp2): boolean {
+  const e = evaluate(["NOT", ["OR", ["NOT", exp1], exp2]]);
+  if (!isArray(e)) return !e;
+  const { vars, clauses } = booleanCnf(e);
+  return !naiveDpll(clauses, vars);
+}
+
+export function extractParams(exp): Expression[] {
   const params = new Set<Expression>();
-  map(exp, (e) => {
+  map(exp, e => {
     if (isArray(e) && e[0] === "PARAM") params.add(e[1]);
     return e;
   });

lib/common/memoize.ts

@@ -39,7 +39,7 @@ function getKey(obj): string {
   return k;
 }
 
-export default function memoize<T extends (...args: any[]) => any>(func: T): T {
+export default function memoize<T extends Function>(func: T): T {
   const funcKey = getKey(func);
   return ((...args) => {
     const key = JSON.stringify(args.map(getKey)) + funcKey;

lib/common/path-set.ts

@@ -73,8 +73,8 @@ export default class PathSet {
 
   public find(
     path: Path,
-    superset = false,
-    subset = false,
+    superset: boolean = false,
+    subset: boolean = false,
     depth: number = path.length
   ): Path[] {
     if (path.alias) throw new Error("PathSet does not support aliased paths");
@@ -106,24 +106,24 @@ export default class PathSet {
       if (!fragmentIndexSet1) {
         if (!fragmentIndexSet2) return [];
         if (!res) res = [...fragmentIndexSet2];
-        else res = res.filter((r) => fragmentIndexSet2.has(r));
+        else res = res.filter(r => fragmentIndexSet2.has(r));
       } else if (!fragmentIndexSet2) {
         if (!res) res = [...fragmentIndexSet1];
-        else res = res.filter((r) => fragmentIndexSet1.has(r));
+        else res = res.filter(r => fragmentIndexSet1.has(r));
       } else {
         if (!res) {
           res = [...fragmentIndexSet1, ...fragmentIndexSet2];
         } else {
           res = res.filter(
-            (r) => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
+            r => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
           );
         }
       }
       if (!res.length) return res;
     }
 
-    if (!res) res = [].concat(...lengthIndex.map((a) => [...a]));
-    else res = res.filter((r) => lengthIndex.some((a) => a.has(r)));
+    if (!res) res = [].concat(...lengthIndex.map(a => [...a]));
+    else res = res.filter(r => lengthIndex.some(a => a.has(r)));
 
     return res;
   }

lib/common/path.ts

@@ -97,7 +97,6 @@ export default class Path {
           )
             ++index;
           const s = pattern.slice(j, index).trim();
-          if (!s) throw new Error("Invalid parameter path");
           segments.push(s);
         }
 
@@ -119,7 +118,7 @@ export default class Path {
       if (Array.isArray(s)) {
         alias |= 1 << i;
         const parts = s.map(
-          (al) => `${al[0].toString()}:${JSON.stringify(al[1])}`
+          al => `${al[0].toString()}:${JSON.stringify(al[1])}`
         );
         return `[${parts.join(",")}]`;
       } else if (s === "*") {
@@ -160,7 +159,7 @@ export default class Path {
     return this._string;
   }
 
-  public slice(start = 0, end: number = this.segments.length): Path {
+  public slice(start: number = 0, end: number = this.segments.length): Path {
     if (start < 0) start = Math.max(0, this.segments.length + start);
     if (end < 0) end = Math.max(0, this.segments.length + end);
 
@@ -212,7 +211,7 @@ export default class Path {
 
   public stripAlias(): Path {
     if (!this.alias) return this;
-    const segments = this.segments.map((s) => (Array.isArray(s) ? "*" : s));
+    const segments = this.segments.map(s => (Array.isArray(s) ? "*" : s));
     const str = segments.join(".");
 
     let path = cache1.get(str);

lib/common/sat-solver.ts

@@ -0,0 +1,175 @@
+/**
+ * Naive DPLL solver for propositional logic adapted from
+ * https://github.com/tammet/logictools
+ *
+ * Origianl code by Tanel Tammet released under the MIT license.
+ *
+ * Check whether a clause set is satisfiable or contradictory, using a naive
+ * implementation of the dpll (Davis-Putnam-Logemann-Loveland) procedure
+ * combining truth table search with the resolution-like unit propagation.
+ *
+ * The standard pure-literal rule is not implemented in this naive version,
+ * since it is not really crucial for the algorithm.
+ *
+ * The code is very similar to the pure table solver, except we use a
+ * unit-propagating, derived-varlist-returing satisfiableAt.
+ */
+
+/* eslint-disable @typescript-eslint/no-use-before-define */
+
+function satisfiableAt(clauses, varVals, varnr, val): boolean {
+  if (varnr !== 0) varVals[varnr] = val;
+
+  const propRes = unitPropagate(clauses, varVals);
+  if (propRes === 1) {
+    varVals[varnr] = 0;
+    return true;
+  }
+
+  // if result false (-1) unitPropagate restores the varVals state itself
+  if (propRes === -1) {
+    varVals[varnr] = 0;
+    return false;
+  }
+
+  // find next unassigned var to split
+  let nextVar = 0;
+  for (let i = 1; i < varVals.length; i++) {
+    if (varVals[i] === 0) {
+      nextVar = i;
+      break;
+    }
+  }
+
+  if (nextVar === 0) throw new Error("Error in satisfiableAt");
+
+  if (
+    satisfiableAt(clauses, varVals, nextVar, 1) ||
+    satisfiableAt(clauses, varVals, nextVar, -1)
+  )
+    // not necessary to restore split var: search is halted anyway
+    return true;
+
+  // restore split var
+  varVals[nextVar] = 0;
+
+  // restore derived vars
+  for (let i = 0; i < propRes.length; i++) varVals[propRes[i]] = 0;
+
+  return false;
+}
+
+/**
+ * Check whether clauses are satisfiable under the varVals assignment, without
+ * search.
+ *
+ * Iterate through all clauses and check whether
+ *
+ *  - all literals in the clause are assigned false: undo derived units (set
+ *    values in varVals to 0) and return false immediately
+ *
+ *  - a clause contains a literal assigned true: skip further actions on this
+ *    clause
+ *
+ *  - a single literal in the clause is unassigned, all the rest are assigned
+ *    false: derive this literal as a unit clause.
+ *
+ * In case at least one literal is derived during the process above, iterate
+ * the whole process until no more new literals are derived.
+ *
+ * Returns:
+ *
+ *   1: all clauses were assigned true, hence the clause set is satisfiable.
+ *   The whole search will terminate after that.
+ *
+ *   -1: clause set is contradictory. Derived units have been restored.
+ *
+ *   a list of derived variables:  the status of the clause set is
+ *   undetermined. The values of variables on the list will be restored (set to
+ *   0) during backtracking.
+ */
+function unitPropagate(clauses, varVals): number[] | 1 | -1 {
+  // all vars with a value derived in the forthcoming main loop
+  const derivedVars = [];
+
+  // the main loop is run, deriving new unit clauses, until either:
+  // - the clause set is detected to be unsatisfiable
+  // - the clause set is detected to be satisfiable
+  // - there are no more derived unit clauses
+
+  for (;;) {
+    let allClausesFound = true; // set to false if we find an undetermined clause
+    const derivedQueue = []; // derived units (literals) during one iteration
+    for (let i = 0; i < clauses.length; i++) {
+      const clause = clauses[i];
+      let clauseVal = 0;
+      let unassignedCount = 0; // count unassigned vars in clause
+      let unassignedLit = 0; // 0 means none found for this clause
+
+      for (let j = 0; j < clause.length; j++) {
+        const lit = clause[j];
+        let nr, polarity;
+        if (lit < 0) {
+          nr = 0 - lit;
+          polarity = -1;
+        } else {
+          nr = lit;
+          polarity = 1;
+        }
+
+        if (varVals[nr] === polarity) {
+          clauseVal = 1;
+          break;
+        } else if (varVals[nr] === 0) {
+          unassignedCount++;
+          unassignedLit = lit;
+        }
+      }
+
+      if (clauseVal === 1) continue; // clause is subsumed
+
+      // clause is not subsumed by varVals
+      if (unassignedCount === 0) {
+        for (let j = 0; j < derivedVars.length; j++)
+          varVals[derivedVars[j]] = 0; // restore derived
+        return -1;
+      } else if (unassignedCount === 1) {
+        // unassignedLit is a derived literal
+        derivedQueue.push(unassignedLit);
+      }
+
+      if (unassignedCount !== 0) allClausesFound = false;
+    }
+
+    // if all clauses were subsumed or all the variables in a clause had a value
+    // no need to restore varVals: whole search finished
+    if (allClausesFound) return 1;
+
+    // in case there are any derived units present, store to varVals and
+    // iterate the main loop
+    if (derivedQueue.length === 0) break;
+
+    // some units were derived: iterate the main loop
+    for (let j = 0; j < derivedQueue.length; j++) {
+      const lit = derivedQueue[j];
+      let nr, polarity;
+      if (lit < 0) {
+        nr = 0 - lit;
+        polarity = -1;
+      } else {
+        nr = lit;
+        polarity = 1;
+      }
+      varVals[nr] = polarity;
+      derivedVars.push(nr);
+    }
+  }
+
+  return derivedVars;
+}
+
+export function naiveDpll(clauses, maxvarnr): boolean {
+  // variable values are 0 if not set, 1 if positive, -1 if negative
+  const varVals = new Int32Array(maxvarnr + 1);
+  return satisfiableAt(clauses, varVals, 0, 0);
+}

lib/config.ts

@@ -32,13 +32,13 @@ while (!existsSync(`${ROOT_DIR}/package.json`)) {
 }
 
 // For compatibility with v1.1
-let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl, fsHostname;
+let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl;
 
 const options = {
   EXT_DIR: { type: "path", default: resolve(ROOT_DIR, "config/ext") },
   MONGODB_CONNECTION_URL: {
     type: "string",
-    default: "mongodb://127.0.0.1/genieacs",
+    default: "mongodb://127.0.0.1/genieacs"
   },
 
   CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
@@ -62,7 +62,7 @@ const options = {
   FS_INTERFACE: { type: "string", default: "0.0.0.0" },
   FS_SSL_CERT: { type: "string", default: "" },
   FS_SSL_KEY: { type: "string", default: "" },
-  FS_URL_PREFIX: { type: "string", default: "" },
+  FS_HOSTNAME: { type: "string", default: "" },
   FS_LOG_FILE: { type: "path", default: "" },
   FS_ACCESS_LOG_FILE: { type: "path", default: "" },
 
@@ -76,7 +76,6 @@ const options = {
   UI_JWT_SECRET: { type: "string", default: "" },
 
   UDP_CONNECTION_REQUEST_PORT: { type: "int", default: 0 },
-  FORWARDED_HEADER: { type: "string", default: "" },
 
   DOWNLOAD_TIMEOUT: { type: "int", default: 3600 },
   EXT_TIMEOUT: { type: "int", default: 3000 },
@@ -99,8 +98,16 @@ const options = {
   CONNECTION_REQUEST_ALLOW_BASIC_AUTH: { type: "bool", default: false },
   MAX_COMMIT_ITERATIONS: { type: "int", default: 32 },
 
+  // XML configuration
+  XML_RECOVER: { type: "bool", default: false },
+  XML_IGNORE_ENC: { type: "bool", default: false },
+  XML_FORMAT: { type: "bool", default: false },
+  XML_NO_DECL: { type: "bool", default: false },
+  XML_NO_EMPTY: { type: "bool", default: false },
+  XML_LIBXMLJS: { type: "bool", default: false },
+
   // Should probably never be changed
-  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
+  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 }
 };
 
 const allConfig: { [name: string]: string | number } = {};
@@ -112,20 +119,37 @@ function setConfig(name, value, commandLineArgument = false): boolean {
   if (name === "CONFIG_DIR" || name === "config-dir")
     configDir = configDir || resolve(ROOT_DIR, value);
 
-  if (name === "CWMP_SSL" || name === "cwmp-ssl")
-    cwmpSsl = cwmpSsl || String(value).toLowerCase().trim();
+  if (name === "CWMP_SSL" || name === "cwmp-ssl") {
+    cwmpSsl =
+      cwmpSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
 
-  if (name === "NBI_SSL" || name === "nbi-ssl")
-    nbiSsl = nbiSsl || String(value).toLowerCase().trim();
+  if (name === "NBI_SSL" || name === "nbi-ssl") {
+    nbiSsl =
+      nbiSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
 
-  if (name === "FS_SSL" || name === "fs-ssl")
-    fsSsl = fsSsl || String(value).toLowerCase().trim();
+  if (name === "FS_SSL" || name === "fs-ssl") {
+    fsSsl =
+      fsSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
 
-  if (name === "UI_SSL" || name === "ui-ssl")
-    uiSsl = uiSsl || String(value).toLowerCase().trim();
-
-  if (name === "FS_HOSTNAME" || name === "fs-hostname")
-    fsHostname = fsHostname || String(value).trim();
+  if (name === "UI_SSL" || name === "ui-ssl") {
+    uiSsl =
+      uiSsl ||
+      String(value)
+        .toLowerCase()
+        .trim();
+  }
 
   // For compatibility with v1.0
   if (name === "PRESETS_CACHE_DURATION" || name === "presets-cache-duration")
@@ -143,6 +167,12 @@ function setConfig(name, value, commandLineArgument = false): boolean {
   )
     setConfig("GPV_BATCH_SIZE", value);
 
+  if (name === "XML_PARSE_IGNORE_ENC" || name === "xml-parse-ignore-enc")
+    setConfig("XML_IGNORE_ENC", value);
+
+  if (name === "XML_PARSE_RECOVER" || name === "xml-parse-recover")
+    setConfig("XML_RECOVER", value);
+
   if (name === "FS_IP" || name === "fs-ip") setConfig("FS_HOSTNAME", value);
 
   function cast(val, type): string | number | boolean {
@@ -150,7 +180,11 @@ function setConfig(name, value, commandLineArgument = false): boolean {
       case "int":
         return Number(val);
       case "bool":
-        return ["true", "1"].includes(String(val).trim().toLowerCase());
+        return ["true", "1"].includes(
+          String(val)
+            .trim()
+            .toLowerCase()
+        );
       case "string":
         return String(val);
       case "path":
@@ -240,15 +274,6 @@ if (["true", "1"].includes(uiSsl)) {
   setConfig("UI_SSL_KEY", `${d}/cwmp.key`);
 }
 
-if (fsHostname) {
-  const FS_PORT = allConfig["FS_PORT"] || 7567;
-  const FS_SSL = !!allConfig["FS_SSL_CERT"];
-  setConfig(
-    "FS_URL_PREFIX",
-    (FS_SSL ? "https" : "http") + `://${fsHostname}:${FS_PORT}/`
-  );
-}
-
 // Defaults
 for (const [k, v] of Object.entries(options))
   if (v["default"] != null) setConfig(k, v["default"]);
@@ -284,7 +309,7 @@ export function get(
   return null;
 }
 
-export function getDefault(optionName: string): string | number | boolean {
+export function getDefault(optionName): string | number | boolean {
   const option = options[optionName];
   if (!option) return null;
 

lib/connection-request.ts

@@ -29,13 +29,15 @@ import * as debug from "./debug";
 
 async function extractAuth(
   exp: Expression,
+  context: {},
+  now: number,
   dflt: any
 ): Promise<[string, string, Expression]> {
   let username, password;
   const _exp = await evaluateAsync(
     exp,
-    {},
-    0,
+    context,
+    now,
     async (e: Expression): Promise<Expression> => {
       if (!username && Array.isArray(e) && e[0] === "FUNC") {
         if (e[1] === "EXT") {
@@ -48,8 +50,8 @@ async function extractAuth(
           return fault ? null : value;
         } else if (e[1] === "AUTH") {
           if (!Array.isArray(e[2]) && !Array.isArray(e[3])) {
-            username = e[2] || "";
-            password = e[3] || "";
+            username = e[2];
+            password = e[3];
           }
           return dflt;
         }
@@ -62,12 +64,13 @@ async function extractAuth(
 
 function httpGet(
   options: http.RequestOptions,
+  timeout,
   _debug: boolean,
   deviceId: string
-): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
+): Promise<{ statusCode: number; headers: {} }> {
   return new Promise((resolve, reject) => {
     const req = http
-      .get(options, (res) => {
+      .get(options, res => {
         res.resume();
         resolve({ statusCode: res.statusCode, headers: res.headers });
         if (_debug) {
@@ -75,13 +78,16 @@ function httpGet(
           debug.incomingHttpResponse(res, deviceId, null);
         }
       })
-      .on("error", (err) => {
-        req.destroy();
-        reject(err);
+      .on("error", err => {
+        req.abort();
+        reject(new Error("Device is offline"));
         if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
       })
-      .on("timeout", () => {
-        req.destroy();
+      .on("socket", socket => {
+        socket.setTimeout(timeout);
+        socket.on("timeout", () => {
+          req.abort();
+        });
       });
   });
 }
@@ -89,22 +95,23 @@ function httpGet(
 export async function httpConnectionRequest(
   address: string,
   authExp: Expression,
+  context: {},
   allowBasicAuth: boolean,
   timeout: number,
   _debug: boolean,
   deviceId: string
-): Promise<string> {
+): Promise<void> {
+  const now = Date.now();
   const options: http.RequestOptions = parse(address);
   if (options.protocol !== "http:")
-    return "Invalid connection request URL or protocol";
+    throw new Error("Invalid connection request URL or protocol");
 
   options.agent = new http.Agent({
     maxSockets: 1,
-    keepAlive: true,
-    timeout: timeout,
+    keepAlive: true
   });
 
-  let authHeader: Record<string, string>;
+  let authHeader: {};
   let username: string;
   let password: string;
 
@@ -112,13 +119,14 @@ export async function httpConnectionRequest(
     let opts = options;
     if (authHeader) {
       if (authHeader["method"] === "Basic") {
-        if (!allowBasicAuth) return "Basic HTTP authentication not allowed";
+        if (!allowBasicAuth)
+          throw new Error("Basic HTTP authentication not allowed");
 
         opts = Object.assign(
           {
             headers: {
-              Authorization: auth.basic(username || "", password || ""),
-            },
+              Authorization: auth.basic(username || "", password || "")
+            }
           },
           options
         );
@@ -133,58 +141,48 @@ export async function httpConnectionRequest(
                 "GET",
                 null,
                 authHeader
-              ),
-            },
+              )
+            }
           },
           options
         );
       } else {
-        return "Unrecognized auth method";
+        throw new Error("Unrecognized auth method");
       }
     }
 
-    let res: { statusCode: number; headers: http.IncomingHttpHeaders };
-    try {
-      res = await httpGet(opts, _debug, deviceId);
-    } catch (err) {
+    let res = await httpGet(opts, timeout, _debug, deviceId);
+
     // Workaround for some devices unexpectedly closing the connection
-      if (authHeader) {
-        try {
-          res = await httpGet(opts, _debug, deviceId);
-        } catch (err) {
-          return `Connection request error: ${err.message}`;
-        }
-      }
-
-      if (err["code"] === "ECONNRESET" || err["code"] === "ECONNREFUSED")
-        return "Device is offline";
-
-      return `Connection request error: ${err.message}`;
-    }
-
-    if (res.statusCode === 200 || res.statusCode === 204) return "";
-
-    // When a Connection Request is received for the Virtual CWMP Device and the Proxied
-    // Device is offline the CPE Proxier MUST respond with an HTTP 503 failure
-    if (res.statusCode === 503) return "Device is offline";
+    if (res.statusCode === 0 && authHeader)
+      res = await httpGet(opts, timeout, _debug, deviceId);
+    if (res.statusCode === 0) throw new Error("Device is offline");
+    if (res.statusCode === 200 || res.statusCode === 204) return;
 
     if (res.statusCode === 401 && res.headers["www-authenticate"]) {
       authHeader = auth.parseWwwAuthenticateHeader(
         res.headers["www-authenticate"]
       );
-      [username, password, authExp] = await extractAuth(authExp, false);
+      [username, password, authExp] = await extractAuth(
+        authExp,
+        context,
+        now,
+        false
+      );
     } else {
-      return `Connection request error: Unexpected status code ${res.statusCode}`;
+      throw new Error(
+        `Unexpected response code from device: ${res.statusCode}`
+      );
     }
   }
-
-  return "Connection request error: Incorrect connection request credentials";
+  throw new Error("Incorrect connection request credentials");
 }
 
 export async function udpConnectionRequest(
   address: string,
   authExp: Expression,
-  sourcePort = 0,
+  context: {},
+  sourcePort: number = 0,
   _debug: boolean,
   deviceId: string
 ): Promise<void> {
@@ -204,7 +202,12 @@ export async function udpConnectionRequest(
   let username: string;
   let password: string;
 
-  [username, password, authExp] = await extractAuth(authExp, null);
+  [username, password, authExp] = await extractAuth(
+    authExp,
+    context,
+    now,
+    null
+  );
 
   if (username == null) username = "";
   if (password == null) password = "";
@@ -230,7 +233,12 @@ export async function udpConnectionRequest(
       });
     }
 
-    [username, password, authExp] = await extractAuth(authExp, null);
+    [username, password, authExp] = await extractAuth(
+      authExp,
+      context,
+      now,
+      null
+    );
   }
   client.close();
 }

lib/db.ts

@@ -17,9 +17,9 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, ObjectID, Collection, Db, GridFSBucket } from "mongodb";
+import { MongoClient, ObjectID, Collection } from "mongodb";
 import { get } from "./config";
-import { encodeTag, escapeRegExp } from "./common";
+import { escapeRegExp } from "./common";
 import { parse } from "./common/expression-parser";
 import {
   DeviceData,
@@ -27,7 +27,7 @@ import {
   SessionFault,
   Task,
   Operation,
-  Expression,
+  Expression
 } from "./types";
 import Path from "./common/path";
 
@@ -39,14 +39,13 @@ export let tasksCollection: Collection,
   virtualParametersCollection: Collection,
   faultsCollection: Collection,
   filesCollection: Collection,
-  uploadsCollection: Collection,
   operationsCollection: Collection,
   permissionsCollection: Collection,
   usersCollection: Collection,
-  configCollection: Collection,
-  uploadsBucket: GridFSBucket;
+  configCollection: Collection;
 
 let clientPromise: Promise<MongoClient>;
+export let client: MongoClient;
 
 function compareAccessLists(list1: string[], list2: string[]): boolean {
   if (list1.length !== list2.length) return false;
@@ -54,24 +53,14 @@ function compareAccessLists(list1: string[], list2: string[]): boolean {
   return true;
 }
 
-const onConnectCallbacks: ((db: Db) => Promise<void>)[] = [];
-
-export function onConnect(callback: (db: Db) => Promise<void>): void {
-  onConnectCallbacks.push(callback);
-}
-
 export async function connect(): Promise<void> {
   clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
-    useNewUrlParser: true,
-    useUnifiedTopology: true,
+    useNewUrlParser: true
   });
 
-  const client = await clientPromise;
+  client = await clientPromise;
   const db = client.db();
-  await Promise.all(onConnectCallbacks.map((c) => c(db)));
-}
 
-onConnect(async (db) => {
   tasksCollection = db.collection("tasks");
   await tasksCollection.createIndex({ device: 1, timestamp: 1 });
 
@@ -79,7 +68,6 @@ onConnect(async (db) => {
   presetsCollection = db.collection("presets");
   objectsCollection = db.collection("objects");
   filesCollection = db.collection("fs.files");
-  uploadsCollection = db.collection("uploads.files");
   provisionsCollection = db.collection("provisions");
   virtualParametersCollection = db.collection("virtualParameters");
   faultsCollection = db.collection("faults");
@@ -87,8 +75,7 @@ onConnect(async (db) => {
   permissionsCollection = db.collection("permissions");
   usersCollection = db.collection("users");
   configCollection = db.collection("config");
-  uploadsBucket = new GridFSBucket(db, { bucketName: "uploads" });
-});
+}
 
 export async function disconnect(): Promise<void> {
   if (clientPromise) await (await clientPromise).close();
@@ -96,7 +83,7 @@ export async function disconnect(): Promise<void> {
 
 // Optimize projection by removing overlaps
 // This can modify the object
-function optimizeProjection(obj: { [path: string]: 1 }): { [path: string]: 1 } {
+function optimizeProjection(obj: {}): {} {
   if (obj[""]) return { "": obj[""] };
 
   const keys = Object.keys(obj).sort();
@@ -123,13 +110,13 @@ export async function fetchDevice(
     [
       Path.parse("Events"),
       timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 0] },
+      { object: [timestamp, 1], writable: [timestamp, 0] }
     ],
     [
       Path.parse("DeviceID"),
       timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 0] },
-    ],
+      { object: [timestamp, 1], writable: [timestamp, 0] }
+    ]
   ];
 
   const device = await devicesCollection.findOne({ _id: id });
@@ -159,7 +146,7 @@ export async function fetchDevice(
     if (obj["_notification"] != null) {
       attrs.notification = [
         obj["_attributesTimestamp"] || 1,
-        obj["_notification"],
+        obj["_notification"]
       ];
     }
 
@@ -188,8 +175,8 @@ export async function fetchDevice(
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]],
-          },
+            value: [+v, [+v, "xsd:dateTime"]]
+          }
         ]);
         break;
       case "_lastBoot":
@@ -199,8 +186,8 @@ export async function fetchDevice(
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]],
-          },
+            value: [+v, [+v, "xsd:dateTime"]]
+          }
         ]);
         break;
       case "_lastBootstrap":
@@ -210,8 +197,8 @@ export async function fetchDevice(
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]],
-          },
+            value: [+v, [+v, "xsd:dateTime"]]
+          }
         ]);
         break;
       case "_registered":
@@ -222,8 +209,8 @@ export async function fetchDevice(
           {
             object: [timestamp, 0],
             writable: [timestamp, 0],
-            value: [timestamp, [+v, "xsd:dateTime"]],
-          },
+            value: [timestamp, [+v, "xsd:dateTime"]]
+          }
         ]);
         break;
       case "_id":
@@ -233,8 +220,8 @@ export async function fetchDevice(
           {
             object: [timestamp, 0],
             writable: [timestamp, 0],
-            value: [timestamp, [v as string, "xsd:string"]],
-          },
+            value: [timestamp, [v as string, "xsd:string"]]
+          }
         ]);
         break;
       case "_tags":
@@ -242,19 +229,20 @@ export async function fetchDevice(
           res.push([
             Path.parse("Tags"),
             timestamp,
-            { object: [timestamp, 1], writable: [timestamp, 0] },
+            { object: [timestamp, 1], writable: [timestamp, 0] }
           ]);
         }
 
-        for (const t of v as string[]) {
+        for (let t of v as string[]) {
+          t = t.replace(/[^a-zA-Z0-9-]+/g, "_");
           res.push([
-            Path.parse("Tags." + encodeTag(t)),
+            Path.parse("Tags." + t),
             timestamp,
             {
               object: [timestamp, 0],
               writable: [timestamp, 1],
-              value: [timestamp, [true, "xsd:boolean"]],
-            },
+              value: [timestamp, [true, "xsd:boolean"]]
+            }
           ]);
         }
         break;
@@ -266,8 +254,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_Manufacturer"], "xsd:string"]],
-            },
+              value: [timestamp, [v["_Manufacturer"], "xsd:string"]]
+            }
           ]);
         }
 
@@ -278,8 +266,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_OUI"], "xsd:string"]],
-            },
+              value: [timestamp, [v["_OUI"], "xsd:string"]]
+            }
           ]);
         }
 
@@ -290,8 +278,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_ProductClass"], "xsd:string"]],
-            },
+              value: [timestamp, [v["_ProductClass"], "xsd:string"]]
+            }
           ]);
         }
 
@@ -302,8 +290,8 @@ export async function fetchDevice(
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_SerialNumber"], "xsd:string"]],
-            },
+              value: [timestamp, [v["_SerialNumber"], "xsd:string"]]
+            }
           ]);
         }
     }
@@ -434,7 +422,7 @@ export async function saveDevice(
           } else {
             if (!update["$pull"]["_tags"]) {
               update["$pull"]["_tags"] = {
-                $in: [],
+                $in: []
               };
             }
             update["$pull"]["_tags"]["$in"].push(path.segments[1]);
@@ -443,17 +431,6 @@ export async function saveDevice(
 
         break;
       default:
-        if (
-          diff[0].segments[0] === "Uploads" &&
-          diff[0].segments[2] === "LastFileName" &&
-          value1 &&
-          value1 !== value2
-        ) {
-          uploadsBucket.delete(value1 as any, () => {
-            // Ignore error due to mising files
-          });
-        }
-
         if (!diff[2]) {
           update["$unset"][path.toString()] = 1;
           continue;
@@ -603,7 +580,7 @@ export async function saveDevice(
   }
 
   const result = await devicesCollection.updateOne({ _id: deviceId }, update, {
-    upsert: isNew,
+    upsert: isNew
   });
 
   if (result.result.n !== 1)
@@ -616,7 +593,7 @@ export async function saveDevice(
 }
 
 export async function getFaults(
-  deviceId: string
+  deviceId
 ): Promise<{ [channel: string]: SessionFault }> {
   const res = await faultsCollection
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
@@ -636,33 +613,26 @@ export async function getFaults(
   return faults;
 }
 
-export async function saveFault(
-  deviceId: string,
-  channel: string,
-  fault: SessionFault
-): Promise<void> {
+export async function saveFault(deviceId, channel, fault): Promise<void> {
   const id = `${deviceId}:${channel}`;
-  const f = Object.assign({}, fault) as Record<string, any>;
-  f["_id"] = id;
-  f["device"] = deviceId;
-  f["channel"] = channel;
-  f["timestamp"] = new Date(fault.timestamp);
-  f["provisions"] = JSON.stringify(fault.provisions);
-  await faultsCollection.replaceOne({ _id: id }, f, { upsert: true });
+  fault = Object.assign({}, fault);
+  fault._id = id;
+  fault.device = deviceId;
+  fault.channel = channel;
+  fault.timestamp = new Date(fault.timestamp);
+  fault.provisions = JSON.stringify(fault.provisions);
+  await faultsCollection.replaceOne({ _id: id }, fault, { upsert: true });
 }
 
-export async function deleteFault(
-  deviceId: string,
-  channel: string
-): Promise<void> {
+export async function deleteFault(deviceId, channel): Promise<void> {
   await faultsCollection.deleteOne({ _id: `${deviceId}:${channel}` });
 }
 
 export async function getDueTasks(
-  deviceId: string,
-  timestamp: number
+  deviceId,
+  timestamp
 ): Promise<[Task[], number]> {
-  const cur = tasksCollection.find({ device: deviceId }).sort({ timestamp: 1 });
+  const cur = tasksCollection.find({ device: deviceId }).sort(["timestamp"]);
   const tasks = [] as Task[];
 
   for await (const task of cur) {
@@ -693,17 +663,14 @@ export async function getDueTasks(
   return [tasks, null];
 }
 
-export async function clearTasks(
-  deviceId: string,
-  taskIds: string[]
-): Promise<void> {
+export async function clearTasks(deviceId, taskIds): Promise<void> {
   await tasksCollection.deleteMany({
-    _id: { $in: taskIds.map((id) => new ObjectID(id)) },
+    _id: { $in: taskIds.map(id => new ObjectID(id)) }
   });
 }
 
 export async function getOperations(
-  deviceId: string
+  deviceId
 ): Promise<{ [commandKey: string]: Operation }> {
   const res = await operationsCollection
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
@@ -713,11 +680,6 @@ export async function getOperations(
   for (const r of res) {
     const commandKey = r._id.slice(deviceId.length + 1);
     delete r._id;
-    // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
-    if (typeof r.provisions !== "string") {
-      operations[commandKey] = r;
-      continue;
-    }
     r.timestamp = +r.timestamp;
     if (r.args) r.args = JSON.parse(r.args);
     r.provisions = JSON.parse(r.provisions);
@@ -728,46 +690,43 @@ export async function getOperations(
 }
 
 export async function saveOperation(
-  deviceId: string,
-  commandKey: string,
-  operation: Operation
+  deviceId,
+  commandKey,
+  operation
 ): Promise<void> {
   const id = `${deviceId}:${commandKey}`;
-  const o = Object.assign({}, operation) as Record<string, any>;
-  o["_id"] = id;
-  o["timestamp"] = new Date(operation.timestamp);
-  o["provisions"] = JSON.stringify(operation.provisions);
-  o["retries"] = JSON.stringify(operation.retries);
-  o["args"] = JSON.stringify(operation.args);
-  await operationsCollection.replaceOne({ _id: id }, o, {
-    upsert: true,
+  operation = Object.assign({}, operation);
+  operation._id = id;
+  operation.timestamp = new Date(operation.timestamp);
+  operation.provisions = JSON.stringify(operation.provisions);
+  operation.retries = JSON.stringify(operation.retries);
+  operation.args = JSON.stringify(operation.args);
+  await operationsCollection.replaceOne({ _id: id }, operation, {
+    upsert: true
   });
 }
 
-export async function deleteOperation(
-  deviceId: string,
-  commandKey: string
-): Promise<void> {
+export async function deleteOperation(deviceId, commandKey): Promise<void> {
   await operationsCollection.deleteOne({ _id: `${deviceId}:${commandKey}` });
 }
 
-export async function getPresets(): Promise<Record<string, any>[]> {
+export async function getPresets(): Promise<{}[]> {
   return presetsCollection.find().toArray();
 }
 
-export async function getObjects(): Promise<Record<string, any>[]> {
+export async function getObjects(): Promise<{}[]> {
   return objectsCollection.find().toArray();
 }
 
-export async function getProvisions(): Promise<Record<string, any>[]> {
+export async function getProvisions(): Promise<{}[]> {
   return provisionsCollection.find().toArray();
 }
 
-export async function getVirtualParameters(): Promise<Record<string, any>[]> {
+export async function getVirtualParameters(): Promise<{}[]> {
   return virtualParametersCollection.find().toArray();
 }
 
-export function getFiles(): Promise<Record<string, any>[]> {
+export function getFiles(): Promise<{}[]> {
   return filesCollection.find().toArray();
 }
 
@@ -775,9 +734,9 @@ export async function getConfig(): Promise<
   { id: string; value: Expression }[]
 > {
   const res = await configCollection.find().toArray();
-  return res.map((c) => ({
+  return res.map(c => ({
     id: c["_id"],
-    value: parse(c["value"]),
+    value: parse(c["value"])
   }));
 }
 
@@ -803,14 +762,3 @@ interface User {
 export async function getUsers(): Promise<User[]> {
   return usersCollection.find().toArray();
 }
-
-export async function deleteDeviceUploads(deviceId: string): Promise<void> {
-  const files = await uploadsCollection
-    .find({
-      _id: {
-        $regex: `^${escapeRegExp(deviceId)}\\/`,
-      },
-    })
-    .toArray();
-  await Promise.all(files.map((f) => uploadsBucket.delete(f["_id"])));
-}

lib/debug.ts

@@ -21,7 +21,7 @@ import {
   IncomingMessage,
   ServerResponse,
   ClientRequest,
-  RequestOptions,
+  RequestOptions
 } from "http";
 import { Socket } from "net";
 import { appendFileSync } from "fs";
@@ -49,7 +49,7 @@ export function incomingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.socket;
+  const con = httpRequest.connection;
   const msg = {
     event: "incoming HTTP request",
     timestamp: now,
@@ -60,7 +60,7 @@ export function incomingHttpRequest(
     method: httpRequest.method,
     url: httpRequest.url,
     headers: httpRequest.headers,
-    body: body,
+    body: body
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -77,7 +77,7 @@ export function outgoingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.socket;
+  const con = httpResponse.connection;
   const msg = {
     event: "outgoing HTTP response",
     timestamp: now,
@@ -86,7 +86,7 @@ export function outgoingHttpResponse(
     connection: getConnectionTimestamp(con),
     statusCode: httpResponse.statusCode,
     headers: httpResponse.getHeaders(),
-    body: body,
+    body: body
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -104,7 +104,7 @@ export function outgoingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.socket;
+  const con = httpRequest.connection;
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
@@ -115,7 +115,7 @@ export function outgoingHttpRequest(
     method: options.method || "GET",
     url: options.path,
     headers: httpRequest.getHeaders(),
-    body: body,
+    body: body
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -143,7 +143,7 @@ export function outgoingHttpRequestError(
     method: options.method,
     url: options.path,
     headers: httpRequest.getHeaders(),
-    error: err.message,
+    error: err.message
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -160,16 +160,16 @@ export function incomingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.socket;
+  const con = httpResponse.connection;
   const msg = {
     event: "incoming HTTP response",
     timestamp: now,
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
-    connection: getConnectionTimestamp(httpResponse.socket),
+    connection: getConnectionTimestamp(httpResponse.connection),
     statusCode: httpResponse.statusCode,
     headers: httpResponse.headers,
-    body: body,
+    body: body
   };
 
   if (DEBUG_FORMAT === "yaml")
@@ -193,7 +193,7 @@ export function outgoingUdpMessage(
     remoteAddress: remoteAddress,
     deviceId: deviceId,
     remotePort: remotePort,
-    body: body,
+    body: body
   };
 
   if (DEBUG_FORMAT === "yaml")

lib/default-provisions.ts

@@ -21,15 +21,10 @@ import Path from "./common/path";
 import * as config from "./config";
 import * as device from "./device";
 import * as scheduling from "./scheduling";
-import { SessionContext, Declaration } from "./types";
 
 const MAX_DEPTH = +config.get("MAX_DEPTH");
 
-export function refresh(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
+export function refresh(sessionContext, provision, declarations): boolean {
   if (
     (provision.length !== 2 || typeof provision[1] !== "string") &&
     (provision.length !== 3 ||
@@ -42,7 +37,7 @@ export function refresh(
   )
     throw new Error("Invalid arguments");
 
-  const every = 1000 * ((provision[2] as number) || 1);
+  const every = 1000 * (provision[2] || 1);
   const offset = scheduling.variance(sessionContext.deviceId, every);
   const t = scheduling.interval(sessionContext.timestamp, every, offset);
 
@@ -54,7 +49,7 @@ export function refresh(
   } else {
     attrGet = {};
     refreshChildren = !!provision[3];
-    for (const a of provision.slice(4)) attrGet[a as string] = t;
+    for (const a of provision.slice(4)) attrGet[a] = t;
   }
 
   let path = Path.parse(provision[1]);
@@ -74,18 +69,14 @@ export function refresh(
       pathSet: null,
       attrGet: attrGet,
       attrSet: null,
-      defer: true,
+      defer: true
     });
   }
 
   return true;
 }
 
-export function value(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
+export function value(sessionContext, provision, declarations): boolean {
   if (
     provision.length < 3 ||
     provision.length > 4 ||
@@ -99,15 +90,15 @@ export function value(
     attr = "value";
     val = provision[2];
   } else {
-    attr = (provision[2] as string) || "";
+    attr = provision[2] || "";
     val = provision[3];
   }
 
   if (attr === "accessList") {
     val = (val || "")
       .split(",")
-      .map((s) => s.trim())
-      .filter((s) => !!s);
+      .map(s => s.trim())
+      .filter(s => !!s);
   } else if (attr === "value") {
     val = [val];
   }
@@ -118,17 +109,13 @@ export function value(
     pathSet: null,
     attrGet: { [attr]: 1 },
     attrSet: { [attr]: val },
-    defer: true,
+    defer: true
   });
 
   return true;
 }
 
-export function tag(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
+export function tag(sessionContext, provision, declarations): boolean {
   if (
     provision.length !== 3 ||
     typeof provision[1] !== "string" ||
@@ -142,17 +129,13 @@ export function tag(
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [provision[2]] },
-    defer: true,
+    defer: true
   });
 
   return true;
 }
 
-export function reboot(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
+export function reboot(sessionContext, provision, declarations): boolean {
   if (provision.length !== 1) throw new Error("Invalid arguments");
 
   declarations.push({
@@ -161,17 +144,13 @@ export function reboot(
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true,
+    defer: true
   });
 
   return true;
 }
 
-export function reset(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
+export function reset(sessionContext, provision, declarations): boolean {
   if (provision.length !== 1) throw new Error("Invalid arguments");
 
   declarations.push({
@@ -180,17 +159,13 @@ export function reset(
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true,
+    defer: true
   });
 
   return true;
 }
 
-export function download(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
+export function download(sessionContext, provision, declarations): boolean {
   if (
     (provision.length !== 3 ||
       typeof provision[1] !== "string" ||
@@ -205,7 +180,7 @@ export function download(
   const alias = [
     `FileType:${JSON.stringify(provision[1] || "")}`,
     `FileName:${JSON.stringify(provision[2] || "")}`,
-    `TargetFileName:${JSON.stringify(provision[3] || "")}`,
+    `TargetFileName:${JSON.stringify(provision[3] || "")}`
   ].join(",");
 
   declarations.push({
@@ -214,7 +189,7 @@ export function download(
     pathSet: 1,
     attrGet: null,
     attrSet: null,
-    defer: true,
+    defer: true
   });
 
   declarations.push({
@@ -223,56 +198,18 @@ export function download(
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true,
-  });
-
-  return true;
-}
-
-export function upload(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
-  if (
-    provision.length !== 3 ||
-    typeof provision[1] !== "string" ||
-    typeof provision[2] !== "string"
-  )
-    throw new Error("Invalid arguments");
-
-  const alias = [
-    `FileType:${JSON.stringify(provision[1] || "")}`,
-    `FileName:${JSON.stringify(provision[2] || "")}`,
-  ].join(",");
-
-  declarations.push({
-    path: Path.parse(`Uploads.[${alias}]`),
-    pathGet: 1,
-    pathSet: 1,
-    attrGet: null,
-    attrSet: null,
-    defer: true,
-  });
-
-  declarations.push({
-    path: Path.parse(`Uploads.[${alias}].Upload`),
-    pathGet: 1,
-    pathSet: null,
-    attrGet: { value: 1 },
-    attrSet: { value: [sessionContext.timestamp] },
-    defer: true,
+    defer: true
   });
 
   return true;
 }
 
 export function instances(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[],
-  startRevision: number,
-  endRevision: number
+  sessionContext,
+  provision,
+  declarations,
+  startRevision,
+  endRevision
 ): boolean {
   if (provision.length !== 3 || typeof provision[1] !== "string")
     throw new Error("Invalid arguments");
@@ -290,7 +227,7 @@ export function instances(
       pathSet: null,
       attrGet: null,
       attrSet: null,
-      defer: true,
+      defer: true
     });
 
     if (endRevision === startRevision) return false;
@@ -309,7 +246,7 @@ export function instances(
     pathSet: count,
     attrGet: null,
     attrSet: null,
-    defer: true,
+    defer: true
   });
 
   return true;

lib/device.ts

@@ -18,20 +18,14 @@
  */
 
 import Path from "./common/path";
-import {
-  DeviceData,
-  Declaration,
-  Clear,
-  AttributeTimestamps,
-  Attributes,
-} from "./types";
+import { DeviceData, Declaration, Clear } from "./types";
 
 const CHANGE_FLAGS = {
   object: 2,
   writable: 4,
   value: 8,
   notification: 16,
-  accessList: 32,
+  accessList: 32
 };
 
 function parseBool(v): boolean {
@@ -44,7 +38,7 @@ function parseBool(v): boolean {
 
 export function sanitizeParameterValue(
   parameterValue: [string | number | boolean, string]
-): [string | number | boolean, string] {
+): [string | number | boolean, string?] {
   if (parameterValue[0] != null) {
     switch (parameterValue[1]) {
       case "xsd:boolean":
@@ -104,8 +98,8 @@ export function getAliasDeclarations(
       pathSet: null,
       attrGet: attrGet,
       attrSet: null,
-      defer: true,
-    },
+      defer: true
+    }
   ];
 
   if (path.alias) {
@@ -115,7 +109,7 @@ export function getAliasDeclarations(
         for (const [p] of alias as [Path, string][]) {
           decs = decs.concat(
             getAliasDeclarations(parent.concat(p), timestamp, {
-              value: timestamp,
+              value: timestamp
             })
           );
         }
@@ -197,10 +191,10 @@ export function unpack(
 }
 
 export function clear(
-  deviceData: DeviceData,
-  path: Path,
-  timestamp: number,
-  attributes: AttributeTimestamps,
+  deviceData,
+  path,
+  timestamp,
+  attributes,
   changeFlags = 0
 ): void {
   const changeTrackers = {};
@@ -279,8 +273,8 @@ function compareEquality(a, b): boolean {
 export function set(
   deviceData: DeviceData,
   path: Path,
-  timestamp: number,
-  attributes: Attributes,
+  timestamp,
+  attributes,
   toClear?: Clear[]
 ): Clear[] {
   path = deviceData.paths.add(path);
@@ -308,7 +302,7 @@ export function set(
       attributes.object[1] &&
       attributes.object[0] >= (attributes.value ? attributes.value[0] : 0)
     )
-      attributes.value = [attributes.object[0], null];
+      attributes.value = [attributes.object[0]];
 
     const newAttributes = Object.assign({}, currentAttributes, attributes);
 
@@ -384,7 +378,7 @@ export function track(
   deviceData: DeviceData,
   path: Path,
   marker: string,
-  attributes?: string[]
+  attributes?
 ): void {
   path = deviceData.paths.add(path);
   let f = 1;
@@ -401,10 +395,7 @@ export function track(
   cur[marker] |= f;
 }
 
-export function clearTrackers(
-  deviceData: DeviceData,
-  tracker: string | string[]
-): void {
+export function clearTrackers(deviceData: DeviceData, tracker): void {
   if (Array.isArray(tracker)) {
     for (const v of deviceData.trackers.values())
       for (const t of tracker) delete v[t];

lib/extensions.ts

@@ -36,8 +36,8 @@ function messageHandler(message): void {
   }
 }
 
-export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
-  return new Promise((resolve) => {
+export function run(args): Promise<{ fault: Fault; value: any }> {
+  return new Promise(resolve => {
     const scriptName = args[0];
 
     const id = crypto.randomBytes(8).toString("hex");
@@ -45,20 +45,19 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
 
     if (!processes[scriptName]) {
       const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
-        stdio: ["inherit", "inherit", "inherit", "ipc"],
+        stdio: ["inherit", "inherit", "inherit", "ipc"]
       });
       processes[scriptName] = p;
 
-      p.on("error", (err) => {
+      p.on("error", err => {
         if (processes[scriptName] === p) {
           if (jobs.delete(id)) {
             resolve({
               fault: { code: err.name, message: err.message },
-              value: null,
+              value: null
             });
           }
 
-          // eslint-disable-next-line @typescript-eslint/no-floating-promises
           kill(processes[scriptName]);
           delete processes[scriptName];
         }
@@ -75,7 +74,7 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
       if (jobs.delete(id)) {
         resolve({
           fault: { code: "timeout", message: "Extension timed out" },
-          value: null,
+          value: null
         });
       }
     }, TIMEOUT);
@@ -87,7 +86,7 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
 }
 
 function kill(process: ChildProcess): Promise<void> {
-  return new Promise((resolve) => {
+  return new Promise(resolve => {
     const timeToKill = Date.now() + 5000;
 
     process.kill();

lib/forwarded.ts

@@ -1,175 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { IncomingMessage } from "http";
-import { TLSSocket } from "tls";
-import { parseCIDR, parse, IPv6, IPv4 } from "ipaddr.js";
-import * as config from "./config";
-
-interface RequestOrigin {
-  localAddress: string;
-  localPort: number;
-  remoteAddress: string;
-  remotePort: number;
-  host: string;
-  encrypted: boolean;
-}
-
-const FORWARDED_HEADER = "" + config.get("FORWARDED_HEADER");
-const cache = new WeakMap<IncomingMessage, RequestOrigin>();
-const cidrs: [IPv4 | IPv6, number][] = [];
-
-for (const str of FORWARDED_HEADER.split(",").map((s) => s.trim())) {
-  try {
-    cidrs.push(parseCIDR(str));
-  } catch (err) {
-    // Not a valid CIDR format, try parsing as IP
-    try {
-      const ip = parse(str);
-      cidrs.push([ip, ip.toByteArray().length * 8]);
-    } catch (err) {
-      // Not a valid IP either, ignore
-    }
-  }
-}
-
-function parseForwardedHeader(str: string): { [name: string]: string } {
-  str = str.toLowerCase();
-  const res: { [name: string]: string } = {};
-  let keyIdx = 0;
-  let valueIdx = -1;
-  let key: string;
-  for (let i = 0; i < str.length; ++i) {
-    const char = str.charCodeAt(i);
-    if (char === 61 /* = */) {
-      if (keyIdx >= 0) {
-        key = str.slice(keyIdx, i).trim();
-        keyIdx = -1;
-        valueIdx = i + 1;
-      }
-    } else if (char === 59 /* ; */) {
-      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
-      valueIdx = -1;
-      keyIdx = i + 1;
-    } else if (char === 44 /* , */) {
-      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
-      return res;
-    } else if (char === 34 /* " */) {
-      if (valueIdx >= 0) {
-        const quoteIdx = i;
-        if (!str.slice(valueIdx, quoteIdx).trim()) {
-          for (i = i + 1; i < str.length; ++i) {
-            const c = str.charCodeAt(i);
-            if (c === 92 /* \ */) ++i;
-            if (c === 34 /* " */) {
-              res[key] = JSON.parse(str.slice(quoteIdx, i + 1).trim());
-              valueIdx = -1;
-              keyIdx = i + 1;
-              break;
-            }
-          }
-        }
-      }
-    }
-  }
-
-  if (valueIdx >= 0) res[key] = str.slice(valueIdx).trim();
-
-  return res;
-}
-
-export function getRequestOrigin(request: IncomingMessage): RequestOrigin {
-  let origin = cache.get(request);
-  if (!origin) {
-    const soc = request.socket;
-    origin = {
-      localAddress: soc.localAddress,
-      localPort: soc.localPort,
-      remoteAddress: soc.remoteAddress,
-      remotePort: soc.remotePort,
-      host: request.headers["host"],
-      encrypted: !!(request.socket as TLSSocket).encrypted,
-    };
-
-    const header = request.headers["forwarded"];
-    if (header) {
-      const ip = parse(soc.remoteAddress) as IPv4;
-      if (cidrs.some((cidr) => ip.match(cidr as [IPv4, number]))) {
-        const parsed = parseForwardedHeader(header);
-
-        if (parsed["proto"] === "https") {
-          origin.encrypted = true;
-          origin.localPort = 443;
-        } else if (parsed["proto"] === "http") {
-          origin.encrypted = false;
-          origin.localPort = 80;
-        }
-
-        if (parsed["host"]) {
-          origin.host = parsed["host"];
-          const [, port] = parsed["host"].split(":", 2);
-          origin.localPort = +port || origin.localPort;
-        }
-
-        if (parsed["for"]) {
-          if (parsed["for"].startsWith("[")) {
-            const i = parsed["for"].lastIndexOf("]");
-            if (i >= 0) {
-              origin.remoteAddress = parsed["for"].slice(1, i);
-              origin.remotePort =
-                parseInt(parsed["for"].slice(i + 2)) || origin.remotePort;
-            }
-          } else {
-            const i = parsed["for"].lastIndexOf(":");
-            if (i >= 0) {
-              origin.remoteAddress = parsed["for"].slice(0, i);
-              origin.remotePort =
-                parseInt(parsed["for"].slice(i + 1)) || origin.remotePort;
-            } else {
-              origin.remoteAddress = parsed["for"];
-            }
-          }
-        }
-
-        if (parsed["by"]) {
-          if (parsed["by"].startsWith("[")) {
-            const i = parsed["by"].lastIndexOf("]");
-            if (i >= 0) {
-              origin.localAddress = parsed["by"].slice(1, i);
-              origin.localPort =
-                parseInt(parsed["by"].slice(i + 2)) || origin.localPort;
-            }
-          } else {
-            const i = parsed["by"].lastIndexOf(":");
-            if (i >= 0) {
-              origin.localAddress = parsed["by"].slice(0, i);
-              origin.localPort =
-                parseInt(parsed["by"].slice(i + 1)) || origin.localPort;
-            } else {
-              origin.localAddress = parsed["by"];
-            }
-          }
-        }
-      }
-    }
-
-    cache.set(request, origin);
-  }
-  return origin;
-}

lib/fs.ts

@@ -19,43 +19,11 @@
 
 import * as url from "url";
 import * as querystring from "querystring";
-import { IncomingMessage, ServerResponse } from "http";
-import { Collection, GridFSBucket, ObjectId } from "mongodb";
-import { onConnect, getOperations } from "./db";
+import { GridFSBucket } from "mongodb";
+import * as db from "./db";
 import * as logger from "./logger";
-import { getRequestOrigin } from "./forwarded";
 
-let filesCollection: Collection;
-let filesBucket: GridFSBucket;
-let uploadsBucket: GridFSBucket;
-
-onConnect(async (db) => {
-  filesCollection = db.collection("fs.files");
-  filesBucket = new GridFSBucket(db);
-  uploadsBucket = new GridFSBucket(db, { bucketName: "uploads" });
-});
-
-async function canUpload(
-  deviceId: string,
-  fileName: string,
-  timeout = Date.now() + 5000
-): Promise<boolean> {
-  const operations = Object.values(await getOperations(deviceId));
-  for (const operation of operations) {
-    if (operation.name === "Upload" && operation.args.fileName === fileName)
-      return true;
-  }
-
-  if (Date.now() >= timeout) return false;
-  // Need to wait and retry in case upload was initiated before session was closed
-  await new Promise((resolve) => setTimeout(resolve, 1000));
-  return canUpload(deviceId, fileName, timeout);
-}
-
-export async function listener(
-  request: IncomingMessage,
-  response: ServerResponse
-): Promise<void> {
+export function listener(request, response): void {
   const urlParts = url.parse(request.url, true);
   if (request.method === "GET") {
     const filename = querystring.unescape(urlParts.pathname.substring(1));
@@ -63,10 +31,12 @@ export async function listener(
     const log = {
       message: "Fetch file",
       filename: filename,
-      remoteAddress: getRequestOrigin(request).remoteAddress,
+      remoteAddress: request.connection.remoteAddress
     };
 
-    const file = await filesCollection.findOne({ _id: filename });
+    db.filesCollection.findOne({ _id: filename }, (err, file) => {
+      if (err) throw err;
+
       if (!file) {
         response.writeHead(404);
         response.end();
@@ -77,47 +47,15 @@ export async function listener(
 
       response.writeHead(200, {
         "Content-Type": file.contentType || "application/octet-stream",
-      "Content-Length": file.length,
+        "Content-Length": file.length
       });
 
-    const downloadStream = filesBucket.openDownloadStreamByName(filename);
+      const bucket = new GridFSBucket(db.client.db());
+      const downloadStream = bucket.openDownloadStreamByName(filename);
       downloadStream.pipe(response);
 
       logger.accessInfo(log);
-  } else if (request.method === "PUT") {
-    const [, deviceId, ...filePath] = urlParts.pathname
-      .split("/")
-      .map(querystring.unescape);
-
-    const fileName = `${deviceId}/${filePath.join("/")}`;
-
-    if (!(await canUpload(deviceId, fileName))) {
-      response.writeHead(403);
-      response.end("403 Forbidden");
-      return;
-    }
-
-    try {
-      await uploadsBucket.delete((fileName as unknown) as ObjectId);
-    } catch (err) {
-      // File not found, do nothing
-    }
-    const uploadStream = uploadsBucket.openUploadStreamWithId(
-      fileName,
-      fileName
-    );
-
-    uploadStream.on("finish", () => {
-      response.writeHead(200);
-      response.end();
     });
-
-    uploadStream.on("error", (err) => {
-      response.writeHead(500);
-      response.end(err.message);
-    });
-
-    request.pipe(uploadStream);
   } else {
     response.writeHead(405, { Allow: "GET" });
     response.end("405 Method Not Allowed");

lib/gpn-heuristic.ts

@@ -31,7 +31,7 @@ const UNDISCOVERED_DEPTH = 7;
 // point has been discovered.
 export function estimateGpnCount(
   gpnPatterns: [Path, number][],
-  depth = 0
+  depth: number = 0
 ): number {
   const children: { [segment: string]: [Path, number][] } = {};
   const wildcardChildren: [Path, number][] = [];

lib/init.ts

@@ -95,11 +95,11 @@ export async function getStatus(): Promise<Status> {
     filters: !Object.keys(ui["filters"]).length,
     device: !Object.keys(ui["device"]).length,
     index: !Object.keys(ui["index"]).length,
-    overview: !Object.keys(ui["overview"]).length,
+    overview: !Object.keys(ui["overview"]).length
   };
 }
 
-export async function seed(options: Record<string, boolean>): Promise<void> {
+export async function seed(options): Promise<void> {
   const resources = {};
   const proms = [];
 
@@ -108,7 +108,6 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
       { role: "admin", resource: "devices", access: 3, validate: "true" },
       { role: "admin", resource: "faults", access: 3, validate: "true" },
       { role: "admin", resource: "files", access: 3, validate: "true" },
-      { role: "admin", resource: "uploads", access: 3, validate: "true" },
       { role: "admin", resource: "presets", access: 3, validate: "true" },
       { role: "admin", resource: "provisions", access: 3, validate: "true" },
       { role: "admin", resource: "config", access: 3, validate: "true" },
@@ -118,12 +117,12 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
         role: "admin",
         resource: "virtualParameters",
         access: 3,
-        validate: "true",
-      },
+        validate: "true"
+      }
     ];
 
     resources["users"] = [
-      { username: "admin", password: "admin", roles: ["admin"] },
+      { username: "admin", password: "admin", roles: ["admin"] }
     ];
   }
 
@@ -136,7 +135,7 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
       { _id: "ui.filters.1.parameter", value: "DeviceID.ProductClass" },
       { _id: "ui.filters.1.type", value: "'string'" },
       { _id: "ui.filters.2.label", value: "'Tag'" },
-      { _id: "ui.filters.2.type", value: "'tag'" },
+      { _id: "ui.filters.2.type", value: "'tag'" }
     ]);
   }
 
@@ -150,118 +149,118 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
       { _id: "ui.device.2.parameters.0.label", value: "'Last inform'" },
       {
         _id: "ui.device.2.parameters.0.components.0.type",
-        value: "'parameter'",
+        value: "'parameter'"
       },
       { _id: "ui.device.2.parameters.0.components.1.chart", value: "'online'" },
       {
         _id: "ui.device.2.parameters.0.components.1.type",
-        value: "'overview-dot'",
+        value: "'overview-dot'"
       },
       {
         _id: "ui.device.2.parameters.0.components.2.type",
-        value: "'summon-button'",
+        value: "'summon-button'"
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.0",
-        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.1",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.2",
         value:
-          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress",
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress"
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.3",
         value:
-          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress",
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress"
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.4",
-        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID",
+        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID"
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.5",
         value:
-          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase",
+          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase"
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.6",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName"
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.7",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress"
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.8",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress"
       },
       {
         _id: "ui.device.2.parameters.0.parameter",
-        value: "DATE_STRING(Events.Inform)",
+        value: "DATE_STRING(Events.Inform)"
       },
       { _id: "ui.device.2.parameters.1.label", value: "'Serial number'" },
       {
         _id: "ui.device.2.parameters.1.parameter",
-        value: "DeviceID.SerialNumber",
+        value: "DeviceID.SerialNumber"
       },
       { _id: "ui.device.2.parameters.2.label", value: "'Product class'" },
       {
         _id: "ui.device.2.parameters.2.parameter",
-        value: "DeviceID.ProductClass",
+        value: "DeviceID.ProductClass"
       },
       { _id: "ui.device.2.parameters.3.label", value: "'OUI'" },
       {
         _id: "ui.device.2.parameters.3.parameter",
-        value: "DeviceID.OUI",
+        value: "DeviceID.OUI"
       },
       { _id: "ui.device.2.parameters.4.label", value: "'Manufacturer'" },
       {
         _id: "ui.device.2.parameters.4.parameter",
-        value: "DeviceID.Manufacturer",
+        value: "DeviceID.Manufacturer"
       },
       { _id: "ui.device.2.parameters.5.label", value: "'Hardware version'" },
       {
         _id: "ui.device.2.parameters.5.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
       },
       { _id: "ui.device.2.parameters.6.label", value: "'Software version'" },
       {
         _id: "ui.device.2.parameters.6.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
       },
       { _id: "ui.device.2.parameters.7.label", value: "'MAC'" },
       {
         _id: "ui.device.2.parameters.7.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress",
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress"
       },
       { _id: "ui.device.2.parameters.8.label", value: "'IP'" },
       {
         _id: "ui.device.2.parameters.8.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
       },
       { _id: "ui.device.2.parameters.9.label", value: "'WLAN SSID'" },
       {
         _id: "ui.device.2.parameters.9.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
       },
       { _id: "ui.device.2.parameters.10.label", value: "'WLAN passphrase'" },
       {
         _id: "ui.device.2.parameters.10.parameter",
         value:
-          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
+          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase"
       },
       { _id: "ui.device.3.type", value: "'parameter-table'" },
       { _id: "ui.device.3.label", value: "'WLAN hosts'" },
       {
         _id: "ui.device.3.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host",
+        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host"
       },
       { _id: "ui.device.3.childParameters.0.label", value: "'Host name'" },
       { _id: "ui.device.3.childParameters.0.parameter", value: "HostName" },
@@ -279,12 +278,12 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
       { _id: "ui.device.5.element", value: "'div.container-full-width'" },
       {
         _id: "ui.device.5.components.0.components.0",
-        value: "'All parameters'",
+        value: "'All parameters'"
       },
       { _id: "ui.device.5.components.0.element", value: "'h3'" },
       { _id: "ui.device.5.components.0.type", value: "'container'" },
       { _id: "ui.device.5.components.1.type", value: "'all-parameters'" },
-      { _id: "ui.device.6.type", value: "'device-actions'" },
+      { _id: "ui.device.6.type", value: "'device-actions'" }
     ]);
   }
 
@@ -299,18 +298,18 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
       { _id: "ui.index.2.label", value: "'Software version'" },
       {
         _id: "ui.index.2.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
       },
       { _id: "ui.index.3.label", value: "'IP'" },
       {
         _id: "ui.index.3.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
       },
       { _id: "ui.index.4.label", value: "'SSID'" },
       {
         _id: "ui.index.4.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
       },
       { _id: "ui.index.5.type", value: "'container'" },
       { _id: "ui.index.5.label", value: "'Last inform'" },
@@ -323,7 +322,7 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
       { _id: "ui.index.6.label", value: "'Tags'" },
       { _id: "ui.index.6.parameter", value: "Tags" },
       { _id: "ui.index.6.unsortable", value: "true" },
-      { _id: "ui.index.6.writable", value: "false" },
+      { _id: "ui.index.6.writable", value: "false" }
     ]);
   }
 
@@ -332,44 +331,43 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
       { _id: "ui.overview.charts.online.label", value: "'Online status'" },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.color",
-        value: "'#31a354'",
+        value: "'#31a354'"
       },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.filter",
-        value: "Events.Inform > NOW() - 5 * 60 * 1000",
+        value: "Events.Inform > NOW() - 5 * 60 * 1000"
       },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.label",
-        value: "'Online now'",
+        value: "'Online now'"
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.color",
-        value: "'#a1d99b'",
+        value: "'#a1d99b'"
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.filter",
         value:
-          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)",
+          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)"
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.label",
-        value: "'Past 24 hours'",
+        value: "'Past 24 hours'"
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.color",
-        value: "'#e5f5e0'",
+        value: "'#e5f5e0'"
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.filter",
-        value:
-          "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)",
+        value: "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)"
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.label",
-        value: "'Others'",
+        value: "'Others'"
       },
       { _id: "ui.overview.groups.online.label", value: "''" },
-      { _id: "ui.overview.groups.online.charts.0", value: "'online'" },
+      { _id: "ui.overview.groups.online.charts.0", value: "'online'" }
     ]);
   }
 
@@ -380,16 +378,16 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
         weight: 0,
         channel: "bootstrap",
         events: "0 BOOTSTRAP",
-        provision: "bootstrap",
+        provision: "bootstrap"
       },
       { _id: "default", weight: 0, channel: "default", provision: "default" },
-      { _id: "inform", weight: 0, channel: "inform", provision: "inform" },
+      { _id: "inform", weight: 0, channel: "inform", provision: "inform" }
     ];
 
     resources["provisions"] = [
       { _id: "bootstrap", script: BOOTSTRAP_SCRIPT },
       { _id: "default", script: DEFAULT_SCRIPT },
-      { _id: "inform", script: INFORM_SCRIPT },
+      { _id: "inform", script: INFORM_SCRIPT }
     ];
   }
 

lib/local-cache.ts

@@ -22,39 +22,30 @@ import * as crypto from "crypto";
 import * as config from "./config";
 import * as db from "./db";
 import * as cache from "./cache";
-import { convertOldPrecondition } from "./mongodb-functions";
+import { mongoQueryToFilter } from "./mongodb-functions";
 import * as logger from "./logger";
 import * as scheduling from "./scheduling";
 import * as expression from "./common/expression";
 import { parse } from "./common/expression-parser";
-import {
-  Preset,
-  Expression,
-  Provisions,
-  VirtualParameters,
-  Files,
-  Users,
-  Permissions,
-  Config,
-  UiConfig,
-} from "./types";
+import { Preset, Expression } from "./types";
 
-interface Snapshot {
-  presets: Preset[];
-  provisions: Provisions;
-  virtualParameters: VirtualParameters;
-  files: Files;
-  permissions: Permissions;
-  users: Users;
-  config: Config;
-  ui: UiConfig;
+interface Permissions {
+  [role: string]: {
+    [access: number]: {
+      [resource: string]: {
+        access: number;
+        filter: Expression;
+        validate?: Expression;
+      };
+    };
+  };
 }
 
 const REFRESH = 3000;
 const EVICT_TIMEOUT = 60000;
 
-const snapshots = new Map<string, Snapshot>();
-let currentSnapshot: string = null;
+const snapshots = new Map();
+let currentSnapshot = null;
 let nextRefresh = 1;
 
 function computeHash(snapshot): string {
@@ -107,15 +98,11 @@ function computeHash(snapshot): string {
   return h.digest("hex");
 }
 
-function flattenObject(
-  src: Record<string, unknown>,
-  prefix = "",
-  dst = {}
-): Record<string, unknown> {
+function flattenObject(src, prefix = "", dst = {}): {} {
   for (const k of Object.keys(src)) {
     const v = src[k];
     if (typeof v === "object" && !Array.isArray(v))
-      flattenObject(v as Record<string, unknown>, `${prefix}${k}.`, dst);
+      flattenObject(v, `${prefix}${k}.`, dst);
     else dst[`${prefix}${k}`] = v;
   }
   return dst;
@@ -125,14 +112,14 @@ async function fetchPresets(): Promise<Preset[]> {
   const res = await db.getPresets();
   let objects = await db.getObjects();
 
-  objects = objects.map((obj) => {
+  objects = objects.map(obj => {
     // Flatten object
     obj = flattenObject(obj);
 
     // If no keys are defined, consider all parameters as keys to keep the
     // same behavior from v1.0
     if (!obj["_keys"] || !obj["_keys"].length)
-      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
+      obj["_keys"] = Object.keys(obj).filter(k => !k.startsWith("_"));
 
     return obj;
   });
@@ -149,9 +136,12 @@ async function fetchPresets(): Promise<Preset[]> {
     if (preset["schedule"]) {
       const parts = preset["schedule"].trim().split(/\s+/);
       schedule = {
-        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
+        md5: crypto
+          .createHash("md5")
+          .update(preset["schedule"])
+          .digest("hex"),
         duration: null,
-        schedule: null,
+        schedule: null
       };
 
       try {
@@ -161,7 +151,7 @@ async function fetchPresets(): Promise<Preset[]> {
         logger.warn({
           message: "Invalid preset schedule",
           preset: preset["_id"],
-          schedule: preset["schedule"],
+          schedule: preset["schedule"]
         });
         schedule.schedule = false;
       }
@@ -173,9 +163,7 @@ async function fetchPresets(): Promise<Preset[]> {
       try {
         precondition = parse(preset["precondition"]);
       } catch (error) {
-        precondition = convertOldPrecondition(
-          JSON.parse(preset["precondition"])
-        );
+        precondition = mongoQueryToFilter(JSON.parse(preset["precondition"]));
       }
 
       // Simplify expression
@@ -188,7 +176,7 @@ async function fetchPresets(): Promise<Preset[]> {
     for (const c of preset["configurations"]) {
       switch (c.type) {
         case "age":
-          _provisions.push(["refresh", c.name, +c.age]);
+          _provisions.push(["refresh", c.name, c.age]);
           break;
 
         case "value":
@@ -211,7 +199,7 @@ async function fetchPresets(): Promise<Preset[]> {
           for (const obj of objects) {
             if (obj["_id"] === c.object) {
               const alias = obj["_keys"]
-                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .map(k => `${k}:${JSON.stringify(obj[k])}`)
                 .join(",");
               const p = `${c.name}.[${alias}]`;
               _provisions.push(["instances", p, 1]);
@@ -229,7 +217,7 @@ async function fetchPresets(): Promise<Preset[]> {
           for (const obj of objects) {
             if (obj["_id"] === c.object) {
               const alias = obj["_keys"]
-                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .map(k => `${k}:${JSON.stringify(obj[k])}`)
                 .join(",");
               const p = `${c.name}.[${alias}]`;
               _provisions.push(["instances", p, 0]);
@@ -249,14 +237,14 @@ async function fetchPresets(): Promise<Preset[]> {
       schedule: schedule,
       events: events,
       precondition: precondition,
-      provisions: _provisions,
+      provisions: _provisions
     });
   }
 
   return presets;
 }
 
-async function fetchProvisions(): Promise<Provisions> {
+async function fetchProvisions(): Promise<{}> {
   const res = await db.getProvisions();
 
   const provisions = {};
@@ -266,9 +254,7 @@ async function fetchProvisions(): Promise<Provisions> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    provisions[
-      r["_id"]
-    ].script = new vm.Script(
+    provisions[r["_id"]].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -277,7 +263,7 @@ async function fetchProvisions(): Promise<Provisions> {
   return provisions;
 }
 
-async function fetchVirtualParameters(): Promise<VirtualParameters> {
+async function fetchVirtualParameters(): Promise<{}> {
   const res = await db.getVirtualParameters();
 
   const virtualParameters = {};
@@ -287,9 +273,7 @@ async function fetchVirtualParameters(): Promise<VirtualParameters> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    virtualParameters[
-      r["_id"]
-    ].script = new vm.Script(
+    virtualParameters[r["_id"]].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -308,7 +292,7 @@ async function fetchPermissions(): Promise<Permissions> {
 
     permissions[p.role][p.access][p.resource] = {
       access: p.access,
-      filter: parse(p.filter || "true"),
+      filter: parse(p.filter || "true")
     };
     if (p.validate)
       permissions[p.role][p.access][p.resource].validate = parse(p.validate);
@@ -317,7 +301,7 @@ async function fetchPermissions(): Promise<Permissions> {
   return permissions;
 }
 
-async function fetchFiles(): Promise<Files> {
+async function fetchFiles(): Promise<{}> {
   const res = await db.getFiles();
   const files = {};
 
@@ -332,7 +316,7 @@ async function fetchFiles(): Promise<Files> {
   return files;
 }
 
-async function fetchUsers(): Promise<Users> {
+async function fetchUsers(): Promise<{}> {
   const _users = await db.getUsers();
   const users = {};
 
@@ -340,14 +324,14 @@ async function fetchUsers(): Promise<Users> {
     users[user._id] = {
       password: user.password,
       salt: user.salt,
-      roles: user.roles.split(",").map((s) => s.trim()),
+      roles: user.roles.split(",").map(s => s.trim())
     };
   }
 
   return users;
 }
 
-async function fetchConfig(): Promise<[Config, UiConfig]> {
+async function fetchConfig(): Promise<[{}, {}]> {
   const conf = await db.getConfig();
 
   conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
@@ -356,7 +340,7 @@ async function fetchConfig(): Promise<[Config, UiConfig]> {
     filters: {},
     device: {},
     index: {},
-    overview: {},
+    overview: {}
   };
   const _config = {};
 
@@ -383,7 +367,7 @@ async function fetchConfig(): Promise<[Config, UiConfig]> {
 
 async function refresh(): Promise<void> {
   if (!nextRefresh) {
-    await new Promise((resolve) => setTimeout(resolve, 20));
+    await new Promise(resolve => setTimeout(resolve, 20));
     await refresh();
     return;
   }
@@ -398,7 +382,7 @@ async function refresh(): Promise<void> {
     return;
   }
 
-  const lockToken = await cache.acquireLock("presets_hash_lock", 4000, 4000);
+  const unlockOrExtend = await cache.lock("presets_hash_lock", 3);
 
   const res = await Promise.all([
     fetchPresets(),
@@ -407,7 +391,7 @@ async function refresh(): Promise<void> {
     fetchFiles(),
     fetchPermissions(),
     fetchUsers(),
-    fetchConfig(),
+    fetchConfig()
   ]);
 
   const snapshot = {
@@ -418,7 +402,7 @@ async function refresh(): Promise<void> {
     permissions: res[4],
     users: res[5],
     config: res[6][0],
-    ui: res[6][1],
+    ui: res[6][1]
   };
 
   if (currentSnapshot) {
@@ -432,7 +416,7 @@ async function refresh(): Promise<void> {
   currentSnapshot = computeHash(snapshot);
   snapshots.set(currentSnapshot, snapshot);
   await cache.set("presets_hash", currentSnapshot, 300);
-  await cache.releaseLock("presets_hash_lock", lockToken);
+  await unlockOrExtend(0);
 
   nextRefresh = now + (REFRESH - (now % REFRESH));
 }
@@ -442,40 +426,45 @@ export async function getCurrentSnapshot(): Promise<string> {
   return currentSnapshot;
 }
 
-export function hasSnapshot(hash: string): boolean {
+export function hasSnapshot(hash): boolean {
   return snapshots.has(hash);
 }
 
-export function getPresets(snapshotKey: string): Preset[] {
+export function getPresets(snapshotKey): Preset[] {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.presets;
 }
 
-export function getProvisions(snapshotKey: string): Provisions {
+export function getProvisions(
+  snapshotKey
+): { [name: string]: { md5: string; script: vm.Script } } {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.provisions;
 }
 
-export function getVirtualParameters(snapshotKey: string): VirtualParameters {
+export function getVirtualParameters(
+  snapshotKey
+): { [name: string]: { md5: string; script: vm.Script } } {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.virtualParameters;
 }
 
-export function getFiles(snapshotKey: string): Files {
+export function getFiles(
+  snapshotKey
+): { [name: string]: { length: number; md5: string; contentType: string } } {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
   return snapshot.files;
 }
 
 export function getConfig(
-  snapshotKey: string,
-  key: string,
-  context: Record<string, unknown>,
-  now: number,
-  cb?: (e: Expression) => Expression
+  snapshotKey,
+  key,
+  context?,
+  now?
 ): string | number | boolean | null {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
@@ -495,52 +484,39 @@ export function getConfig(
       "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
     "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
     "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
-    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT"
   };
 
   if (!(key in snapshot.config)) {
-    if (key in oldOpts) {
-      let id;
-      if (context && context["id"]) {
-        id = context["id"];
-      } else if (cb) {
-        id = cb(["PARAM", "DeviceID.ID"]);
-        if (Array.isArray(id)) id = null;
+    if (key in oldOpts)
+      return config.get(oldOpts[key], context ? context.id : null);
+    else return null;
   }
-      return config.get(oldOpts[key], id);
-    }
-    return null;
+  return expression.evaluate(snapshot.config[key], context, now || Date.now());
 }
 
-  const v = expression.evaluate(snapshot.config[key], context, now, cb);
-  return Array.isArray(v) ? null : v;
-}
-
-export function getConfigExpression(
-  snapshotKey: string,
-  key: string
-): Expression {
+export function getConfigExpression(snapshotKey, key): Expression {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 
   return snapshot.config[key];
 }
 
-export function getUsers(snapshotKey: string): Users {
+export function getUsers(snapshotKey): {} {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 
   return snapshot.users;
 }
 
-export function getPermissions(snapshotKey: string): Permissions {
+export function getPermissions(snapshotKey): {} {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 
   return snapshot.permissions;
 }
 
-export function getUiConfig(snapshotKey: string): UiConfig {
+export function getUiConfig(snapshotKey): {} {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");
 

lib/logger.ts

@@ -21,15 +21,6 @@ import * as fs from "fs";
 import * as os from "os";
 
 import * as config from "./config";
-import { getRequestOrigin } from "./forwarded";
-import {
-  SessionContext,
-  AcsRequest,
-  CpeRequest,
-  CpeFault,
-  InformRequest,
-  Fault,
-} from "./types";
 
 const REOPEN_EVERY = 60000;
 
@@ -75,7 +66,7 @@ function reopen(): void {
       if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
         logStream.end();
         logStream = fs.createWriteStream(null, {
-          fd: fs.openSync(LOG_FILE, "a"),
+          fd: fs.openSync(LOG_FILE, "a")
         });
         logStat = fs.fstatSync(logStream.fd);
       }
@@ -99,7 +90,7 @@ function reopen(): void {
       ) {
         accessLogStream.end();
         accessLogStream = fs.createWriteStream(null, {
-          fd: fs.openSync(ACCESS_LOG_FILE, "a"),
+          fd: fs.openSync(ACCESS_LOG_FILE, "a")
         });
         accessLogStat = fs.fstatSync(accessLogStream.fd);
       }
@@ -113,7 +104,7 @@ function reopen(): void {
     setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
 }
 
-export function init(service: string, version: string): void {
+export function init(service, version): void {
   defaultMeta.hostname = os.hostname();
   defaultMeta.pid = process.pid;
   defaultMeta.name = `genieacs-${service}`;
@@ -129,7 +120,7 @@ export function init(service: string, version: string): void {
 
   if (ACCESS_LOG_FILE) {
     accessLogStream = fs.createWriteStream(null, {
-      fd: fs.openSync(ACCESS_LOG_FILE, "a"),
+      fd: fs.openSync(ACCESS_LOG_FILE, "a")
     });
     accessLogStat = fs.fstatSync(accessLogStream.fd);
   }
@@ -155,75 +146,61 @@ export function close(): void {
   logStream.end();
 }
 
-export function flatten(
-  details: Record<string, unknown>
-): Record<string, unknown> {
+export function flatten(details): {} {
   if (details.sessionContext) {
-    const sessionContext = details.sessionContext as SessionContext;
-    details.deviceId = sessionContext.deviceId;
-    details.remoteAddress = getRequestOrigin(
-      sessionContext.httpRequest
-    ).remoteAddress;
+    details.deviceId = details.sessionContext.deviceId;
+    details.remoteAddress =
+      details.sessionContext.httpRequest.connection.remoteAddress;
     delete details.sessionContext;
   }
 
   if (details.exception) {
-    const err = details.exception as Error;
-    details.exceptionName = err.name;
-    details.exceptionMessage = err.message;
-    details.exceptionStack = err.stack;
+    details.exceptionName = details.exception.name;
+    details.exceptionMessage = details.exception.message;
+    details.exceptionStack = details.exception.stack;
     delete details.exception;
   }
 
   if (details.task) {
-    details.taskId = details.task["_id"];
+    details.taskId = details.task._id;
     delete details.task;
   }
 
   if (details.rpc) {
-    const rpc = details.rpc as {
-      id: string;
-      acsRequest?: AcsRequest;
-      cpeRequest?: CpeRequest;
-      cpeFault?: CpeFault;
-    };
-    if (rpc.acsRequest) {
-      details.acsRequestId = rpc.id;
-      details.acsRequestName = rpc.acsRequest.name;
-      if (rpc.acsRequest["commandKey"])
-        details.acsRequestCommandKey = rpc.acsRequest["commandKey"];
-    } else if (rpc.cpeRequest) {
-      details.cpeRequestId = rpc.id;
-      if (rpc.cpeRequest.name === "Inform") {
-        details.informEvent = (rpc.cpeRequest as InformRequest).event.join(",");
-        details.informRetryCount = (rpc.cpeRequest as InformRequest).retryCount;
+    if (details.rpc.acsRequest) {
+      details.acsRequestId = details.rpc.id;
+      details.acsRequestName = details.rpc.acsRequest.name;
+      if (details.rpc.acsRequest.commandKey)
+        details.acsRequestCommandKey = details.rpc.acsRequest.commandKey;
+    } else if (details.rpc.cpeRequest) {
+      details.cpeRequestId = details.rpc.id;
+      if (details.rpc.cpeRequest.name === "Inform") {
+        details.informEvent = details.rpc.cpeRequest.event.join(",");
+        details.informRetryCount = details.rpc.cpeRequest.retryCount;
       } else {
-        details.cpeRequestName = rpc.cpeRequest.name;
-        if (rpc.cpeRequest["commandKey"])
-          details.cpeRequestCommandKey = rpc.cpeRequest["commandKey"];
+        details.cpeRequestName = details.rpc.cpeRequest.name;
+        if (details.rpc.cpeRequest.commandKey)
+          details.cpeRequestCommandKey = details.rpc.cpeRequest.commandKey;
       }
-    } else if (rpc.cpeFault) {
-      details.acsRequestId = rpc.id;
-      details.cpeFaultCode = rpc.cpeFault.detail.faultCode;
-      details.cpeFaultString = rpc.cpeFault.detail.faultString;
+    } else if (details.rpc.cpeFault) {
+      details.acsRequestId = details.rpc.id;
+      details.cpeFaultCode = details.rpc.cpeFault.detail.faultCode;
+      details.cpeFaultString = details.rpc.cpeFault.detail.faultString;
     }
     delete details.rpc;
   }
 
   if (details.fault) {
-    const fault = details.fault as Fault;
-    details.faultCode = fault.code;
-    details.faultMessage = fault.message;
+    details.faultCode = details.fault.code;
+    details.faultMessage = details.fault.message;
     delete details.fault;
   }
 
   // For genieacs-ui
   if (details.context) {
-    details.remoteAddress = getRequestOrigin(
-      details.context["req"]
-    ).remoteAddress;
-    if (details.context["state"].user)
-      details.user = details.context["state"].user.username;
+    details.remoteAddress = details.context.request.ip;
+    if (details.context.state.user)
+      details.user = details.context.state.user.username;
     delete details.context;
   }
 
@@ -233,10 +210,7 @@ export function flatten(
   return details;
 }
 
-function formatJson(
-  details: Record<string, unknown>,
-  systemd: boolean
-): string {
+function formatJson(details, systemd): string {
   if (systemd) {
     let severity = "";
     if (details.severity === "info") severity = "<6>";
@@ -249,17 +223,14 @@ function formatJson(
   return `${JSON.stringify(flatten(details))}${os.EOL}`;
 }
 
-function formatSimple(
-  details: Record<string, unknown>,
-  systemd: boolean
-): string {
+function formatSimple(details, systemd): string {
   const skip = {
     user: true,
     remoteAddress: true,
     severity: true,
     timestamp: true,
     message: true,
-    deviceId: !!details.sessionContext,
+    deviceId: !!details.sessionContext
   };
 
   flatten(details);
@@ -292,14 +263,12 @@ function formatSimple(
     return `${severity}${remote}${details.message}${meta}${os.EOL}`;
   }
 
-  return `${
-    details.timestamp
-  } [${(details.severity as string).toUpperCase()}] ${remote}${
+  return `${details.timestamp} [${details.severity.toUpperCase()}] ${remote}${
     details.message
   }${meta}${os.EOL}`;
 }
 
-function log(details: Record<string, unknown>): void {
+function log(details): void {
   details.timestamp = new Date().toISOString();
   if (LOG_FORMAT === "json") {
     details = Object.assign({}, defaultMeta, details);
@@ -309,22 +278,22 @@ function log(details: Record<string, unknown>): void {
   }
 }
 
-export function info(details: Record<string, unknown>): void {
+export function info(details): void {
   details.severity = "info";
   log(details);
 }
 
-export function warn(details: Record<string, unknown>): void {
+export function warn(details): void {
   details.severity = "warn";
   log(details);
 }
 
-export function error(details: Record<string, unknown>): void {
+export function error(details): void {
   details.severity = "error";
   log(details);
 }
 
-export function accessLog(details: Record<string, unknown>): void {
+export function accessLog(details): void {
   details.timestamp = new Date().toISOString();
   if (ACCESS_LOG_FORMAT === "json") {
     Object.assign(details, defaultMeta);
@@ -334,17 +303,17 @@ export function accessLog(details: Record<string, unknown>): void {
   }
 }
 
-export function accessInfo(details: Record<string, unknown>): void {
+export function accessInfo(details): void {
   details.severity = "info";
   accessLog(details);
 }
 
-export function accessWarn(details: Record<string, unknown>): void {
+export function accessWarn(details): void {
   details.severity = "warn";
   accessLog(details);
 }
 
-export function accessError(details: Record<string, unknown>): void {
+export function accessError(details): void {
   details.severity = "error";
   accessLog(details);
 }

lib/mongodb-functions.ts

@@ -18,15 +18,14 @@
  */
 
 import { ObjectID } from "mongodb";
-import { map, parse, stringify, parseList } from "./common/expression-parser";
-import { likePatternToRegExp } from "./common/expression";
+import { map, parse, stringify } from "./common/expression-parser";
+import { likePatternToRegExp, evaluate } from "./common/expression";
 import { Expression, Fault, Task } from "./types";
-import { decodeTag, encodeTag } from "./common";
 
 const isArray = Array.isArray;
 
-export function processDeviceFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
+export function processDeviceFilter(filter): Expression {
+  return map(filter, exp => {
     if (!isArray(exp)) return exp;
 
     if (exp[0] === "PARAM") {
@@ -44,26 +43,17 @@ export function processDeviceFilter(filter: Expression): Expression {
       exp[1][0] === "PARAM" &&
       exp[1][1].startsWith("Tags.")
     ) {
-      const t = decodeTag(exp[1][1].slice(5));
-      if (exp[0] === "IS NULL") return ["_tags", t, false];
-      else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
-      else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
-      else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
+      const t = exp[1][1].slice(5);
+      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
+      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "=" && exp[2] === true)
+        return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "<>" && exp[2] !== true)
+        return ["=", ["PARAM", "_tags"], t];
     } else if (
-      [
-        "=",
-        "<>",
-        ">",
-        ">=",
-        "<",
-        "<=",
-        "LIKE",
-        "NOT LIKE",
-        "IS NULL",
-        "IS NOT NULL",
-      ].includes(exp[0])
+      ["=", "<>", ">", ">=", "<", "<=", "LIKE", "NOT LIKE"].includes(exp[0])
     ) {
-      let e = map(exp, (ee) => {
+      let e = map(exp, ee => {
         if (
           isArray(ee) &&
           ee[0] === "PARAM" &&
@@ -85,8 +75,8 @@ export function processDeviceFilter(filter: Expression): Expression {
   });
 }
 
-export function processTasksFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
+export function processTasksFilter(filter): Expression {
+  return map(filter, exp => {
     if (!isArray(exp)) return exp;
     if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
       const e = exp.slice();
@@ -101,8 +91,8 @@ export function processTasksFilter(filter: Expression): Expression {
   });
 }
 
-export function processFaultsFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
+export function processFaultsFilter(filter): Expression {
+  return map(filter, exp => {
     if (!isArray(exp)) return exp;
     if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
       const e = exp.slice();
@@ -115,137 +105,140 @@ export function processFaultsFilter(filter: Expression): Expression {
   });
 }
 
-export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
-  function recursive(filter, negate): Record<string, unknown> {
+export function filterToMongoQuery(exp: Expression): {} {
+  const ops = {
+    OR: 0,
+    AND: 0,
+    NOT: 0,
+    "=": 1,
+    "<>": 1,
+    ">": 1,
+    ">=": 1,
+    "<": 1,
+    "<=": 1,
+    LIKE: 1,
+    "NOT LIKE": 1,
+    "IS NULL": 1,
+    "IS NOT NULL": 1
+  };
+
+  function recursive(filter, negate, res = {}): {} {
     const op = filter[0];
 
-    if (op === "AND") {
-      filter = filter.filter((f) => f !== true);
-      if (filter.length === 2) return recursive(filter[1], negate);
+    if (ops[op] === 0) {
+      for (let i = 1; i < filter.length; ++i) {
+        if (!isArray(filter[i]) || ops[filter[i][0]] == null)
+          throw new Error(`Invalid expression in ${op} clause`);
       }
 
-    if (op === "OR") {
-      filter = filter.filter((f) => f !== false);
-      if (filter.length === 2) return recursive(filter[1], negate);
+      if ((!negate && op === "AND") || (negate && op === "OR")) {
+        res["$and"] = res["$and"] || [];
+        for (let i = 1; i < filter.length; ++i)
+          res["$and"].push(recursive(filter[i], negate));
+      } else if ((!negate && op === "OR") || (negate && op === "AND")) {
+        res["$or"] = res["$or"] || [];
+
+        for (let i = 1; i < filter.length; ++i)
+          res["$or"].push(recursive(filter[i], negate));
+      } else if (op === "NOT") {
+        recursive(filter[1], !negate, res);
+      }
+    } else if (ops[op] === 1) {
+      if (isArray(filter[2])) {
+        throw new Error(`Right operand of ${op} clause is not a primitive`);
+      } else if (
+        ["LIKE", "NOT LIKE"].includes(op) &&
+        isArray(filter[1]) &&
+        filter[1][0] === "FUNC" &&
+        ["UPPER", "LOWER"].includes(filter[1][1])
+      ) {
+        if (
+          (filter[1][1] === "UPPER" && filter[2] !== filter[2].toUpperCase()) ||
+          (filter[1][1] === "LOWER" && filter[2] !== filter[2].toLowerCase())
+        ) {
+          throw new Error(
+            `Cannot compare ${
+              filter[1][1]
+            }() against non-${filter[1][1].toLowerCase()}case pattern`
+          );
+        }
+      } else if (!isArray(filter[1]) || filter[1][0] !== "PARAM") {
+        throw new Error(`Left operand of ${op} clause is not a parameter`);
       }
 
-    if ((!negate && op === "AND") || (negate && op === "OR"))
-      return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
-
-    if ((!negate && op === "OR") || (negate && op === "AND"))
-      return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
-
-    if (op === "NOT") return recursive(filter[1], !negate);
-
-    if (isArray(filter[2]))
-      throw new Error(`Invalid RHS operand of ${op} clause`);
-
-    if (op === "LIKE" || op === "NOT LIKE") {
+      if (op === "=") {
+        const param = filter[1][1];
+        const p = (res[param] = res[param] || {});
+        if (negate) p["$ne"] = filter[2];
+        else p["$eq"] = filter[2];
+      } else if (op === "<>") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$ne"] = filter[2];
+        if (param !== "_tags") p["$exists"] = true;
+      } else if (op === ">") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$gt"] = filter[2];
+      } else if (op === ">=") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$gte"] = filter[2];
+      } else if (op === "<") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$lt"] = filter[2];
+      } else if (op === "<=") {
+        const param = filter[1][1];
+        let p = (res[param] = res[param] || {});
+        if (negate) p = p["$not"] = p["$not"] || {};
+        p["$lte"] = filter[2];
+      } else if (op === "IS NULL") {
+        const param = filter[1][1];
+        res[param] = { $exists: negate };
+      } else if (op === "IS NOT NULL") {
+        const param = filter[1][1];
+        res[param] = { $exists: !negate };
+      } else {
         if (op === "NOT LIKE") negate = !negate;
         let param;
         let flags;
-      if (!Array.isArray(filter[1]))
-        throw new Error(`Invalid LHS operand of ${op} clause`);
         if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
-        if (filter[2] !== filter[2].toUpperCase())
-          throw new Error(`Invalid RHS operand of ${op} clause`);
           param = filter[1][2][1];
           flags = "i";
         } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
-        if (filter[2] !== filter[2].toLowerCase())
-          throw new Error(`Invalid RHS operand of ${op} clause`);
           param = filter[1][2][1];
           flags = "i";
-      } else if (filter[1][0] === "PARAM") {
+        } else {
           param = filter[1][1];
           flags = "";
-      } else {
-        throw new Error(`Invalid LHS operand of ${op} clause`);
         }
         const r = likePatternToRegExp(filter[2], filter[3], flags);
-      if (negate) return { [param]: { $nin: [r, null] } };
-      else return { [param]: r };
+        if (negate) res[param] = { $not: r };
+        else res[param] = r;
       }
-
-    if (op === "_tags") {
-      let t = filter[2];
-      if (negate) t = !t;
-      if (t) return { _tags: filter[1] };
-      else return { _tags: { $ne: filter[1] } };
-    }
-
-    if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
-      throw new Error(`Invalid LHS operand of ${op} clause`);
-
-    if (op === "IS NULL") {
-      const p = filter[1][1];
-      return { [p]: null };
-    }
-
-    if (op === "IS NOT NULL") {
-      const p = filter[1][1];
-      return { [p]: { $ne: null } };
-    }
-
-    if (isArray(filter[2]) || filter[2] == null)
-      throw new Error(`Invalid RHS operand of ${op} clause`);
-
-    if (op === "=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (!negate) return { [p]: v };
-      else return { [p]: { $nin: [v, null] } };
-    }
-
-    if (op === "<>") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: v };
-      else return { [p]: { $nin: [v, null] } };
-    }
-
-    if (op === ">") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $lte: v } };
-      else return { [p]: { $gt: v } };
-    }
-
-    if (op === ">=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $lt: v } };
-      else return { [p]: { $gte: v } };
-    }
-
-    if (op === "<") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $gte: v } };
-      else return { [p]: { $lt: v } };
-    }
-
-    if (op === "<=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $gt: v } };
-      else return { [p]: { $lte: v } };
-    }
-
+    } else {
       throw new Error(`Unrecognized operator ${op}`);
     }
 
-  if (!isArray(exp)) {
-    if (exp === true) return {};
+    return res;
+  }
+
+  const _exp = evaluate(exp, null, Date.now());
+
+  if (!isArray(_exp)) {
+    if (_exp === true) return {};
     throw new Error("Primitives are not valid queries");
   }
 
-  return recursive(exp, false);
+  return recursive(_exp, false);
 }
 
-export function processDeviceProjection(
-  projection: Record<string, 1>
-): Record<string, 1> {
+export function processDeviceProjection(projection: {}): {} {
   if (!projection) return projection;
   const p = {};
   for (const [k, v] of Object.entries(projection)) {
@@ -271,9 +264,7 @@ export function processDeviceProjection(
   return p;
 }
 
-export function processDeviceSort(
-  sort: Record<string, number>
-): Record<string, number> {
+export function processDeviceSort(sort: {}): {} {
   if (!sort) return sort;
   const s = {};
   for (const [k, v] of Object.entries(sort)) {
@@ -289,7 +280,7 @@ export function processDeviceSort(
   return s;
 }
 
-function parseDate(d: Date): number | string {
+function parseDate(d): number | string {
   const n = +d;
   return isNaN(n) ? "" + d : n;
 }
@@ -311,7 +302,7 @@ interface FlatDevice {
   [param: string]: FlatAttributes;
 }
 
-export function flattenDevice(device: Record<string, unknown>): FlatDevice {
+export function flattenDevice(device): FlatDevice {
   function recursive(
     input,
     root: string,
@@ -322,39 +313,39 @@ export function flattenDevice(device: Record<string, unknown>): FlatDevice {
       if (!root) {
         if (name === "_lastInform") {
           output["Events.Inform"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
+            value: [parseDate(tree), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp,
+            objectTimestamp: timestamp
           };
         } else if (name === "_registered") {
           output["Events.Registered"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
+            value: [parseDate(tree), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp,
+            objectTimestamp: timestamp
           };
         } else if (name === "_lastBoot") {
           output["Events.1_BOOT"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
+            value: [parseDate(tree), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp,
+            objectTimestamp: timestamp
           };
         } else if (name === "_lastBootstrap") {
           output["Events.0_BOOTSTRAP"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
+            value: [parseDate(tree), "xsd:dateTime"],
             valueTimestamp: timestamp,
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp,
+            objectTimestamp: timestamp
           };
         } else if (name === "_id") {
           output["DeviceID.ID"] = {
@@ -363,7 +354,7 @@ export function flattenDevice(device: Record<string, unknown>): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp,
+            objectTimestamp: timestamp
           };
         } else if (name === "_deviceId") {
           output["DeviceID.Manufacturer"] = {
@@ -372,7 +363,7 @@ export function flattenDevice(device: Record<string, unknown>): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp,
+            objectTimestamp: timestamp
           };
           output["DeviceID.OUI"] = {
             value: [tree["_OUI"], "xsd:string"],
@@ -380,7 +371,7 @@ export function flattenDevice(device: Record<string, unknown>): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp,
+            objectTimestamp: timestamp
           };
           output["DeviceID.ProductClass"] = {
             value: [tree["_ProductClass"], "xsd:string"],
@@ -388,7 +379,7 @@ export function flattenDevice(device: Record<string, unknown>): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp,
+            objectTimestamp: timestamp
           };
           output["DeviceID.SerialNumber"] = {
             value: [tree["_SerialNumber"], "xsd:string"],
@@ -396,24 +387,24 @@ export function flattenDevice(device: Record<string, unknown>): FlatDevice {
             writable: false,
             writableTimestamp: timestamp,
             object: false,
-            objectTimestamp: timestamp,
+            objectTimestamp: timestamp
           };
         } else if (name === "_tags") {
           output["Tags"] = {
             writable: false,
             writableTimestamp: timestamp,
             object: true,
-            objectTimestamp: timestamp,
+            objectTimestamp: timestamp
           };
 
           for (const t of tree as string[]) {
-            output[`Tags.${encodeTag(t)}`] = {
+            output[`Tags.${t}`] = {
               value: [true, "xsd:boolean"],
               valueTimestamp: timestamp,
               writable: true,
               writableTimestamp: timestamp,
               object: false,
-              objectTimestamp: timestamp,
+              objectTimestamp: timestamp
             };
           }
         }
@@ -431,7 +422,7 @@ export function flattenDevice(device: Record<string, unknown>): FlatDevice {
       if (tree["_value"] != null) {
         attrs.value = [
           tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
-          tree["_type"],
+          tree["_type"]
         ];
         attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
         attrs.object = false;
@@ -465,29 +456,27 @@ export function flattenDevice(device: Record<string, unknown>): FlatDevice {
   }
 
   const newDevice: FlatDevice = {};
-  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
+  const timestamp = new Date(device["_lastInform"] || 1).getTime();
   recursive(device, "", newDevice, timestamp);
   return newDevice;
 }
 
-export function flattenFault(fault: unknown): Fault {
-  const f = Object.assign({}, fault) as Fault;
+export function flattenFault(fault): Fault {
+  const f = Object.assign({}, fault);
   if (f.timestamp) f.timestamp = +f.timestamp;
-  if (f["expiry"]) f["expiry"] = +f["expiry"];
-  return f as Fault;
+  if (f.expiry) f.expiry = +f.expiry;
+  return f;
 }
 
-export function flattenTask(task: unknown): Task {
-  const t = Object.assign({}, task) as Task;
+export function flattenTask(task): Task {
+  const t = Object.assign({}, task);
   t._id = "" + t._id;
-  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
+  if (t.timestamp) t.timestamp = +t.timestamp;
   if (t.expiry) t.expiry = +t.expiry;
   return t;
 }
 
-export function convertOldPrecondition(
-  query: Record<string, unknown>
-): Expression {
+export function mongoQueryToFilter(query): Expression {
   function recursive(_query): Expression {
     const expressions: Expression[] = [];
     for (const [k, v] of Object.entries(_query)) {
@@ -506,27 +495,14 @@ export function convertOldPrecondition(
       } else if (k === "_tags") {
         if (typeof v === "object") {
           if (isArray(v)) throw new Error(`Invalid type`);
-          const conjs: Expression[] = [];
-          for (const [op, val] of Object.entries(v)) {
-            if (op === "$ne") {
-              if (typeof v["$ne"] !== "string")
-                throw new Error("Only string values are allowed for _tags");
-              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
-            } else if (op === "$eq") {
-              if (typeof v["$eq"] !== "string")
-                throw new Error("Only string values are allowed for _tags");
-              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+
+          if (v.hasOwnProperty("$ne"))
+            expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]);
+          else if (v.hasOwnProperty("$eq"))
+            expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v["$eq"]}`]]);
+          else throw new Error(`Invalid tag query`);
         } else {
-              throw new Error(`Invalid tag query`);
-            }
-          }
-          if (conjs.length === 1) expressions.push(conjs[0]);
-          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
-        } else {
-          expressions.push([
-            "IS NOT NULL",
-            ["PARAM", `Tags.${encodeTag(v as string)}`],
-          ]);
+          expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
         }
       } else if (k.startsWith("Tags.")) {
         let exists: boolean;
@@ -542,31 +518,34 @@ export function convertOldPrecondition(
 
         const exps: Expression[] = [];
         for (const [kk, vv] of Object.entries(v)) {
-          if (kk === "$eq") {
-            exps.push(["=", ["PARAM", k], vv]);
-          } else if (kk === "$ne") {
-            exps.push([
-              "OR",
-              ["<>", ["PARAM", k], vv],
-              ["IS NULL", ["PARAM", k]],
-            ]);
-          } else if (kk === "$lt") {
-            exps.push(["<", ["PARAM", k], vv]);
-          } else if (kk === "$lte") {
-            exps.push(["<=", ["PARAM", k], vv]);
-          } else if (kk === "$gt") {
-            exps.push([">", ["PARAM", k], vv]);
-          } else if (kk === "$gte") {
-            exps.push([">=", ["PARAM", k], vv]);
-          } else {
+          let op;
+          switch (kk) {
+            case "$eq":
+              op = "=";
+              break;
+            case "$ne":
+              op = "<>";
+              break;
+            case "$lt":
+              op = "<";
+              break;
+            case "$lte":
+              op = "<=";
+              break;
+            case "$gt":
+              op = ">";
+              break;
+            case "$gte":
+              op = ">=";
+              break;
+            default:
               throw new Error(`Operator ${kk} not supported`);
           }
-          if (!["string", "number", "boolean"].includes(typeof vv))
-            throw new Error(`Invalid value for ${kk} operator`);
+          exps.push([op, ["PARAM", k], vv]);
         }
         if (exps.length === 1) {
           expressions.push(exps[0]);
-        } else if (exps.length > 1) {
+        } else {
           const and: Expression = ["AND"];
           expressions.push(and.concat(exps));
         }
@@ -575,8 +554,8 @@ export function convertOldPrecondition(
       }
     }
     if (expressions.length === 1) return expressions[0];
-    if (expressions.length === 0) return true;
-    return ["AND", ...expressions];
+    const and: Expression = ["AND"];
+    return and.concat(expressions);
   }
 
   // empty filter
@@ -585,21 +564,15 @@ export function convertOldPrecondition(
   return recursive(query);
 }
 
-export function flattenPreset(
-  preset: Record<string, unknown>
-): Record<string, unknown> {
+export function flattenPreset(preset): {} {
   const p = Object.assign({}, preset);
   if (p.precondition) {
     try {
       // Try parse to check expression validity
-      parse(p.precondition as string);
+      parse(p.precondition);
     } catch (error) {
-      p.precondition = convertOldPrecondition(
-        JSON.parse(p.precondition as string)
-      );
-      p.precondition = (p.precondition as string).length
-        ? stringify(p.precondition as Expression)
-        : "";
+      p.precondition = mongoQueryToFilter(JSON.parse(p.precondition));
+      p.precondition = p.precondition.length ? stringify(p.precondition) : "";
     }
   }
 
@@ -611,14 +584,14 @@ export function flattenPreset(
 
   const provision = p.configurations[0];
   if (
-    (p.configurations as any[]).length === 1 &&
+    p.configurations.length === 1 &&
     provision.type === "provision" &&
     provision.name &&
     provision.name.length
   ) {
     p.provision = provision.name;
     p.provisionArgs = provision.args
-      ? provision.args.map((a) => stringify(a)).join(", ")
+      ? JSON.stringify(provision.args).slice(1, -1)
       : "";
   }
 
@@ -626,42 +599,30 @@ export function flattenPreset(
   return p;
 }
 
-export function flattenFile(
-  file: Record<string, unknown>
-): Record<string, unknown> {
+export function flattenFile(file): {} {
   const f = {};
   f["_id"] = file["_id"];
   if (file.metadata) {
-    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
-    f["metadata.oui"] = file["metadata"]["oui"] || "";
-    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
-    f["metadata.version"] = file["metadata"]["version"] || "";
+    f["metadata.fileType"] = file.metadata.fileType || "";
+    f["metadata.oui"] = file.metadata.oui || "";
+    f["metadata.productClass"] = file.metadata.productClass || "";
+    f["metadata.version"] = file.metadata.version || "";
   }
   return f;
 }
 
-export function flattenUpload(
-  file: Record<string, unknown>
-): Record<string, unknown> {
-  const f = {};
-  f["_id"] = file["_id"];
-  return f;
-}
-
-export function preProcessPreset(
-  data: Record<string, unknown>
-): Record<string, unknown> {
+export function preProcessPreset(data): {} {
   const preset = Object.assign({}, data);
 
   if (!preset.precondition) preset.precondition = "";
   // Try parse to check expression validity
-  parse(preset.precondition as string);
+  parse(preset.precondition);
 
-  preset.weight = parseInt(preset.weight as string) || 0;
+  preset.weight = parseInt(preset.weight) || 0;
 
   const events = {};
   if (preset.events) {
-    for (let e of (preset.events as string).split(",")) {
+    for (let e of preset.events.split(",")) {
       let v = true;
       e = e.trim();
       if (e.startsWith("-")) {
@@ -679,11 +640,11 @@ export function preProcessPreset(
   const configuration = {
     type: "provision",
     name: preset.provision,
-    args: null,
+    args: null
   };
 
   if (preset.provisionArgs)
-    configuration.args = parseList(preset.provisionArgs as string);
+    configuration.args = JSON.parse(`[${preset.provisionArgs}]`);
 
   delete preset.provision;
   delete preset.provisionArgs;

lib/nbi.ts

@@ -18,14 +18,13 @@
  */
 
 import * as url from "url";
-import { Collection, GridFSBucket, ObjectId } from "mongodb";
+import * as mongodb from "mongodb";
 import * as querystring from "querystring";
 import * as vm from "vm";
 import * as config from "./config";
-import { onConnect } from "./db";
+import * as db from "./db";
 import * as query from "./query";
 import * as apiFunctions from "./api-functions";
-import { IncomingMessage, ServerResponse } from "http";
 import * as cache from "./cache";
 import { version as VERSION } from "../package.json";
 import { ping } from "./ping";
@@ -43,24 +42,7 @@ const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
 const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
 
-const collections = {
-  tasks: null as Collection,
-  devices: null as Collection,
-  presets: null as Collection,
-  objects: null as Collection,
-  "fs.files": null as Collection,
-  provisions: null as Collection,
-  virtualParameters: null as Collection,
-  faults: null as Collection,
-};
-let filesBucket: GridFSBucket;
-
-onConnect(async (db) => {
-  for (const k of Object.keys(collections)) collections[k] = db.collection(k);
-  filesBucket = new GridFSBucket(db);
-});
-
-function throwError(err: Error, httpResponse?: ServerResponse): never {
+function throwError(err, httpResponse?): never {
   if (httpResponse) {
     httpResponse.writeHead(500, { Connection: "close" });
     httpResponse.end(`${err.name}: ${err.message}`);
@@ -68,10 +50,7 @@ function throwError(err: Error, httpResponse?: ServerResponse): never {
   throw err;
 }
 
-export function listener(
-  request: IncomingMessage,
-  response: ServerResponse
-): void {
+export function listener(request, response): void {
   const chunks = [];
   let bytes = 0;
   response.setHeader("GenieACS-Version", VERSION);
@@ -102,11 +81,11 @@ export function listener(
       if (request.method === "PUT") {
         const preset = JSON.parse(body.toString());
         preset._id = presetName;
-        collections.presets.replaceOne(
+        db.presetsCollection.replaceOne(
           { _id: presetName },
           preset,
           { upsert: true },
-          (err) => {
+          err => {
             if (err) return void throwError(err, response);
 
             cache
@@ -115,7 +94,7 @@ export function listener(
                 response.writeHead(200);
                 response.end();
               })
-              .catch((err) => {
+              .catch(err => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -123,7 +102,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        collections.presets.deleteOne({ _id: presetName }, (err) => {
+        db.presetsCollection.deleteOne({ _id: presetName }, err => {
           if (err) return void throwError(err, response);
 
           cache
@@ -132,7 +111,7 @@ export function listener(
               response.writeHead(200);
               response.end();
             })
-            .catch((err) => {
+            .catch(err => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -149,11 +128,11 @@ export function listener(
       if (request.method === "PUT") {
         const object = JSON.parse(body.toString());
         object._id = objectName;
-        collections.objects.replaceOne(
+        db.objectsCollection.replaceOne(
           { _id: objectName },
           object,
           { upsert: true },
-          (err) => {
+          err => {
             if (err) return void throwError(err, response);
 
             cache
@@ -162,7 +141,7 @@ export function listener(
                 response.writeHead(200);
                 response.end();
               })
-              .catch((err) => {
+              .catch(err => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -170,7 +149,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        collections.objects.deleteOne({ _id: objectName }, (err) => {
+        db.objectsCollection.deleteOne({ _id: objectName }, err => {
           if (err) return void throwError(err, response);
 
           cache
@@ -179,7 +158,7 @@ export function listener(
               response.writeHead(200);
               response.end();
             })
-            .catch((err) => {
+            .catch(err => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -196,7 +175,7 @@ export function listener(
       if (request.method === "PUT") {
         const object = {
           _id: provisionName,
-          script: body.toString(),
+          script: body.toString()
         };
 
         try {
@@ -207,11 +186,11 @@ export function listener(
           return;
         }
 
-        collections.provisions.replaceOne(
+        db.provisionsCollection.replaceOne(
           { _id: provisionName },
           object,
           { upsert: true },
-          (err) => {
+          err => {
             if (err) return void throwError(err, response);
 
             cache
@@ -220,7 +199,7 @@ export function listener(
                 response.writeHead(200);
                 response.end();
               })
-              .catch((err) => {
+              .catch(err => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -228,7 +207,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        collections.provisions.deleteOne({ _id: provisionName }, (err) => {
+        db.provisionsCollection.deleteOne({ _id: provisionName }, err => {
           if (err) return void throwError(err, response);
 
           cache
@@ -237,7 +216,7 @@ export function listener(
               response.writeHead(200);
               response.end();
             })
-            .catch((err) => {
+            .catch(err => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -254,7 +233,7 @@ export function listener(
       if (request.method === "PUT") {
         const object = {
           _id: virtualParameterName,
-          script: body.toString(),
+          script: body.toString()
         };
 
         try {
@@ -265,11 +244,11 @@ export function listener(
           return;
         }
 
-        collections.virtualParameters.replaceOne(
+        db.virtualParametersCollection.replaceOne(
           { _id: virtualParameterName },
           object,
           { upsert: true },
-          (err) => {
+          err => {
             if (err) return void throwError(err, response);
 
             cache
@@ -278,7 +257,7 @@ export function listener(
                 response.writeHead(200);
                 response.end();
               })
-              .catch((err) => {
+              .catch(err => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -286,9 +265,9 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        collections.virtualParameters.deleteOne(
+        db.virtualParametersCollection.deleteOne(
           { _id: virtualParameterName },
-          (err) => {
+          err => {
             if (err) return void throwError(err, response);
 
             cache
@@ -297,7 +276,7 @@ export function listener(
                 response.writeHead(200);
                 response.end();
               })
-              .catch((err) => {
+              .catch(err => {
                 setTimeout(() => {
                   throwError(err, response);
                 });
@@ -313,20 +292,20 @@ export function listener(
       const deviceId = querystring.unescape(r[1]);
       const tag = querystring.unescape(r[2]);
       if (request.method === "POST") {
-        collections.devices.updateOne(
+        db.devicesCollection.updateOne(
           { _id: deviceId },
           { $addToSet: { _tags: tag } },
-          (err) => {
+          err => {
             if (err) return void throwError(err, response);
             response.writeHead(200);
             response.end();
           }
         );
       } else if (request.method === "DELETE") {
-        collections.devices.updateOne(
+        db.devicesCollection.updateOne(
           { _id: deviceId },
           { $pull: { _tags: tag } },
-          (err) => {
+          err => {
             if (err) return void throwError(err, response);
 
             response.writeHead(200);
@@ -344,12 +323,12 @@ export function listener(
         );
         const deviceId = faultId.split(":", 1)[0];
         const channel = faultId.slice(deviceId.length + 1);
-        collections.faults.deleteOne({ _id: faultId }, (err) => {
+        db.faultsCollection.deleteOne({ _id: faultId }, err => {
           if (err) return void throwError(err, response);
 
           if (channel.startsWith("task_")) {
-            const objId = new ObjectId(channel.slice(5));
-            return void collections.tasks.deleteOne({ _id: objId }, (err) => {
+            const objId = new mongodb.ObjectID(channel.slice(5));
+            return void db.tasksCollection.deleteOne({ _id: objId }, err => {
               if (err) return void throwError(err, response);
 
               cache
@@ -358,7 +337,7 @@ export function listener(
                   response.writeHead(200);
                   response.end();
                 })
-                .catch((err) => {
+                .catch(err => {
                   setTimeout(() => {
                     throwError(err, response);
                   });
@@ -372,7 +351,7 @@ export function listener(
               response.writeHead(200);
               response.end();
             })
-            .catch((err) => {
+            .catch(err => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -392,114 +371,77 @@ export function listener(
           task.device = deviceId;
           apiFunctions
             .insertTasks(task)
-            .then(async () => {
-              const lastInform = Date.now();
-
-              const socketTimeout: number = request.socket["timeout"];
-
-              // Disable socket timeout while waiting for session
-              if (socketTimeout) request.socket.setTimeout(0);
-
-              const notInSession = await apiFunctions.awaitSessionEnd(
-                deviceId,
-                30000
-              );
-              await cache.del(`${deviceId}_tasks_faults_operations`);
-              if (urlParts.query.connection_request == null) {
-                if (socketTimeout) request.socket.setTimeout(socketTimeout);
-                response.writeHead(202, {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(task));
-                return;
-              }
-
-              if (!notInSession) {
-                if (socketTimeout) request.socket.setTimeout(socketTimeout);
-                response.writeHead(202, "Task queued but not processed", {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(task));
-                return;
-              }
-
-              const status = await apiFunctions.connectionRequest(deviceId);
-
-              if (status) {
-                if (socketTimeout) request.socket.setTimeout(socketTimeout);
-                response.writeHead(202, status, {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(task));
-                return;
-              }
-
-              const onlineThreshold =
+            .then(() => {
+              cache
+                .del(`${deviceId}_tasks_faults_operations`)
+                .then(() => {
+                  if (urlParts.query.connection_request != null) {
+                    apiFunctions
+                      .connectionRequest(deviceId)
+                      .then(() => {
+                        const taskTimeout =
                           (urlParts.query.timeout &&
                             parseInt(urlParts.query.timeout as string)) ||
-                (config.get("DEVICE_ONLINE_THRESHOLD", deviceId) as number);
+                          config.get("DEVICE_ONLINE_THRESHOLD", deviceId);
 
-              const sessionStarted = await apiFunctions.awaitSessionStart(
-                deviceId,
-                lastInform,
-                onlineThreshold
-              );
-              if (!sessionStarted) {
-                if (socketTimeout) request.socket.setTimeout(socketTimeout);
-                response.writeHead(202, "Task queued but not processed", {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(task));
-                return;
-              }
-
-              const sessionEnded = await apiFunctions.awaitSessionEnd(
-                deviceId,
-                120000
-              );
-              if (!sessionEnded) {
-                if (socketTimeout) request.socket.setTimeout(socketTimeout);
-                response.writeHead(202, "Task queued but not processed", {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(task));
-                return;
-              }
-
-              const prom1 = collections.tasks.findOne(
-                { _id: task._id },
-                { projection: { _id: 1 } }
-              );
-              const prom2 = collections.faults.findOne(
-                { _id: `${deviceId}:task_${task._id}` },
+                        apiFunctions
+                          .watchTask(deviceId, task._id, taskTimeout)
+                          .then(status => {
+                            if (status === "timeout") {
+                              response.writeHead(
+                                202,
+                                "Task queued but not processed",
                                 {
-                  projection: { _id: 1 },
+                                  "Content-Type": "application/json"
                                 }
                               );
+                              response.end(JSON.stringify(task));
+                            } else if (status === "fault") {
+                              db.tasksCollection.findOne(
+                                { _id: task._id },
+                                (err, task2) => {
+                                  if (err)
+                                    return void throwError(err, response);
 
-              const [t, f] = await Promise.all([prom1, prom2]);
-
-              // Restore socket timeout
-              if (socketTimeout) request.socket.setTimeout(socketTimeout);
-
-              if (f) {
                                   response.writeHead(202, "Task faulted", {
-                  "Content-Type": "application/json",
+                                    "Content-Type": "application/json"
                                   });
-                response.end(JSON.stringify(t || task));
-              } else if (t) {
-                response.writeHead(202, "Task queued but not processed", {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(t));
+                                  response.end(JSON.stringify(task2));
+                                }
+                              );
                             } else {
                               response.writeHead(200, {
-                  "Content-Type": "application/json",
+                                "Content-Type": "application/json"
                               });
                               response.end(JSON.stringify(task));
                             }
                           })
-            .catch((err) => {
+                          .catch(err => {
+                            setTimeout(() => {
+                              throwError(err, response);
+                            });
+                          });
+                      })
+                      .catch(err => {
+                        response.writeHead(202, err.message, {
+                          "Content-Type": "application/json"
+                        });
+                        response.end(JSON.stringify(task));
+                      });
+                  } else {
+                    response.writeHead(202, {
+                      "Content-Type": "application/json"
+                    });
+                    response.end(JSON.stringify(task));
+                  }
+                })
+                .catch(err => {
+                  setTimeout(() => {
+                    throwError(err, response);
+                  });
+                });
+            })
+            .catch(err => {
               setTimeout(() => {
                 throwError(err, response);
               });
@@ -512,7 +454,7 @@ export function listener(
               response.writeHead(200);
               response.end();
             })
-            .catch((err) => {
+            .catch(err => {
               response.writeHead(504);
               response.end(`${err.name}: ${err.message}`);
             });
@@ -530,8 +472,8 @@ export function listener(
       const action = r[2];
       if (!action || action === "/") {
         if (request.method === "DELETE") {
-          collections.tasks.findOne(
-            { _id: new ObjectId(taskId) },
+          db.tasksCollection.findOne(
+            { _id: new mongodb.ObjectID(taskId) },
             { projection: { device: 1 } },
             (err, task) => {
               if (err) return void throwError(err, response);
@@ -543,14 +485,14 @@ export function listener(
               }
 
               const deviceId = task.device;
-              collections.tasks.deleteOne(
-                { _id: new ObjectId(taskId) },
-                (err) => {
+              db.tasksCollection.deleteOne(
+                { _id: new mongodb.ObjectID(taskId) },
+                err => {
                   if (err) return void throwError(err, response);
 
-                  collections.faults.deleteOne(
+                  db.faultsCollection.deleteOne(
                     { _id: `${deviceId}:task_${taskId}` },
-                    (err) => {
+                    err => {
                       if (err) return void throwError(err, response);
 
                       cache
@@ -559,7 +501,7 @@ export function listener(
                           response.writeHead(200);
                           response.end();
                         })
-                        .catch((err) => {
+                        .catch(err => {
                           setTimeout(() => {
                             throwError(err, response);
                           });
@@ -576,16 +518,16 @@ export function listener(
         }
       } else if (action === "/retry") {
         if (request.method === "POST") {
-          collections.tasks.findOne(
-            { _id: new ObjectId(taskId) },
+          db.tasksCollection.findOne(
+            { _id: new mongodb.ObjectID(taskId) },
             { projection: { device: 1 } },
             (err, task) => {
               if (err) return void throwError(err, response);
 
               const deviceId = task.device;
-              collections.faults.deleteOne(
+              db.faultsCollection.deleteOne(
                 { _id: `${deviceId}:task_${taskId}` },
-                (err) => {
+                err => {
                   if (err) return void throwError(err, response);
 
                   cache
@@ -594,7 +536,7 @@ export function listener(
                       response.writeHead(200);
                       response.end();
                     })
-                    .catch((err) => {
+                    .catch(err => {
                       setTimeout(() => {
                         throwError(err, response);
                       });
@@ -620,18 +562,14 @@ export function listener(
           fileType: request.headers.filetype,
           oui: request.headers.oui,
           productClass: request.headers.productclass,
-          version: request.headers.version,
+          version: request.headers.version
         };
-        filesBucket.delete((filename as unknown) as ObjectId, () => {
-          const uploadStream = filesBucket.openUploadStreamWithId(
-            filename,
-            filename,
-            {
-              metadata: metadata,
-            }
-          );
+        const bucket = new mongodb.GridFSBucket(db.client.db());
+        const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
+          metadata: metadata
+        });
 
-          uploadStream.on("error", (err) => {
+        uploadStream.on("error", err => {
           throwError(err, response);
         });
 
@@ -639,17 +577,10 @@ export function listener(
           response.writeHead(201);
           response.end();
         });
-        });
       } else if (request.method === "DELETE") {
-        filesBucket.delete((filename as unknown) as ObjectId, (err) => {
-          if (err) {
-            if (err.message.startsWith("FileNotFound")) {
-              response.writeHead(404);
-              response.end("404 Not Found");
-              return;
-            }
-            return void throwError(err, response);
-          }
+        const bucket = new mongodb.GridFSBucket(db.client.db());
+        bucket.delete((filename as unknown) as mongodb.ObjectId, err => {
+          if (err) return void throwError(err, response);
 
           response.writeHead(200);
           response.end();
@@ -662,11 +593,7 @@ export function listener(
       const host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1]);
       ping(host, (err, res, stdout) => {
         if (err) {
-          if (!res) {
-            response.writeHead(500, { Connection: "close" });
-            response.end(`${err.name}: ${err.message}`);
-            return;
-          }
+          if (!res) throwError(err, response);
           response.writeHead(404, { "Cache-Control": "no-cache" });
           response.end(`${err.name}: ${err.message}`);
           return;
@@ -674,7 +601,7 @@ export function listener(
 
         response.writeHead(200, {
           "Content-Type": "text/plain",
-          "Cache-Control": "no-cache",
+          "Cache-Control": "no-cache"
         });
         response.end(stdout);
       });
@@ -694,7 +621,7 @@ export function listener(
           response.writeHead(200);
           response.end();
         })
-        .catch((err) => {
+        .catch(err => {
           setTimeout(() => {
             throwError(err, response);
           });
@@ -718,7 +645,7 @@ export function listener(
         return;
       }
 
-      const collection = collections[collectionName];
+      const collection = db[`${collectionName}Collection`];
       if (!collection) {
         response.writeHead(404);
         response.end("404 Not Found");
@@ -742,15 +669,15 @@ export function listener(
           break;
         case "tasks":
           q = query.sanitizeQueryTypes(q, {
-            _id: (v) => new ObjectId(v as string),
-            timestamp: (v) => new Date(v as number),
-            retries: Number,
+            _id: v => new mongodb.ObjectID(v),
+            timestamp: v => new Date(v),
+            retries: Number
           });
           break;
         case "faults":
           q = query.sanitizeQueryTypes(q, {
-            timestamp: (v) => new Date(v as number),
-            retries: Number,
+            timestamp: v => new Date(v),
+            retries: Number
           });
       }
 
@@ -786,7 +713,7 @@ export function listener(
 
         response.writeHead(200, {
           "Content-Type": "application/json",
-          total: total,
+          total: total
         });
 
         if (request.method === "HEAD") {
@@ -797,11 +724,11 @@ export function listener(
         response.write("[\n");
         i = 0;
         cur.forEach(
-          (item) => {
+          item => {
             if (i++) response.write(",\n");
             response.write(JSON.stringify(item));
           },
-          (err) => {
+          err => {
             if (err) return void throwError(err);
             response.end("\n]");
           }

lib/ping.ts

@@ -20,7 +20,7 @@
 import { platform } from "os";
 import { exec } from "child_process";
 
-export interface PingResult {
+interface Ping {
   packetsTransmitted: number;
   packetsReceived: number;
   packetLoss: number;
@@ -32,7 +32,7 @@ export interface PingResult {
 
 export function ping(
   host: string,
-  callback: (err: Error, res?: PingResult, stdout?: string) => void
+  callback: (err, res?, stdout?) => void
 ): void {
   let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
   switch (platform()) {
@@ -55,7 +55,7 @@ export function ping(
   }
 
   exec(cmd, (err, stdout) => {
-    let parsed: PingResult;
+    let parsed: Ping;
     if (stdout) {
       const m1 = stdout.match(parseRegExp1);
       if (m1) {
@@ -66,7 +66,7 @@ export function ping(
           min: +m1[4],
           avg: +m1[5],
           max: +m1[6],
-          mdev: +m1[7],
+          mdev: +m1[7]
         };
       } else {
         const m2 = stdout.match(parseRegExp2);
@@ -78,7 +78,7 @@ export function ping(
             min: null,
             avg: null,
             max: null,
-            mdev: null,
+            mdev: null
           };
         }
       }

lib/query.ts

@@ -17,9 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-function isObject(obj: any): boolean {
-  return Object.prototype.toString.call(obj) === "[object Object]";
-}
+import * as common from "./common";
 
 function stringToRegexp(input, flags?): RegExp | false {
   if (input.indexOf("*") === -1) return false;
@@ -36,8 +34,8 @@ function stringToRegexp(input, flags?): RegExp | false {
 }
 
 function normalize(input): any {
-  if (typeof input === "string") {
-    const vals: any = [input];
+  if (common.typeOf(input) === common.STRING_TYPE) {
+    const vals = [input];
     const m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input);
     if (m) vals.push({ $regex: new RegExp(m[1], m[2]) });
 
@@ -54,14 +52,14 @@ function normalize(input): any {
   return input;
 }
 
-function expandValue(value: unknown): unknown[] {
-  if (Array.isArray(value)) {
+export function expandValue(value): any[] {
+  if (common.typeOf(value) === common.ARRAY_TYPE) {
     let a = [];
     for (const j of value) a = a.concat(expandValue(j));
     return [a];
-  } else if (!isObject(value)) {
+  } else if (common.typeOf(value) !== common.OBJECT_TYPE) {
     const n = normalize(value);
-    if (!Array.isArray(n)) return [n];
+    if (common.typeOf(n) !== common.ARRAY_TYPE) return [n];
     else return n;
   }
 
@@ -105,14 +103,12 @@ function permute(param, val): any[] {
   return conditions;
 }
 
-export function expand(
-  query: Record<string, unknown>
-): Record<string, unknown> {
+export function expand(query): {} {
   const newQuery = {};
   for (const [k, v] of Object.entries(query)) {
     if (k[0] === "$") {
       // Operator
-      newQuery[k] = (v as any[]).map((e) => expand(e));
+      newQuery[k] = (v as any[]).map(e => expand(e));
     } else {
       const conditions = permute(k, v);
       if (conditions.length > 1) {
@@ -133,16 +129,100 @@ export function expand(
   return newQuery;
 }
 
-export function sanitizeQueryTypes(
-  query: Record<string, unknown>,
-  types: Record<string, (v: unknown) => unknown>
-): Record<string, unknown> {
+function testExpressions(params, expressions, lop): boolean {
+  for (const f of expressions) {
+    const res = test(params, f);
+    switch (lop) {
+      case "$and":
+        if (!res) return false;
+        break;
+      case "$or":
+        if (res) return true;
+        break;
+      case "$nor":
+        if (res) return false;
+    }
+  }
+
+  switch (lop) {
+    case "$and":
+      return true;
+    case "$or":
+      return false;
+    case "$nor":
+      return true;
+    default:
+      throw new Error("Unknown logical operator");
+  }
+}
+
+export function test(params, query): boolean {
+  let res;
+  for (const [k, v] of Object.entries(query)) {
+    if (k.charAt(0) === "$") {
+      // Logical operator
+      res = testExpressions(params, v, k);
+    } else {
+      const value = params[k];
+      if (common.typeOf(v) !== common.OBJECT_TYPE) {
+        if (common.typeOf(value) === common.ARRAY_TYPE) {
+          // TODO comparing array to regex, array to array, and object to object
+          res = value.indexOf(v) !== -1;
+        } else {
+          if (common.typeOf(v) === common.REGEXP_TYPE)
+            res = (v as RegExp).test(value);
+          else res = v === value;
+        }
+      } else {
+        for (const [k2, v2] of Object.entries(v)) {
+          switch (k2) {
+            case "$ne":
+              if (common.typeOf(value) === common.ARRAY_TYPE)
+                res = value.indexOf(v2) === -1;
+              else res = value !== v2;
+              break;
+            case "$lt":
+              res = value < v2;
+              break;
+            case "$lte":
+              res = value <= v2;
+              break;
+            case "$gt":
+              res = value > v2;
+              break;
+            case "$gte":
+              res = value >= v2;
+              break;
+            case "$regex":
+              res = v2.test(value);
+              break;
+            case "$in":
+              throw new Error("Operator not supported");
+            case "$nin":
+              throw new Error("Operator not supported");
+            case "$all":
+              throw new Error("Operator not supported");
+            case "$exists":
+              throw new Error("Operator not supported");
+            default:
+              throw new Error("Operator not supported");
+          }
+        }
+      }
+    }
+
+    if (!res) return false;
+  }
+  return true;
+}
+
+export function sanitizeQueryTypes(query, types): {} {
   for (const [k, v] of Object.entries(query)) {
     if (k[0] === "$") {
       // Logical operator
       for (const vv of v as any[]) sanitizeQueryTypes(vv, types);
     } else if (k in types) {
-      if (isObject(v)) {
+      if (common.typeOf(v) === common.OBJECT_TYPE) {
         for (const [kk, vv] of Object.entries(v)) {
           switch (kk) {
             case "$in":

lib/sandbox.ts

@@ -109,16 +109,6 @@ class SandboxDate {
 
     return t;
   }
-
-  public static parse(dateString: string): number {
-    return Date.parse(dateString);
-  }
-
-  public static UTC(
-    ...args: [number, number?, number?, number?, number?, number?, number?]
-  ): number {
-    return Date.UTC(...args);
-  }
 }
 
 function random(): number {
@@ -157,7 +147,7 @@ class ParameterWrapper {
           if (!attr) return UNDEFINED;
 
           return attr[1];
-        },
+        }
       });
     }
 
@@ -177,7 +167,7 @@ class ParameterWrapper {
         if (!unpacked.length) return UNDEFINED;
 
         return unpacked[0].toString();
-      },
+      }
     });
 
     Object.defineProperty(this, "size", {
@@ -196,7 +186,7 @@ class ParameterWrapper {
         if (!unpacked.length) return UNDEFINED;
 
         return unpacked.length;
-      },
+      }
     });
 
     this[Symbol.iterator] = function*() {
@@ -235,7 +225,7 @@ function declare(
     pathSet: null,
     attrGet: null,
     attrSet: null,
-    defer: true,
+    defer: true
   };
 
   const attrs = new Set();
@@ -290,9 +280,9 @@ function commit(): void {
   }
 }
 
-function ext(...args: unknown[]): any {
+function ext(): any {
   ++state.extCounter;
-  const extCall = args.map(String);
+  const extCall = Array.from(arguments).map(String);
   const key = `${state.revision}: ${JSON.stringify(extCall)}`;
 
   if (key in state.sessionContext.extensionsCache)
@@ -302,11 +292,11 @@ function ext(...args: unknown[]): any {
   throw EXT;
 }
 
-function log(msg: string, meta: Record<string, unknown>): void {
+function log(msg: string, meta: {}): void {
   if (state.revision === state.maxRevision && state.extCounter >= 0) {
     const details = Object.assign({}, meta, {
       sessionContext: state.sessionContext,
-      message: `Script: ${msg}`,
+      message: `Script: ${msg}`
     });
 
     delete details["hostname"];
@@ -342,8 +332,8 @@ function errorToFault(err: Error): Fault {
     message: err.message,
     detail: {
       name: err.name,
-      message: err.message,
-    },
+      message: err.message
+    }
   };
 
   if (err.stack) {
@@ -365,7 +355,7 @@ function errorToFault(err: Error): Fault {
 
 export async function run(
   script: vm.Script,
-  globals: Record<string, unknown>,
+  globals: {},
   sessionContext: SessionContext,
   startRevision: number,
   maxRevision: number,
@@ -380,7 +370,7 @@ export async function run(
     extensions: {},
     clear: [],
     rng: null,
-    extCounter: extCounter,
+    extCounter: extCounter
   };
 
   for (const n of Object.keys(context)) delete context[n];
@@ -403,7 +393,7 @@ export async function run(
         clear: null,
         declare: null,
         done: false,
-        returnValue: null,
+        returnValue: null
       };
     }
   }
@@ -423,7 +413,7 @@ export async function run(
       clear: null,
       declare: null,
       done: false,
-      returnValue: null,
+      returnValue: null
     };
   }
 
@@ -443,6 +433,6 @@ export async function run(
     clear: _state.clear,
     declare: _state.declarations,
     done: status === 0,
-    returnValue: ret,
+    returnValue: ret
   };
 }

lib/scheduling.ts

@@ -21,7 +21,10 @@ import * as crypto from "crypto";
 import * as later from "later";
 
 function md532(str): number {
-  const digest = crypto.createHash("md5").update(str).digest();
+  const digest = crypto
+    .createHash("md5")
+    .update(str)
+    .digest();
   return (
     digest.readUInt32LE(0) ^
     digest.readUInt32LE(4) ^
@@ -30,37 +33,29 @@ function md532(str): number {
   );
 }
 
-export function variance(deviceId: string, vrnc: number): number {
+export function variance(deviceId, vrnc): number {
   return (md532(deviceId) >>> 0) % vrnc;
 }
 
-export function interval(
-  timestamp: number,
-  intrvl: number,
-  offset = 0
-): number {
+export function interval(timestamp, intrvl, offset = 0): number {
   return Math.trunc((timestamp + offset) / intrvl) * intrvl - offset;
 }
 
-export function parseCron(cronExp: string): any {
+export function parseCron(cronExp): any {
   const parts = cronExp.trim().split(/\s+/);
   if (parts.length === 5) parts.unshift("*");
 
   return later.schedule(later.parse.cron(parts.join(" "), true));
 }
 
-export function cron(
-  timestamp: number,
-  schedule: unknown,
-  offset = 0
-): number[] {
+export function cron(timestamp, schedule, offset = 0): number[] {
   // TODO later.js doesn't throw erorr if expression is invalid!
   const ret = [0, 0];
 
-  const prev = (schedule as any).prev(1, new Date(timestamp + offset));
+  const prev = schedule.prev(1, new Date(timestamp + offset));
   if (prev) ret[0] = prev.setMilliseconds(0) - offset;
 
-  const next = (schedule as any).next(1, new Date(timestamp + offset + 1000));
+  const next = schedule.next(1, new Date(timestamp + offset + 1000));
   if (next) ret[1] = next.setMilliseconds(0) - offset;
 
   return ret;

lib/server.ts

@@ -20,7 +20,6 @@
 import * as fs from "fs";
 import * as http from "http";
 import * as https from "https";
-import { Socket } from "net";
 import * as path from "path";
 import { ROOT_DIR } from "./config";
 
@@ -52,43 +51,35 @@ function closeServer(timeout, callback): void {
   });
 }
 
-interface ServerOptions {
-  port?: number;
-  host?: string;
-  ssl?: { key: string; cert: string };
-  timeout?: number;
-  keepAliveTimeout?: number;
-  onConnection?: (socket: Socket) => void;
-}
-
 export function start(
-  options: ServerOptions,
-  _listener: http.RequestListener
+  port,
+  networkInterface,
+  ssl,
+  _listener,
+  onConnection?,
+  keepAliveTimeout: number = -1
 ): void {
   listener = _listener;
 
-  if (options.ssl) {
-    const opts = {
-      key: options.ssl.key
+  if (ssl && ssl.key && ssl.cert) {
+    const options = {
+      key: ssl.key
         .split(":")
-        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
-      cert: options.ssl.cert
+        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
+      cert: ssl.cert
         .split(":")
-        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
+        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim())))
     };
 
-    server = https.createServer(opts, listener);
-    if (options.onConnection)
-      server.on("secureConnection", options.onConnection);
+    server = https.createServer(options, listener);
+    if (onConnection != null) server.on("secureConnection", onConnection);
   } else {
     server = http.createServer(listener);
-    if (options.onConnection) server.on("connection", options.onConnection);
+    if (onConnection != null) server.on("connection", onConnection);
   }
 
-  server.timeout = options.timeout || 0;
-  if (options.keepAliveTimeout != null)
-    server.keepAliveTimeout = options.keepAliveTimeout;
-  server.listen({ port: options.port, host: options.host });
+  if (keepAliveTimeout >= 0) server.keepAliveTimeout = keepAliveTimeout;
+  server.listen(port, networkInterface);
 }
 
 export function stop(): Promise<void> {

lib/soap-libxmljs.ts

@@ -0,0 +1,745 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { parseXml, Document, Element, ParserOptions } from "libxmljs";
+import * as config from "./config";
+import { version as VERSION } from "../package.json";
+import {
+  CpeGetResponse,
+  CpeSetResponse,
+  InformRequest,
+  AcsResponse,
+  CpeRequest,
+  FaultStruct,
+  SpvFault,
+  CpeFault,
+  SoapMessage,
+  TransferCompleteRequest
+} from "./types";
+
+const SERVER_NAME = `GenieACS/${VERSION}`;
+
+const NAMESPACES = {
+  "1.0": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-0"
+  },
+  "1.1": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-1"
+  },
+  "1.2": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.3": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.4": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    xsd: "http://www.w3.org/2001/XMLSchema",
+    xsi: "http://www.w3.org/2001/XMLSchema-instance",
+    cwmp: "urn:dslforum-org:cwmp-1-3"
+  }
+};
+
+const LIBXMLJS_PARSE_OPTIONS: ParserOptions = {
+  nocdata: true,
+  recover: !!config.get("XML_RECOVER"),
+  /* eslint-disable-next-line @typescript-eslint/camelcase */
+  ignore_enc: !!config.get("XML_IGNORE_ENC")
+};
+
+interface WriterOptions {
+  declaration?: boolean;
+  format?: boolean;
+  selfCloseEmpty?: boolean;
+  whitespace?: boolean;
+  type?: "XML" | "HTML" | "XHTML";
+}
+
+declare module "libxmljs" {
+  interface Document {
+    toString(formatting: boolean | WriterOptions): string;
+  }
+}
+
+const LIBXMLJS_SAVE_OPTIONS: WriterOptions = {
+  format: !!config.get("XML_FORMAT"),
+  declaration: !config.get("XML_NO_DECL"),
+  selfCloseEmpty: !config.get("XML_NO_EMPTY")
+};
+
+let warnings;
+
+function parseBool(v): boolean {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else return null;
+}
+
+function event(xml): string[] {
+  return xml
+    .childNodes()
+    .filter(n => n.name() === "EventStruct")
+    .map(c =>
+      c
+        .childNodes()
+        .find(n => n.name() === "EventCode")
+        .text()
+        .trim()
+    );
+}
+
+function parameterInfoList(xml): [string, boolean][] {
+  return xml
+    .childNodes()
+    .filter(e => e.name() === "ParameterInfoStruct")
+    .map(e => {
+      let param, value;
+      for (const c of e.childNodes()) {
+        switch (c.name()) {
+          case "Name":
+            param = c.text();
+            break;
+          case "Writable":
+            value = c.text();
+            break;
+        }
+      }
+
+      let parsed = parseBool(value);
+
+      if (parsed === null) {
+        warnings.push({
+          message: "Invalid writable attribute",
+          parameter: param
+        });
+        parsed = false;
+      }
+
+      return [param, parsed];
+    });
+}
+
+function parameterValueList(
+  xml
+): [string, string | number | boolean, string][] {
+  return xml
+    .childNodes()
+    .filter(e => e.name() === "ParameterValueStruct")
+    .map(e => {
+      let valueElement, param;
+      for (const c of e.childNodes()) {
+        switch (c.name()) {
+          case "Name":
+            param = c.text();
+            break;
+          case "Value":
+            valueElement = c;
+            break;
+        }
+      }
+
+      const valueType = valueElement.attr("type").value();
+      const value = valueElement.text();
+      let parsed = value;
+      if (valueType === "xsd:boolean") {
+        parsed = parseBool(value);
+        if (parsed === null) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      } else if (valueType === "xsd:int" || valueType === "xsd:unsignedInt") {
+        parsed = parseInt(value);
+        if (isNaN(parsed)) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      } else if (valueType === "xsd:dateTime") {
+        parsed = Date.parse(value);
+        if (isNaN(parsed)) {
+          warnings.push({
+            message: "Invalid value attribute",
+            parameter: param
+          });
+          parsed = value;
+        }
+      }
+
+      return [param, parsed, valueType];
+    });
+}
+
+function GetParameterNames(xml, methodRequest): void {
+  const el = xml.node("cwmp:GetParameterNames");
+  el.node("ParameterPath").text(methodRequest.parameterPath);
+  el.node("NextLevel").text(+methodRequest.nextLevel);
+}
+
+function GetParameterNamesResponse(xml): CpeGetResponse {
+  return {
+    name: "GetParameterNamesResponse",
+    parameterList: parameterInfoList(
+      xml.childNodes().find(n => n.name() === "ParameterList")
+    )
+  };
+}
+
+function GetParameterValues(xml, methodRequest): void {
+  const el = xml.node("cwmp:GetParameterValues").node("ParameterNames");
+  el.attr({
+    "soap-enc:arrayType": `xsd:string[${methodRequest.parameterNames.length}]`
+  });
+
+  for (const p of methodRequest.parameterNames) el.node("string").text(p);
+}
+
+function GetParameterValuesResponse(xml): CpeGetResponse {
+  return {
+    name: "GetParameterValuesResponse",
+    parameterList: parameterValueList(
+      xml.childNodes().find(n => n.name() === "ParameterList")
+    )
+  };
+}
+
+function SetParameterValues(xml, methodRequest): void {
+  const el = xml.node("cwmp:SetParameterValues");
+  const paramList = el.node("ParameterList");
+  paramList.attr({
+    "soap-enc:arrayType": `cwmp:ParameterValueStruct[${methodRequest.parameterList.length}]`
+  });
+
+  for (const p of methodRequest.parameterList) {
+    const pvs = paramList.node("ParameterValueStruct");
+    pvs.node("Name").text(p[0]);
+    let val = p[1];
+    if (p[2] === "xsd:dateTime" && typeof val === "number") {
+      val = new Date(val).toISOString();
+      if (methodRequest.DATETIME_MILLISECONDS === false)
+        val = val.replace(".000", "");
+    }
+    if (p[2] === "xsd:boolean" && typeof val === "boolean")
+      if (methodRequest.BOOLEAN_LITERAL === false) val = +val;
+
+    pvs
+      .node("Value")
+      .attr({ "xsi:type": p[2] })
+      .text(val);
+  }
+
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+function SetParameterValuesResponse(xml): CpeSetResponse {
+  return {
+    name: "SetParameterValuesResponse",
+    status: parseInt(
+      xml
+        .childNodes()
+        .find(n => n.name() === "Status")
+        .text()
+    )
+  };
+}
+
+function AddObject(xml, methodRequest): void {
+  const el = xml.node("cwmp:AddObject");
+  el.node("ObjectName").text(methodRequest.objectName);
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+function AddObjectResponse(xml): CpeSetResponse {
+  let instanceNumber, status;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "InstanceNumber":
+        instanceNumber = parseInt(c.text());
+        break;
+      case "Status":
+        status = parseInt(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "AddObjectResponse",
+    instanceNumber: instanceNumber,
+    status: status
+  };
+}
+
+function DeleteObject(xml, methodRequest): void {
+  const el = xml.node("cwmp:DeleteObject");
+  el.node("ObjectName").text(methodRequest.objectName);
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+function DeleteObjectResponse(xml): CpeSetResponse {
+  return {
+    name: "DeleteObjectResponse",
+    status: parseInt(
+      xml
+        .childNodes()
+        .find(n => n.name() === "Status")
+        .text()
+    )
+  };
+}
+
+function Reboot(xml, methodRequest): void {
+  const el = xml.node("cwmp:Reboot");
+  el.node("CommandKey").text(methodRequest.commandKey || "");
+}
+
+function RebootResponse(): CpeSetResponse {
+  return {
+    name: "RebootResponse"
+  };
+}
+
+function FactoryReset(xml): void {
+  xml.node("cwmp:FactoryReset");
+}
+
+function FactoryResetResponse(): CpeSetResponse {
+  return {
+    name: "FactoryResetResponse"
+  };
+}
+
+function Download(xml, methodRequest): void {
+  const el = xml.node("cwmp:Download");
+  el.node("CommandKey").text(methodRequest.commandKey || "");
+  el.node("FileType").text(methodRequest.fileType);
+  el.node("URL").text(methodRequest.url);
+  el.node("Username").text(methodRequest.username || "");
+  el.node("Password").text(methodRequest.password || "");
+  el.node("FileSize").text(methodRequest.fileSize || "0");
+  el.node("TargetFileName").text(methodRequest.targetFileName || "");
+  el.node("DelaySeconds").text(methodRequest.delaySeconds || "0");
+  el.node("SuccessURL").text(methodRequest.successUrl || "");
+  el.node("FailureURL").text(methodRequest.failureUrl || "");
+}
+
+function DownloadResponse(xml): CpeSetResponse {
+  let status, startTime, completeTime;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "Status":
+        status = parseInt(c.text());
+        break;
+      case "StartTime":
+        startTime = Date.parse(c.text());
+        break;
+      case "CompleteTime":
+        completeTime = Date.parse(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "DownloadResponse",
+    status: status,
+    startTime: startTime,
+    completeTime: completeTime
+  };
+}
+
+function Inform(xml): InformRequest {
+  let retryCount, evnt, parameterList;
+  const deviceId = {
+    Manufacturer: null,
+    OUI: null,
+    ProductClass: null,
+    SerialNumber: null
+  };
+
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "ParameterList":
+        parameterList = parameterValueList(c);
+        break;
+      case "DeviceId":
+        for (const cc of c.childNodes()) {
+          const n = cc.name();
+          if (n in deviceId) deviceId[n] = cc.text();
+        }
+        break;
+      case "Event":
+        evnt = event(c);
+        break;
+      case "RetryCount":
+        retryCount = parseInt(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "Inform",
+    parameterList: parameterList,
+    deviceId: deviceId,
+    event: evnt,
+    retryCount: retryCount
+  };
+}
+
+function InformResponse(xml): void {
+  xml
+    .node("cwmp:InformResponse")
+    .node("MaxEnvelopes")
+    .text(1);
+}
+
+function GetRPCMethods(): AcsResponse {
+  return { name: "GetRPCMethods" };
+}
+
+function GetRPCMethodsResponse(xml, methodResponse): void {
+  const el = xml.node("cwmp:GetRPCMethodsResponse").node("MethodList");
+  el.attr({
+    "soap-enc:arrayType": `xsd:string[${methodResponse.methodList.length}]`
+  });
+
+  for (const m of methodResponse.methodList) el.node("string").text(m);
+}
+
+function TransferComplete(xml): TransferCompleteRequest {
+  let commandKey, _faultStruct, startTime, completeTime;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "CommandKey":
+        commandKey = c.text();
+        break;
+      case "FaultStruct":
+        _faultStruct = faultStruct(c);
+        break;
+      case "StartTime":
+        startTime = Date.parse(c.text());
+        break;
+      case "CompleteTime":
+        completeTime = Date.parse(c.text());
+        break;
+    }
+  }
+
+  return {
+    name: "TransferComplete",
+    commandKey: commandKey,
+    faultStruct: _faultStruct,
+    startTime: startTime,
+    completeTime: completeTime
+  };
+}
+
+function TransferCompleteResponse(xml): void {
+  xml.node("cwmp:TransferCompleteResponse").text("");
+}
+
+function RequestDownload(xml): CpeRequest {
+  return {
+    name: "RequestDownload",
+    fileType: xml
+      .childNodes()
+      .find(n => n.name() === "FileType")
+      .text()
+  };
+}
+
+function RequestDownloadResponse(xml): void {
+  xml.node("cwmp:RequestDownloadResponse").text("");
+}
+
+function faultStruct(xml): FaultStruct {
+  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "FaultCode":
+        faultCode = c.text();
+        break;
+      case "FaultString":
+        faultString = c.text();
+        break;
+      case "SetParameterValuesFault":
+        setParameterValuesFault = setParameterValuesFault || [];
+        for (const cc of c.childNodes()) {
+          switch (cc.name()) {
+            case "ParameterName":
+              pn = cc.text();
+              break;
+            case "FaultCode":
+              fc = cc.text();
+              break;
+            case "FaultString":
+              fs = cc.text();
+              break;
+          }
+        }
+        setParameterValuesFault.push({
+          parameterName: pn,
+          faultCode: fc,
+          faultString: fs
+        });
+    }
+  }
+
+  return { faultCode, faultString, setParameterValuesFault };
+}
+
+function fault(xml): CpeFault {
+  let faultCode, faultString, detail;
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "faultcode":
+        faultCode = c.text();
+        break;
+      case "faultstring":
+        faultString = c.text();
+        break;
+      case "detail":
+        detail = faultStruct(c.childNodes().find(n => n.name() === "Fault"));
+        break;
+    }
+  }
+
+  return { faultCode, faultString, detail };
+}
+
+export function request(data, cwmpVersion, warn): SoapMessage {
+  warnings = warn;
+
+  const rpc = {
+    id: null,
+    cwmpVersion: cwmpVersion,
+    sessionTimeout: null,
+    cpeRequest: null,
+    cpeFault: null,
+    cpeResponse: null
+  };
+
+  if (!data.length) return rpc;
+
+  const xml = parseXml(data, LIBXMLJS_PARSE_OPTIONS);
+
+  let headerElement, bodyElement;
+
+  for (const c of xml.childNodes()) {
+    switch (c.name()) {
+      case "Header":
+        headerElement = c;
+        break;
+      case "Body":
+        bodyElement = c;
+        break;
+    }
+  }
+
+  if (headerElement) {
+    for (const c of headerElement.childNodes()) {
+      switch (c.name()) {
+        case "ID":
+          rpc.id = c.text();
+          break;
+        case "sessionTimeout":
+          rpc.sessionTimeout = parseInt(c.text());
+          break;
+      }
+    }
+  }
+
+  for (const methodElement of bodyElement.childNodes()) {
+    const methodName = methodElement.name();
+    if (methodName === "text") continue;
+
+    if (!rpc.cwmpVersion && methodName !== "Fault") {
+      switch (methodElement.namespace().href()) {
+        case "urn:dslforum-org:cwmp-1-0":
+          rpc.cwmpVersion = "1.0";
+          break;
+        case "urn:dslforum-org:cwmp-1-1":
+          rpc.cwmpVersion = "1.1";
+          break;
+        case "urn:dslforum-org:cwmp-1-2":
+          if (rpc.sessionTimeout) rpc.cwmpVersion = "1.3";
+          else rpc.cwmpVersion = "1.2";
+
+          break;
+        case "urn:dslforum-org:cwmp-1-3":
+          rpc.cwmpVersion = "1.4";
+          break;
+        default:
+          throw new Error("Unrecognized CWMP version");
+      }
+    }
+
+    switch (methodName) {
+      case "Inform":
+        rpc.cpeRequest = Inform(methodElement);
+        break;
+      case "GetRPCMethods":
+        rpc.cpeRequest = GetRPCMethods();
+        break;
+      case "TransferComplete":
+        rpc.cpeRequest = TransferComplete(methodElement);
+        break;
+      case "RequestDownload":
+        rpc.cpeRequest = RequestDownload(methodElement);
+        break;
+      case "GetParameterNamesResponse":
+        rpc.cpeResponse = GetParameterNamesResponse(methodElement);
+        break;
+      case "GetParameterValuesResponse":
+        rpc.cpeResponse = GetParameterValuesResponse(methodElement);
+        break;
+      case "SetParameterValuesResponse":
+        rpc.cpeResponse = SetParameterValuesResponse(methodElement);
+        break;
+      case "AddObjectResponse":
+        rpc.cpeResponse = AddObjectResponse(methodElement);
+        break;
+      case "DeleteObjectResponse":
+        rpc.cpeResponse = DeleteObjectResponse(methodElement);
+        break;
+      case "RebootResponse":
+        rpc.cpeResponse = RebootResponse();
+        break;
+      case "FactoryResetResponse":
+        rpc.cpeResponse = FactoryResetResponse();
+        break;
+      case "DownloadResponse":
+        rpc.cpeResponse = DownloadResponse(methodElement);
+        break;
+      case "Fault":
+        rpc.cpeFault = fault(methodElement);
+        break;
+      default:
+        throw new Error(`8000 Method not supported ${methodName}`);
+    }
+  }
+
+  return rpc;
+}
+
+function createSoapEnv(cwmpVersion: string): Element {
+  const xml = new Document();
+  const env = xml.node("soap-env:Envelope");
+
+  for (const prefix of Object.keys(NAMESPACES[cwmpVersion]))
+    env.defineNamespace(prefix, NAMESPACES[cwmpVersion][prefix]);
+
+  return env;
+}
+
+export function response(rpc): { code: number; headers: {}; data: string } {
+  const headers = {
+    Server: SERVER_NAME,
+    SOAPServer: SERVER_NAME
+  };
+
+  if (!rpc) return { code: 204, headers: headers, data: "" };
+
+  const env = createSoapEnv(rpc.cwmpVersion);
+  const header = env.node("soap-env:Header");
+  header
+    .node("cwmp:ID")
+    .attr({
+      "soap-env:mustUnderstand": "1"
+    })
+    .text(rpc.id);
+
+  const body = env.node("soap-env:Body");
+
+  if (rpc.acsResponse) {
+    switch (rpc.acsResponse.name) {
+      case "InformResponse":
+        InformResponse(body);
+        break;
+      case "GetRPCMethodsResponse":
+        GetRPCMethodsResponse(body, rpc.acsResponse);
+        break;
+      case "TransferCompleteResponse":
+        TransferCompleteResponse(body);
+        break;
+      case "RequestDownloadResponse":
+        RequestDownloadResponse(body);
+        break;
+      default:
+        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
+    }
+  } else if (rpc.acsRequest) {
+    switch (rpc.acsRequest.name) {
+      case "GetParameterNames":
+        GetParameterNames(body, rpc.acsRequest);
+        break;
+      case "GetParameterValues":
+        GetParameterValues(body, rpc.acsRequest);
+        break;
+      case "SetParameterValues":
+        SetParameterValues(body, rpc.acsRequest);
+        break;
+      case "AddObject":
+        AddObject(body, rpc.acsRequest);
+        break;
+      case "DeleteObject":
+        DeleteObject(body, rpc.acsRequest);
+        break;
+      case "Reboot":
+        Reboot(body, rpc.acsRequest);
+        break;
+      case "FactoryReset":
+        FactoryReset(body);
+        break;
+      case "Download":
+        Download(body, rpc.acsRequest);
+        break;
+      default:
+        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
+    }
+  }
+
+  headers["Content-Type"] = 'text/xml; charset="utf-8"';
+  return {
+    code: 200,
+    headers: headers,
+    data: env.doc().toString(LIBXMLJS_SAVE_OPTIONS)
+  };
+}

lib/soap.ts

@@ -22,35 +22,22 @@ import {
   Element,
   parseAttrs,
   encodeEntities,
-  decodeEntities,
+  decodeEntities
 } from "./xml-parser";
 import memoize from "./common/memoize";
 import { version as VERSION } from "../package.json";
 import {
+  CpeGetResponse,
+  CpeSetResponse,
   InformRequest,
+  AcsResponse,
+  CpeRequest,
   FaultStruct,
   SpvFault,
   CpeFault,
   SoapMessage,
-  TransferCompleteRequest,
-  AcsRequest,
-  GetParameterNamesResponse,
-  GetParameterValuesResponse,
-  GetParameterAttributesResponse,
-  SetParameterValuesResponse,
-  SetParameterAttributesResponse,
-  AddObjectResponse,
-  DeleteObject,
-  DeleteObjectResponse,
-  RebootResponse,
-  FactoryResetResponse,
-  DownloadResponse,
-  UploadResponse,
-  GetRPCMethodsRequest,
-  RequestDownloadRequest,
-  AcsResponse,
+  TransferCompleteRequest
 } from "./types";
-import Path from "./common/path";
 
 const SERVER_NAME = `GenieACS/${VERSION}`;
 
@@ -60,58 +47,60 @@ const NAMESPACES = {
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-0",
+    cwmp: "urn:dslforum-org:cwmp-1-0"
   },
   "1.1": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-1",
+    cwmp: "urn:dslforum-org:cwmp-1-1"
   },
   "1.2": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
   },
   "1.3": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2",
+    cwmp: "urn:dslforum-org:cwmp-1-2"
   },
   "1.4": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-3",
-  },
+    cwmp: "urn:dslforum-org:cwmp-1-3"
+  }
 };
 
-let warnings: Record<string, unknown>[];
+let warnings;
 
 const memoizedParseAttrs = memoize(parseAttrs);
 
-function parseBool(v: string): boolean {
-  if (v === "true" || v === "1") return true;
-  if (v === "false" || v === "0") return false;
-  return null;
+function parseBool(v): boolean {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else return null;
 }
 
 function event(xml: Element): string[] {
   return xml.children
-    .filter((n) => n.localName === "EventStruct")
-    .map((c) => c.children.find((n) => n.localName === "EventCode").text);
+    .filter(n => n.localName === "EventStruct")
+    .map(c => c.children.find(n => n.localName === "EventCode").text.trim());
 }
 
-function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
+function parameterInfoList(xml: Element): [string, boolean][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterInfoStruct")
-    .map<[Path, boolean, boolean]>((e) => {
+    .filter(e => e.localName === "ParameterInfoStruct")
+    .map<[string, boolean]>(e => {
       let param: string, value: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -126,34 +115,30 @@ function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
 
       let parsed: boolean = parseBool(value);
 
-      if (parsed == null) {
+      if (parsed === null) {
         warnings.push({
-          message: "Missing or invalid XML node",
-          element: "Writable",
-          parameter: param,
+          message: "Invalid writable attribute",
+          parameter: param
         });
         parsed = false;
       }
 
-      if (param && !param.endsWith("."))
-        return [Path.parse(param), false, parsed];
-      else return [Path.parse(param.slice(0, -1)), true, parsed];
+      return [param, parsed];
     });
 }
 
-const getValueType = memoize((str: string) => {
-  const attrs = parseAttrs(str);
-  for (const attr of attrs) if (attr.localName === "type") return attr.value;
-
-  return null;
-});
+const getValueType = memoize((str: string) =>
+  parseAttrs(str)
+    .find(s => s.localName === "type")
+    .value.trim()
+);
 
 function parameterValueList(
   xml: Element
-): [Path, string | number | boolean, string][] {
+): [string, string | number | boolean, string][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterValueStruct")
-    .map<[Path, string | number | boolean, string]>((e) => {
+    .filter(e => e.localName === "ParameterValueStruct")
+    .map<[string, string | number | boolean, string]>(e => {
       let valueElement: Element, param: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -166,25 +151,16 @@ function parameterValueList(
         }
       }
 
-      let valueType = getValueType(valueElement.attrs);
-      if (!valueType) {
-        warnings.push({
-          message: "Missing or invalid XML node",
-          attribute: "type",
-          parameter: param,
-        });
-        valueType = "xsd:string";
-      }
+      const valueType = getValueType(valueElement.attrs);
 
       const value = decodeEntities(valueElement.text);
       let parsed: string | number | boolean = value;
       if (valueType === "xsd:boolean") {
         parsed = parseBool(value);
-        if (parsed == null) {
+        if (parsed === null) {
           warnings.push({
-            message: "Missing or invalid XML node",
-            element: "Value",
-            parameter: param,
+            message: "Invalid value attribute",
+            parameter: param
           });
           parsed = value;
         }
@@ -192,9 +168,8 @@ function parameterValueList(
         parsed = parseInt(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Missing or invalid XML node",
-            element: "Value",
-            parameter: param,
+            message: "Invalid value attribute",
+            parameter: param
           });
           parsed = value;
         }
@@ -202,22 +177,21 @@ function parameterValueList(
         parsed = Date.parse(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Missing or invalid XML node",
-            element: "Value",
-            parameter: param,
+            message: "Invalid value attribute",
+            parameter: param
           });
           parsed = value;
         }
       }
 
-      return [Path.parse(param), parsed, valueType];
+      return [param, parsed, valueType];
     });
 }
 
-function parameterAttributeList(xml: Element): [Path, number, string[]][] {
+function parameterAttributeList(xml: Element): [string, number, string[]][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterAttributeStruct")
-    .map<[Path, number, string[]]>((e) => {
+    .filter(e => e.localName === "ParameterAttributeStruct")
+    .map<[string, number, string[]]>(e => {
       let notificationElement: Element,
         accessListElement: Element,
         param: string;
@@ -235,21 +209,21 @@ function parameterAttributeList(xml: Element): [Path, number, string[]][] {
         }
       }
 
-      let notification = parseInt(notificationElement.text);
-      if (isNaN(notification)) {
+      const notification = decodeEntities(notificationElement.text);
+      const notif = +notification;
+
+      if (notif !== parseInt(notification)) {
         warnings.push({
-          message: "Missing or invalid XML node",
-          element: "Notification",
-          parameter: param,
+          message: "Invalid notification attribute",
+          parameter: param
         });
-        notification = 0;
       }
 
       const accessList = accessListElement.children
-        .filter((c) => c.localName === "string")
-        .map((c) => decodeEntities(c.text));
+        .filter(c => c.localName === "string")
+        .map(c => decodeEntities(c.text));
 
-      return [Path.parse(param), notification, accessList];
+      return [param, notif, accessList];
     });
 }
 
@@ -259,12 +233,12 @@ function GetParameterNames(methodRequest): string {
   }</ParameterPath><NextLevel>${+methodRequest.nextLevel}</NextLevel></cwmp:GetParameterNames>`;
 }
 
-function GetParameterNamesResponse(xml): GetParameterNamesResponse {
+function GetParameterNamesResponse(xml): CpeGetResponse {
   return {
     name: "GetParameterNamesResponse",
     parameterList: parameterInfoList(
-      xml.children.find((n) => n.localName === "ParameterList")
-    ),
+      xml.children.find(n => n.localName === "ParameterList")
+    )
   };
 }
 
@@ -272,16 +246,16 @@ function GetParameterValues(methodRequest): string {
   return `<cwmp:GetParameterValues><ParameterNames soap-enc:arrayType="xsd:string[${
     methodRequest.parameterNames.length
   }]">${methodRequest.parameterNames
-    .map((p) => `<string>${p}</string>`)
+    .map(p => `<string>${p}</string>`)
     .join("")}</ParameterNames></cwmp:GetParameterValues>`;
 }
 
-function GetParameterValuesResponse(xml: Element): GetParameterValuesResponse {
+function GetParameterValuesResponse(xml: Element): CpeGetResponse {
   return {
     name: "GetParameterValuesResponse",
     parameterList: parameterValueList(
-      xml.children.find((n) => n.localName === "ParameterList")
-    ),
+      xml.children.find(n => n.localName === "ParameterList")
+    )
   };
 }
 
@@ -289,23 +263,21 @@ function GetParameterAttributes(methodRequest): string {
   return `<cwmp:GetParameterAttributes><ParameterNames soap-enc:arrayType="xsd:string[${
     methodRequest.parameterNames.length
   }]">${methodRequest.parameterNames
-    .map((p) => `<string>${p}</string>`)
+    .map(p => `<string>${p}</string>`)
     .join("")}</ParameterNames></cwmp:GetParameterAttributes>`;
 }
 
-function GetParameterAttributesResponse(
-  xml: Element
-): GetParameterAttributesResponse {
+function GetParameterAttributesResponse(xml: Element): CpeGetResponse {
   return {
     name: "GetParameterAttributesResponse",
     parameterList: parameterAttributeList(
-      xml.children.find((n) => n.localName === "ParameterList")
-    ),
+      xml.children.find(n => n.localName === "ParameterList")
+    )
   };
 }
 
 function SetParameterValues(methodRequest): string {
-  const params = methodRequest.parameterList.map((p) => {
+  const params = methodRequest.parameterList.map(p => {
     let val = p[1];
     if (p[2] === "xsd:dateTime" && typeof val === "number") {
       val = new Date(val).toISOString();
@@ -321,38 +293,21 @@ function SetParameterValues(methodRequest): string {
 
   return `<cwmp:SetParameterValues><ParameterList soap-enc:arrayType="cwmp:ParameterValueStruct[${
     methodRequest.parameterList.length
-  }]">${params.join("")}</ParameterList><ParameterKey>${
-    methodRequest.parameterKey || ""
-  }</ParameterKey></cwmp:SetParameterValues>`;
-}
-
-function SetParameterValuesResponse(xml: Element): SetParameterValuesResponse {
-  let status: number;
-
-  for (const c of xml.children) {
-    switch (c.localName) {
-      case "Status":
-        status = parseInt(c.text);
-        break;
-    }
-  }
-
-  if (!(status >= 0)) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "Status",
-    });
-    status = 0;
+  }]">${params.join(
+    ""
+  )}</ParameterList><ParameterKey>${methodRequest.parameterKey ||
+    ""}</ParameterKey></cwmp:SetParameterValues>`;
 }
 
+function SetParameterValuesResponse(xml: Element): CpeSetResponse {
   return {
     name: "SetParameterValuesResponse",
-    status: status,
+    status: parseInt(xml.children.find(n => n.localName === "Status").text)
   };
 }
 
 function SetParameterAttributes(methodRequest): string {
-  const params = methodRequest.parameterList.map((p) => {
+  const params = methodRequest.parameterList.map(p => {
     return `<SetParameterAttributesStruct><Name>${
       p[0]
     }</Name><NotificationChange>${
@@ -366,7 +321,7 @@ function SetParameterAttributes(methodRequest): string {
     }]">${
       p[2] == null
         ? ""
-        : p[2].map((s) => `<string>${encodeEntities(s)}</string>`).join("")
+        : p[2].map(s => `<string>${encodeEntities(s)}</string>`).join("")
     }</AccessList></SetParameterAttributesStruct>`;
   });
 
@@ -375,26 +330,25 @@ function SetParameterAttributes(methodRequest): string {
   }]">${params.join("")}</ParameterList></cwmp:SetParameterAttributes>`;
 }
 
-function SetParameterAttributesResponse(): SetParameterAttributesResponse {
+function SetParameterAttributesResponse(): CpeSetResponse {
   return {
-    name: "SetParameterAttributesResponse",
+    name: "SetParameterAttributesResponse"
   };
 }
 
 function AddObject(methodRequest): string {
   return `<cwmp:AddObject><ObjectName>${
     methodRequest.objectName
-  }</ObjectName><ParameterKey>${
-    methodRequest.parameterKey || ""
-  }</ParameterKey></cwmp:AddObject>`;
+  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
+    ""}</ParameterKey></cwmp:AddObject>`;
 }
 
-function AddObjectResponse(xml: Element): AddObjectResponse {
-  let instanceNumber: string, status: number;
+function AddObjectResponse(xml: Element): CpeSetResponse {
+  let instanceNumber, status;
   for (const c of xml.children) {
     switch (c.localName) {
       case "InstanceNumber":
-        instanceNumber = c.text;
+        instanceNumber = parseInt(c.text);
         break;
       case "Status":
         status = parseInt(c.text);
@@ -402,66 +356,35 @@ function AddObjectResponse(xml: Element): AddObjectResponse {
     }
   }
 
-  if (!/^[0-9]+$/.test(instanceNumber))
-    throw new Error("Missing or invalid instance number");
-
-  if (!(status >= 0)) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "Status",
-    });
-    status = 0;
-  }
-
   return {
     name: "AddObjectResponse",
     instanceNumber: instanceNumber,
-    status: status,
+    status: status
   };
 }
 
 function DeleteObject(methodRequest): string {
   return `<cwmp:DeleteObject><ObjectName>${
     methodRequest.objectName
-  }</ObjectName><ParameterKey>${
-    methodRequest.parameterKey || ""
-  }</ParameterKey></cwmp:DeleteObject>`;
-}
-
-function DeleteObjectResponse(xml: Element): DeleteObjectResponse {
-  let status: number;
-
-  for (const c of xml.children) {
-    switch (c.localName) {
-      case "Status":
-        status = parseInt(c.text);
-        break;
-    }
-  }
-
-  if (!(status >= 0)) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "Status",
-    });
-    status = 0;
+  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
+    ""}</ParameterKey></cwmp:DeleteObject>`;
 }
 
+function DeleteObjectResponse(xml: Element): CpeSetResponse {
   return {
     name: "DeleteObjectResponse",
-    status: status,
+    status: parseInt(xml.children.find(n => n.localName === "Status").text)
   };
 }
 
 function Reboot(methodRequest): string {
-  return `<cwmp:Reboot><CommandKey>${
-    methodRequest.commandKey || ""
-  }</CommandKey></cwmp:Reboot>`;
+  return `<cwmp:Reboot><CommandKey>${methodRequest.commandKey ||
+    ""}</CommandKey></cwmp:Reboot>`;
 }
 
-function RebootResponse(): RebootResponse {
+function RebootResponse(): CpeSetResponse {
   return {
-    name: "RebootResponse",
+    name: "RebootResponse"
   };
 }
 
@@ -469,54 +392,33 @@ function FactoryReset(): string {
   return "<cwmp:FactoryReset></cwmp:FactoryReset>";
 }
 
-function FactoryResetResponse(): FactoryResetResponse {
+function FactoryResetResponse(): CpeSetResponse {
   return {
-    name: "FactoryResetResponse",
+    name: "FactoryResetResponse"
   };
 }
 
 function Download(methodRequest): string {
-  return `<cwmp:Download><CommandKey>${
-    methodRequest.commandKey || ""
-  }</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
+  return `<cwmp:Download><CommandKey>${methodRequest.commandKey ||
+    ""}</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
     methodRequest.url
   }</URL><Username>${encodeEntities(
     methodRequest.username || ""
   )}</Username><Password>${encodeEntities(
     methodRequest.password || ""
-  )}</Password><FileSize>${
-    methodRequest.fileSize || "0"
-  }</FileSize><TargetFileName>${encodeEntities(
+  )}</Password><FileSize>${methodRequest.fileSize ||
+    "0"}</FileSize><TargetFileName>${encodeEntities(
     methodRequest.targetFileName || ""
-  )}</TargetFileName><DelaySeconds>${
-    methodRequest.delaySeconds || "0"
-  }</DelaySeconds><SuccessURL>${encodeEntities(
+  )}</TargetFileName><DelaySeconds>${methodRequest.delaySeconds ||
+    "0"}</DelaySeconds><SuccessURL>${encodeEntities(
     methodRequest.successUrl || ""
   )}</SuccessURL><FailureURL>${encodeEntities(
     methodRequest.failureUrl || ""
   )}</FailureURL></cwmp:Download>`;
 }
 
-function Upload(methodRequest): string {
-  return `<cwmp:Upload><CommandKey>${
-    methodRequest.commandKey || ""
-  }</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
-    methodRequest.url
-  }</URL><Username>${encodeEntities(
-    methodRequest.username || ""
-  )}</Username><Password>${encodeEntities(
-    methodRequest.password || ""
-  )}</Password><DelaySeconds>${
-    methodRequest.delaySeconds || "0"
-  }</DelaySeconds><SuccessURL>${encodeEntities(
-    methodRequest.successUrl || ""
-  )}</SuccessURL><FailureURL>${encodeEntities(
-    methodRequest.failureUrl || ""
-  )}</FailureURL></cwmp:Upload>`;
-}
-
-function DownloadResponse(xml: Element): DownloadResponse {
-  let status: number, startTime: number, completeTime: number;
+function DownloadResponse(xml: Element): CpeSetResponse {
+  let status, startTime, completeTime;
   for (const c of xml.children) {
     switch (c.localName) {
       case "Status":
@@ -531,75 +433,21 @@ function DownloadResponse(xml: Element): DownloadResponse {
     }
   }
 
-  if (!(status >= 0)) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "Status",
-    });
-    status = 0;
-  }
-
-  if (startTime == null || isNaN(startTime)) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "StartTime",
-    });
-    startTime = Date.parse("0001-01-01T00:00:00Z");
-  }
-
-  if (completeTime == null || isNaN(completeTime)) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "CompleteTime",
-    });
-    completeTime = Date.parse("0001-01-01T00:00:00Z");
-  }
-
   return {
     name: "DownloadResponse",
     status: status,
     startTime: startTime,
-    completeTime: completeTime,
-  };
-}
-
-function UploadResponse(xml: Element): UploadResponse {
-  let status: number, startTime: number, completeTime: number;
-  for (const c of xml.children) {
-    switch (c.localName) {
-      case "Status":
-        status = parseInt(c.text);
-        break;
-      case "StartTime":
-        startTime = Date.parse(c.text);
-        break;
-      case "CompleteTime":
-        completeTime = Date.parse(c.text);
-        break;
-    }
-  }
-
-  if (!(status >= 0)) {
-    warnings.push({ message: "Invalid UploadResponse status" });
-    status = 0;
-  }
-
-  return {
-    name: "UploadResponse",
-    status: status,
-    startTime: startTime,
-    completeTime: completeTime,
+    completeTime: completeTime
   };
 }
 
 function Inform(xml: Element): InformRequest {
-  let retryCount: number, evnt: string[];
-  let parameterList: [Path, string | number | boolean, string][];
+  let retryCount, evnt, parameterList;
   const deviceId = {
     Manufacturer: null,
     OUI: null,
     ProductClass: null,
-    SerialNumber: null,
+    SerialNumber: null
   };
 
   for (const c of xml.children) {
@@ -622,36 +470,12 @@ function Inform(xml: Element): InformRequest {
     }
   }
 
-  if (!deviceId || !deviceId.SerialNumber || !deviceId.OUI)
-    throw new Error("Missing or invalid DeviceId element");
-
-  if (!parameterList) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "ParameterList",
-    });
-    parameterList = [];
-  }
-
-  if (!evnt) {
-    warnings.push({ message: "Missing or invalid XML node", element: "Event" });
-    evnt = [];
-  }
-
-  if (retryCount == null || isNaN(retryCount)) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "RetryCount",
-    });
-    retryCount = 0;
-  }
-
   return {
     name: "Inform",
     parameterList: parameterList,
     deviceId: deviceId,
     event: evnt,
-    retryCount: retryCount,
+    retryCount: retryCount
   };
 }
 
@@ -659,7 +483,7 @@ function InformResponse(): string {
   return "<cwmp:InformResponse><MaxEnvelopes>1</MaxEnvelopes></cwmp:InformResponse>";
 }
 
-function GetRPCMethods(): GetRPCMethodsRequest {
+function GetRPCMethods(): AcsResponse {
   return { name: "GetRPCMethods" };
 }
 
@@ -667,15 +491,12 @@ function GetRPCMethodsResponse(methodResponse): string {
   return `<cwmp:GetRPCMethodsResponse><MethodList soap-enc:arrayType="xsd:string[${
     methodResponse.methodList.length
   }]">${methodResponse.methodList
-    .map((m) => `<string>${m}</string>`)
+    .map(m => `<string>${m}</string>`)
     .join("")}</MethodList></cwmp:GetRPCMethodsResponse>`;
 }
 
 function TransferComplete(xml: Element): TransferCompleteRequest {
-  let commandKey: string,
-    _faultStruct: FaultStruct,
-    startTime: number,
-    completeTime: number;
+  let commandKey, _faultStruct, startTime, completeTime;
   for (const c of xml.children) {
     switch (c.localName) {
       case "CommandKey":
@@ -693,44 +514,12 @@ function TransferComplete(xml: Element): TransferCompleteRequest {
     }
   }
 
-  if (commandKey == null) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "CommandKey",
-    });
-    commandKey = "";
-  }
-
-  if (!_faultStruct) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "FaultStruct",
-    });
-    _faultStruct = { faultCode: "0", faultString: "" };
-  }
-
-  if (startTime == null || isNaN(startTime)) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "StartTime",
-    });
-    startTime = Date.parse("0001-01-01T00:00:00Z");
-  }
-
-  if (completeTime == null || isNaN(completeTime)) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "CompleteTime",
-    });
-    completeTime = Date.parse("0001-01-01T00:00:00Z");
-  }
-
   return {
     name: "TransferComplete",
     commandKey: commandKey,
     faultStruct: _faultStruct,
     startTime: startTime,
-    completeTime: completeTime,
+    completeTime: completeTime
   };
 }
 
@@ -738,10 +527,10 @@ function TransferCompleteResponse(): string {
   return "<cwmp:TransferCompleteResponse></cwmp:TransferCompleteResponse>";
 }
 
-function RequestDownload(xml: Element): RequestDownloadRequest {
+function RequestDownload(xml: Element): CpeRequest {
   return {
     name: "RequestDownload",
-    fileType: xml.children.find((n) => n.localName === "FileType").text,
+    fileType: xml.children.find(n => n.localName === "FileType").text
   };
 }
 
@@ -750,12 +539,7 @@ function RequestDownloadResponse(): string {
 }
 
 function faultStruct(xml: Element): FaultStruct {
-  let faultCode: string,
-    faultString: string,
-    setParameterValuesFault: SpvFault[],
-    pn: string,
-    fc: string,
-    fs: string;
+  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
   for (const c of xml.children) {
     switch (c.localName) {
       case "FaultCode":
@@ -766,7 +550,6 @@ function faultStruct(xml: Element): FaultStruct {
         break;
       case "SetParameterValuesFault":
         setParameterValuesFault = setParameterValuesFault || [];
-        pn = fc = fs = null;
         for (const cc of c.children) {
           switch (cc.localName) {
             case "ParameterName":
@@ -783,32 +566,16 @@ function faultStruct(xml: Element): FaultStruct {
         setParameterValuesFault.push({
           parameterName: pn,
           faultCode: fc,
-          faultString: fs,
+          faultString: fs
         });
     }
   }
 
-  if (faultCode == null) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "FaultCode",
-    });
-    faultCode = "";
-  }
-
-  if (faultString == null) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "FaultString",
-    });
-    faultString = "";
-  }
-
   return { faultCode, faultString, setParameterValuesFault };
 }
 
 function fault(xml: Element): CpeFault {
-  let faultCode: string, faultString: string, detail: FaultStruct;
+  let faultCode, faultString, detail;
   for (const c of xml.children) {
     switch (c.localName) {
       case "faultcode":
@@ -818,37 +585,15 @@ function fault(xml: Element): CpeFault {
         faultString = decodeEntities(c.text);
         break;
       case "detail":
-        detail = faultStruct(c.children.find((n) => n.localName === "Fault"));
+        detail = faultStruct(c.children.find(n => n.localName === "Fault"));
         break;
     }
   }
 
-  if (!detail) throw new Error("Missing detail element");
-
-  if (faultCode == null) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "faultcode",
-    });
-    faultCode = "Client";
+  return { faultCode, faultString, detail };
 }
 
-  if (faultString == null) {
-    warnings.push({
-      message: "Missing or invalid XML node",
-      element: "faultstring",
-    });
-    faultString = "CWMP fault";
-  }
-
-  return { faultCode, faultString, detail } as CpeFault;
-}
-
-export function request(
-  body: string,
-  cwmpVersion: string,
-  warn: Record<string, unknown>[]
-): SoapMessage {
+export function request(body: string, cwmpVersion, warn): SoapMessage {
   warnings = warn;
 
   const rpc = {
@@ -857,7 +602,7 @@ export function request(
     sessionTimeout: null,
     cpeRequest: null,
     cpeFault: null,
-    cpeResponse: null,
+    cpeResponse: null
   };
 
   if (!body.length) return rpc;
@@ -904,9 +649,9 @@ export function request(
         const attrs = memoizedParseAttrs(e.attrs);
         const attr = namespace
           ? attrs.find(
-              (s) => s.namespace === "xmlns" && s.localName === namespace
+              s => s.namespace === "xmlns" && s.localName === namespace
             )
-          : attrs.find((s) => s.name === "xmlns");
+          : attrs.find(s => s.name === "xmlns");
 
         if (attr) namespaceHref = attr.value;
       }
@@ -975,9 +720,6 @@ export function request(
     case "DownloadResponse":
       rpc.cpeResponse = DownloadResponse(methodElement);
       break;
-    case "UploadResponse":
-      rpc.cpeResponse = UploadResponse(methodElement);
-      break;
     case "Fault":
       rpc.cpeFault = fault(methodElement);
       break;
@@ -1003,18 +745,13 @@ const namespacesAttrs = {
     .join(" "),
   "1.4": Object.entries(NAMESPACES["1.4"])
     .map(([k, v]) => `xmlns:${k}="${v}"`)
-    .join(" "),
+    .join(" ")
 };
 
-export function response(rpc: {
-  id: string;
-  acsRequest?: AcsRequest;
-  acsResponse?: AcsResponse;
-  cwmpVersion?: string;
-}): { code: number; headers: Record<string, string>; data: string } {
+export function response(rpc): { code: number; headers: {}; data: string } {
   const headers = {
     Server: SERVER_NAME,
-    SOAPServer: SERVER_NAME,
+    SOAPServer: SERVER_NAME
   };
 
   if (!rpc) return { code: 204, headers: headers, data: "" };
@@ -1035,11 +772,7 @@ export function response(rpc: {
         body = RequestDownloadResponse();
         break;
       default:
-        throw new Error(
-          `Unknown method response type ${
-            (rpc.acsResponse as AcsResponse).name
-          }`
-        );
+        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
     }
   } else if (rpc.acsRequest) {
     switch (rpc.acsRequest.name) {
@@ -1073,13 +806,8 @@ export function response(rpc: {
       case "Download":
         body = Download(rpc.acsRequest);
         break;
-      case "Upload":
-        body = Upload(rpc.acsRequest);
-        break;
       default:
-        throw new Error(
-          `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`
-        );
+        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
     }
   }
 
@@ -1091,6 +819,6 @@ export function response(rpc: {
       namespacesAttrs[rpc.cwmpVersion]
     }><soap-env:Header><cwmp:ID soap-env:mustUnderstand="1">${
       rpc.id
-    }</cwmp:ID></soap-env:Header><soap-env:Body>${body}</soap-env:Body></soap-env:Envelope>`,
+    }</cwmp:ID></soap-env:Header><soap-env:Body>${body}</soap-env:Body></soap-env:Envelope>`
   };
 }

lib/types.ts

@@ -22,7 +22,6 @@ import PathSet from "./common/path-set";
 import VersionedMap from "./versioned-map";
 import InstanceSet from "./instance-set";
 import { IncomingMessage, ServerResponse } from "http";
-import { Script } from "vm";
 
 export type Expression = string | number | boolean | null | any[];
 
@@ -121,10 +120,6 @@ export interface SyncState {
   downloadsToCreate: InstanceSet;
   downloadsValues: Map<Path, string | number>;
   downloadsDownload: Map<Path, number>;
-  uploadsToDelete: Set<Path>;
-  uploadsToCreate: InstanceSet;
-  uploadsValues: Map<Path, string | number>;
-  uploadsUpload: Map<Path, number>;
   reboot: number;
   factoryReset: number;
 }
@@ -154,6 +149,7 @@ export interface SessionContext {
   faults?: { [channel: string]: SessionFault };
   retries?: { [channel: string]: number };
   cacheSnapshot?: string;
+  configContext?: { [name: string]: any };
   httpResponse?: ServerResponse;
   httpRequest?: IncomingMessage;
   faultsTouched?: { [channel: string]: boolean };
@@ -161,13 +157,11 @@ export interface SessionContext {
   new?: boolean;
   debug?: boolean;
   state: number;
-  authState: number;
   tasks?: Task[];
   operations?: { [commandKey: string]: Operation };
   cacheUntil?: number;
   syncState?: SyncState;
   lastActivity?: number;
-  extendLock?: number;
   rpcRequest?: AcsRequest;
   operationsTouched?: { [commandKey: string]: 1 | 0 };
   provisionsRet?: any[];
@@ -175,7 +169,7 @@ export interface SessionContext {
 }
 
 export interface Task {
-  _id?: string;
+  _id: string;
   name: string;
   parameterNames?: string[];
   parameterValues?: [string, string | number | boolean, string?][];
@@ -197,103 +191,51 @@ export interface Operation {
     instance: string;
     fileType: string;
     fileName: string;
-    targetFileName?: string;
+    targetFileName: string;
   };
 }
 
-export type AcsRequest =
-  | GetParameterNames
-  | GetParameterValues
-  | GetParameterAttributes
-  | SetParameterValues
-  | SetParameterAttributes
-  | AddObject
-  | DeleteObject
-  | FactoryReset
-  | Reboot
-  | Download
-  | Upload;
-
-export interface GetParameterNames {
-  name: "GetParameterNames";
-  parameterPath: string;
-  nextLevel: boolean;
+export interface AcsRequest {
+  name: string;
+  next?: string;
 }
 
-export interface GetParameterValues {
-  name: "GetParameterValues";
-  parameterNames: string[];
+export interface GetAcsRequest extends AcsRequest {
+  name: "GetParameterNames" | "GetParameterValues" | "GetParameterAttributes";
+  objectName?: string;
+  parameterNames?: string[];
+  parameterPath?: string;
+  nextLevel?: boolean;
+  instanceValues?: { [name: string]: string };
 }
 
-export interface GetParameterAttributes {
-  name: "GetParameterAttributes";
-  parameterNames: string[];
-}
-
-export interface SetParameterValues {
-  name: "SetParameterValues";
-  parameterList: [string, boolean | number | string, string][];
-  parameterKey?: string;
+export interface SetAcsRequest extends AcsRequest {
+  name:
+    | "SetParameterValues"
+    | "SetParameterAttributes"
+    | "AddObject"
+    | "DeleteObject"
+    | "FactoryReset"
+    | "Reboot"
+    | "Download";
+  parameterList?:
+    | [string, string | number | boolean, string][]
+    | [string, number, string[]][];
+  instanceValues?: { [param: string]: string | number | boolean };
+  objectName?: string;
   DATETIME_MILLISECONDS?: boolean;
   BOOLEAN_LITERAL?: boolean;
-}
-
-export interface SetParameterAttributes {
-  name: "SetParameterAttributes";
-  parameterList: [string, number, string[]][];
-}
-
-export interface AddObject {
-  name: "AddObject";
-  objectName: string;
-  parameterKey?: string;
-  instanceValues: Record<string, string>;
-}
-
-export interface DeleteObject {
-  name: "DeleteObject";
-  objectName: string;
-  parameterKey?: string;
-}
-
-export interface FactoryReset {
-  name: "FactoryReset";
   commandKey?: string;
-}
-
-export interface Reboot {
-  name: "Reboot";
-  commandKey?: string;
-}
-
-export interface Download {
-  name: "Download";
-  commandKey: string;
-  instance: string;
-  fileType: string;
-  fileName?: string;
-  url?: string;
-  username?: string;
-  password?: string;
+  instance?: string;
+  fileType?: string;
   fileSize?: number;
+  url?: string;
+  fileName?: string;
   targetFileName?: string;
-  delaySecods?: number;
-  successUrl?: string;
-  failureUrl?: string;
 }
 
-export interface Upload {
-  name: "Upload";
-  commandKey: string;
-  instance: string;
-  fileType: string;
-  fileName?: string;
-  url?: string;
-  username?: string;
-  password?: string;
-  delaySecods?: number;
-  successUrl?: string;
-  failureUrl?: string;
+export interface CpeResponse {
+  name: string;
 }
 
 export interface SpvFault {
@@ -309,88 +251,43 @@ export interface FaultStruct {
 }
 
 export interface CpeFault {
-  faultCode: "Client" | "Server";
-  faultString: "CWMP fault";
+  faultCode: string;
+  faultString: string;
   detail?: FaultStruct;
 }
 
-export type CpeResponse =
-  | GetParameterNamesResponse
-  | GetParameterValuesResponse
-  | GetParameterAttributesResponse
-  | SetParameterValuesResponse
-  | SetParameterAttributesResponse
-  | AddObjectResponse
-  | DeleteObjectResponse
-  | RebootResponse
-  | FactoryResetResponse
-  | DownloadResponse
-  | UploadResponse;
-
-export interface GetParameterNamesResponse {
-  name: "GetParameterNamesResponse";
-  parameterList: [Path, boolean, boolean][];
+export interface CpeGetResponse extends CpeResponse {
+  name:
+    | "GetParameterNamesResponse"
+    | "GetParameterValuesResponse"
+    | "GetParameterAttributesResponse";
+  parameterList?:
+    | [string, boolean][]
+    | [string, string | number | boolean, string][]
+    | [string, number, string[]][];
 }
 
-export interface GetParameterValuesResponse {
-  name: "GetParameterValuesResponse";
-  parameterList: [Path, string | number | boolean, string][];
-}
-
-export interface GetParameterAttributesResponse {
-  name: "GetParameterAttributesResponse";
-  parameterList: [Path, number, string[]][];
-}
-
-export interface SetParameterValuesResponse {
-  name: "SetParameterValuesResponse";
-  status: number;
-}
-
-export interface SetParameterAttributesResponse {
-  name: "SetParameterAttributesResponse";
-}
-
-export interface AddObjectResponse {
-  name: "AddObjectResponse";
-  instanceNumber: string;
-  status: number;
-}
-
-export interface DeleteObjectResponse {
-  name: "DeleteObjectResponse";
-  status: number;
-}
-
-export interface RebootResponse {
-  name: "RebootResponse";
-}
-
-export interface FactoryResetResponse {
-  name: "FactoryResetResponse";
-}
-
-export interface DownloadResponse {
-  name: "DownloadResponse";
-  status: number;
+export interface CpeSetResponse extends CpeResponse {
+  name:
+    | "SetParameterValuesResponse"
+    | "SetParameterAttributesResponse"
+    | "AddObjectResponse"
+    | "DeleteObjectResponse"
+    | "RebootResponse"
+    | "FactoryResetResponse"
+    | "DownloadResponse";
+  status?: number;
+  instanceNumber?: string;
   startTime?: number;
   completeTime?: number;
 }
 
-export interface UploadResponse {
-  name: "UploadResponse";
-  status: number;
-  startTime?: number;
-  completeTime?: number;
+export interface CpeRequest {
+  name: string;
+  fileType?: string;
 }
 
-export type CpeRequest =
-  | InformRequest
-  | TransferCompleteRequest
-  | GetRPCMethodsRequest
-  | RequestDownloadRequest;
-
-export interface InformRequest {
+export interface InformRequest extends CpeRequest {
   name: "Inform";
   deviceId: {
     Manufacturer: string;
@@ -400,10 +297,10 @@ export interface InformRequest {
   };
   event: string[];
   retryCount: number;
-  parameterList: [Path, string | number | boolean, string][];
+  parameterList: [string, string | number | boolean, string][];
 }
 
-export interface TransferCompleteRequest {
+export interface TransferCompleteRequest extends CpeRequest {
   name: "TransferComplete";
   commandKey?: string;
   faultStruct?: FaultStruct;
@@ -411,36 +308,10 @@ export interface TransferCompleteRequest {
   completeTime?: number;
 }
 
-export interface GetRPCMethodsRequest {
-  name: "GetRPCMethods";
-}
-
-export interface RequestDownloadRequest {
-  name: "RequestDownload";
-  fileType: string;
-}
-
-export type AcsResponse =
-  | InformResponse
-  | GetRPCMethodsResponse
-  | TransferCompleteResponse
-  | RequestDownloadResponse;
-
-export interface InformResponse {
-  name: "InformResponse";
-}
-
-export interface GetRPCMethodsResponse {
-  name: "GetRPCMethodsResponse";
-  methodList: string[];
-}
-
-export interface TransferCompleteResponse {
-  name: "TransferCompleteResponse";
-}
-
-export interface RequestDownloadResponse {
-  name: "RequestDownloadResponse";
+export interface AcsResponse {
+  name: string;
+  commandKey?: string;
+  faultStruct?: FaultStruct;
 }
 
 export interface QueryOptions {
@@ -455,7 +326,7 @@ export interface QueryOptions {
 export interface Declaration {
   path: Path;
   pathGet: number;
-  pathSet?: number | [number, number];
+  pathSet?: [number, number];
   attrGet?: {
     object?: number;
     writable?: number;
@@ -491,36 +362,8 @@ export interface Preset {
   channel: string;
   schedule?: { md5: string; duration: number; schedule: any };
   events?: { [event: string]: boolean };
-  precondition?: Expression;
-  provisions: [string, ...Expression[]][];
-}
-
-export interface Provisions {
-  [name: string]: { md5: string; script: Script };
-}
-
-export interface VirtualParameters {
-  [name: string]: { md5: string; script: Script };
-}
-
-export interface Files {
-  [name: string]: { length: number; md5: string; contentType: string };
-}
-
-export interface Users {
-  [name: string]: { password: string; salt: string; roles: string[] };
-}
-
-export interface Permissions {
-  [role: string]: {
-    [access: number]: {
-      [resource: string]: {
-        access: number;
-        filter: Expression;
-        validate?: Expression;
-      };
-    };
-  };
+  precondition?: {};
+  provisions: string[][];
 }
 
 export type PermissionSet = {
@@ -531,21 +374,6 @@ export type PermissionSet = {
   };
 }[];
 
-export interface Config {
-  [name: string]: Expression;
-}
-
-export interface UiConfig {
-  filters: Record<string, unknown>;
-  device: Record<string, unknown>;
-  index: Record<string, unknown>;
-  overview: {
-    charts?: Record<string, unknown>;
-    groups?: Record<string, unknown>;
-  };
-  pageSize?: Expression;
-}
-
 export interface SoapMessage {
   id: string;
   cwmpVersion: string;

lib/ui.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { constants } from "zlib";
+import { Z_SYNC_FLUSH } from "zlib";
 import Koa from "koa";
 import Router from "koa-router";
 import * as jwt from "jsonwebtoken";
@@ -34,7 +34,6 @@ import { PermissionSet } from "./types";
 import { authLocal } from "./ui/api-functions";
 import * as init from "./init";
 import { version as VERSION } from "../package.json";
-import memoize from "./common/memoize";
 
 declare module "koa" {
   interface Request {
@@ -48,22 +47,9 @@ const router = new Router();
 const JWT_SECRET = "" + config.get("UI_JWT_SECRET");
 const JWT_COOKIE = "genieacs-ui-jwt";
 
-const getAuthorizer = memoize(
-  (snapshot: string, rolesStr: string): Authorizer => {
-    const roles: string[] = JSON.parse(rolesStr);
-    const allPermissions = localCache.getPermissions(snapshot);
-    const permissionSets: PermissionSet[] = roles.map((r) =>
-      Object.values(allPermissions[r] || {})
-    );
-    return new Authorizer(permissionSets);
-  }
-);
-
-koa.on("error", async (err) => {
-  setTimeout(() => {
+koa.on("error", async err => {
   throw err;
 });
-});
 
 koa.use(async (ctx, next) => {
   const configSnapshot = await localCache.getCurrentSnapshot();
@@ -86,14 +72,16 @@ koa.use(
       }
 
       return true;
-    },
+    }
   })
 );
 
 koa.use(async (ctx, next) => {
-  let roles: string[] = [];
+  const permissionSets: PermissionSet[] = [];
 
   if (ctx.state.user && ctx.state.user.username) {
+    const allPermissions = localCache.getPermissions(ctx.state.configSnapshot);
+
     let user;
     if (ctx.state.user.authMethod === "local") {
       user = localCache.getUsers(ctx.state.configSnapshot)[
@@ -102,18 +90,17 @@ koa.use(async (ctx, next) => {
     } else {
       throw new Error("Invalid auth method");
     }
-    roles = user.roles || [];
+
+    for (const role of (user.roles || []).sort())
+      permissionSets.push(Object.values(allPermissions[role] || {}));
   }
 
-  ctx.state.authorizer = getAuthorizer(
-    ctx.state.configSnapshot,
-    JSON.stringify(roles)
-  );
+  ctx.state.authorizer = new Authorizer(permissionSets);
 
   return next();
 });
 
-router.post("/login", async (ctx) => {
+router.post("/login", async ctx => {
   if (!JWT_SECRET) {
     ctx.status = 500;
     ctx.body = "UI_JWT_SECRET is not set";
@@ -128,7 +115,7 @@ router.post("/login", async (ctx) => {
     message: "Log in",
     context: ctx,
     username: username,
-    method: null,
+    method: null
   };
 
   function success(authMethod): void {
@@ -152,13 +139,13 @@ router.post("/login", async (ctx) => {
   failure();
 });
 
-router.post("/logout", async (ctx) => {
+router.post("/logout", async ctx => {
   ctx.cookies.set(JWT_COOKIE); // Delete cookie
   ctx.body = "";
 
   logger.accessInfo({
     message: "Log out",
-    context: ctx,
+    context: ctx
   });
 });
 
@@ -172,11 +159,11 @@ koa.use(async (ctx, next) => {
 koa.use(koaBodyParser());
 router.use("/api", api.routes(), api.allowedMethods());
 
-router.get("/status", (ctx) => {
+router.get("/status", ctx => {
   ctx.body = "OK";
 });
 
-router.get("/init", async (ctx) => {
+router.get("/init", async ctx => {
   const status = await init.getStatus();
   if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
     if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
@@ -197,7 +184,7 @@ router.get("/init", async (ctx) => {
   ctx.body = status;
 });
 
-router.post("/init", async (ctx) => {
+router.post("/init", async ctx => {
   const status = ctx.request.body;
   if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
     if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
@@ -218,8 +205,8 @@ router.post("/init", async (ctx) => {
   ctx.body = "";
 });
 
-router.get("/", async (ctx) => {
-  const permissionSets: PermissionSet[] = ctx.state.authorizer.getPermissionSets();
+router.get("/", async ctx => {
+  const permissionSets = ctx.state.authorizer.getPermissionSets();
 
   let wizard = "";
   if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
@@ -236,7 +223,7 @@ router.get("/", async (ctx) => {
     <noscript>GenieACS UI requires JavaScript to work. Please enable JavaScript in your browser.</noscript>
       <script>
         window.clientConfig = ${JSON.stringify({
-          ui: localCache.getUiConfig(ctx.state.configSnapshot),
+          ui: localCache.getUiConfig(ctx.state.configSnapshot)
         })};
         window.configSnapshot = ${JSON.stringify(ctx.state.configSnapshot)};
         window.genieacsVersion = ${JSON.stringify(VERSION)};
@@ -253,12 +240,7 @@ router.get("/", async (ctx) => {
 
 koa.use(
   koaCompress({
-    gzip: {
-      flush: constants.Z_SYNC_FLUSH,
-    },
-    deflate: {
-      flush: constants.Z_SYNC_FLUSH,
-    },
+    flush: Z_SYNC_FLUSH
   })
 );
 

lib/ui/api-functions.ts

@@ -24,12 +24,10 @@ import { getUsers } from "../local-cache";
 import { hashPassword } from "../auth";
 import {
   insertTasks,
-  awaitSessionEnd,
-  awaitSessionStart,
+  watchTask,
   connectionRequest,
-  deleteDevice,
+  deleteDevice
 } from "../api-functions";
-import { Task } from "../types";
 
 async function deleteFault(id): Promise<void> {
   const deviceId = id.split(":", 1)[0];
@@ -39,16 +37,13 @@ async function deleteFault(id): Promise<void> {
     db.deleteFault(id),
     channel.startsWith("task_")
       ? db.deleteTask(new ObjectID(channel.slice(5)))
-      : null,
+      : null
   ]);
 
   await del(`${deviceId}_tasks_faults_operations`);
 }
 
-export async function deleteResource(
-  resource: string,
-  id: string
-): Promise<void> {
+export async function deleteResource(resource, id): Promise<void> {
   switch (resource) {
     case "devices":
       await deleteDevice(id);
@@ -58,10 +53,6 @@ export async function deleteResource(
       await db.deleteFile(id);
       break;
 
-    case "uploads":
-      await db.deleteUpload(id);
-      break;
-
     case "faults":
       await deleteFault(id);
       break;
@@ -95,59 +86,35 @@ export async function deleteResource(
 }
 
 export async function postTasks(
-  deviceId: string,
-  tasks: Task[],
-  timeout: number,
-  device: Record<string, { value?: [boolean | number | string, string] }>
+  deviceId,
+  tasks,
+  timeout
 ): Promise<{ connectionRequest: string; tasks: any[] }> {
   for (const task of tasks) {
     delete task._id;
-    task["device"] = deviceId;
+    task.device = deviceId;
   }
 
   tasks = await insertTasks(tasks);
-  const statuses = tasks.map((t) => {
+  const statuses = tasks.map(t => {
     return { _id: t._id, status: "pending" };
   });
-  const lastInform = Date.now();
-  const notInSession = await awaitSessionEnd(deviceId, 30000);
-  if (!notInSession) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
 
   await del(`${deviceId}_tasks_faults_operations`);
 
-  const status = await connectionRequest(deviceId, device);
-
-  if (status) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+  try {
+    await connectionRequest(deviceId);
+  } catch (err) {
     return {
-      connectionRequest: status,
-      tasks: statuses,
+      connectionRequest: err.message,
+      tasks: statuses
     };
   }
 
-  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
-  if (!sessionStarted) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    return {
-      connectionRequest: "No contact from CPE",
-      tasks: statuses,
-    };
-  }
+  const sample = tasks[tasks.length - 1];
 
-  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
-  if (!sessionEnded) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
+  // Waiting for session to finish or timeout
+  await watchTask(deviceId, sample._id, timeout);
 
   const promises = [];
   for (const s of statuses) {
@@ -163,20 +130,25 @@ export async function postTasks(
       r.status = "done";
     } else if (res[i * 2 + 1].length === 1) {
       r.status = "fault";
-      r["fault"] = res[i * 2 + 1][0];
+      r.fault = res[i * 2 + 1][0];
     }
+    db.deleteTask(r._id);
   }
 
-  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-
   return { connectionRequest: "OK", tasks: statuses };
 }
 
-export async function putResource(
-  resource: string,
-  id: string,
-  data: Record<string, unknown>
-): Promise<void> {
+interface PingResponse {
+  packetsTransmitted: number;
+  packetsReceived: number;
+  packetLoss: string;
+  min: number;
+  avg: number;
+  max: number;
+  mdev: number;
+}
+
+export async function putResource(resource, id, data): Promise<void> {
   if (resource === "presets") {
     await db.putPreset(id, data);
   } else if (resource === "provisions") {
@@ -188,25 +160,21 @@ export async function putResource(
   } else if (resource === "permissions") {
     await db.putPermission(id, data);
   } else if (resource === "users") {
-    delete data["password"];
-    delete data["salt"];
+    delete data.password;
+    delete data.salt;
     await db.putUser(id, data);
   }
 
   await del("presets_hash");
 }
 
-export function authLocal(
-  snapshot: string,
-  username: string,
-  password: string
-): Promise<boolean> {
+export function authLocal(snapshot, username, password): Promise<boolean> {
   return new Promise((resolve, reject) => {
     const users = getUsers(snapshot);
     const user = users[username];
     if (!user || !user.password) return void resolve(null);
     hashPassword(password, user.salt)
-      .then((hash) => {
+      .then(hash => {
         if (hash === user.password) resolve(true);
         else resolve(false);
       })

lib/ui/api.ts

@@ -30,8 +30,6 @@ import { generateSalt, hashPassword } from "../auth";
 import { del } from "../cache";
 import Authorizer from "../common/authorizer";
 import { ping } from "../ping";
-import * as url from "url";
-import { decodeTag } from "../common";
 
 const router = new Router();
 export default router;
@@ -49,13 +47,12 @@ const RESOURCE_IDS = {
   presets: "_id",
   provisions: "_id",
   files: "_id",
-  uploads: "_id",
   virtualParameters: "_id",
   config: "_id",
   permissions: "_id",
   users: "_id",
   faults: "_id",
-  tasks: "_id",
+  tasks: "_id"
 };
 
 const resources = {
@@ -63,32 +60,31 @@ const resources = {
   presets: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   provisions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   files: 0 | RESOURCE_DELETE,
-  uploads: 0 | RESOURCE_DELETE,
   virtualParameters: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   config: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   permissions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   users: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   faults: 0 | RESOURCE_DELETE,
-  tasks: 0,
+  tasks: 0
 };
 
-router.get(`/devices/:id.csv`, async (ctx) => {
+router.get(`/devices/:id.csv`, async ctx => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Query device (CSV)",
     context: ctx,
-    id: ctx.params.id,
+    id: ctx.params.id
   };
 
   const filter = and(authorizer.getFilter("devices", 2), [
     "=",
     ["PARAM", RESOURCE_IDS.devices],
-    ctx.params.id,
+    ctx.params.id
   ]);
 
   if (!authorizer.hasAccess("devices", 2)) {
     logUnauthorizedWarning(log);
-    return void (ctx.status = 403);
+    return void (ctx.status = 404);
   }
 
   const res = await db.query("devices", filter);
@@ -120,16 +116,16 @@ router.get(`/devices/:id.csv`, async (ctx) => {
       p.notification,
       p.notificationTimestamp,
       p.accessList ? p.accessList.join(", ") : "",
-      p.accessListTimestamp,
+      p.accessListTimestamp
     ];
-    ctx.body.write(row.map((r) => (r != null ? r : "")).join(",") + "\n");
+    ctx.body.write(row.map(r => (r != null ? r : "")).join(",") + "\n");
   }
   ctx.body.end();
   logger.accessInfo(log);
 });
 
 for (const [resource, flags] of Object.entries(resources)) {
-  router.head(`/${resource}`, async (ctx) => {
+  router.head(`/${resource}`, async (ctx, next) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     let filter: Expression = authorizer.getFilter(resource, 1);
     if (ctx.request.query.filter)
@@ -139,20 +135,19 @@ for (const [resource, flags] of Object.entries(resources)) {
       message: `Count ${resource}`,
       context: ctx,
       filter: ctx.request.query.filter,
-      count: null,
+      count: null
     };
 
     if (!authorizer.hasAccess(resource, 1)) {
       logUnauthorizedWarning(log);
-      return void (ctx.status = 403);
+      return void next();
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "OR",
-        [">=", ["PARAM", "expiry"], Date.now() + 60000],
-        ["IS NULL", ["PARAM", "expiry"]],
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
       ]);
     }
 
@@ -165,7 +160,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.get(`/${resource}`, async (ctx) => {
+  router.get(`/${resource}`, async (ctx, next) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = {};
     let filter: Expression = authorizer.getFilter(resource, 2);
@@ -188,20 +183,19 @@ for (const [resource, flags] of Object.entries(resources)) {
       limit: options.limit,
       skip: options.skip,
       sort: options.sort,
-      projection: options.projection,
+      projection: options.projection
     };
 
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void (ctx.status = 403);
+      return void next();
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "OR",
-        [">=", ["PARAM", "expiry"], Date.now() + 60000],
-        ["IS NULL", ["PARAM", "expiry"]],
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
       ]);
     }
 
@@ -210,7 +204,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     let c = 0;
     ctx.body.write("[\n");
-    await db.query(resource, filter, options, (obj) => {
+    await db.query(resource, filter, options, obj => {
       ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
     });
     ctx.body.end("]");
@@ -219,7 +213,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   });
 
   // CSV download
-  router.get(`/${resource}.csv`, async (ctx) => {
+  router.get(`/${resource}.csv`, async (ctx, next) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = { projection: {} };
     let filter: Expression = authorizer.getFilter(resource, 2);
@@ -236,17 +230,15 @@ for (const [resource, flags] of Object.entries(resources)) {
       filter: ctx.request.query.filter,
       limit: options.limit,
       skip: options.skip,
-      sort: options.sort,
+      sort: options.sort
     };
 
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void (ctx.status = 403);
+      return void next();
     }
 
-    const columns: Record<string, Expression> = JSON.parse(
-      ctx.request.query.columns
-    );
+    const columns = JSON.parse(ctx.request.query.columns);
     const now = Date.now();
 
     for (const [k, v] of Object.entries(columns)) {
@@ -259,9 +251,8 @@ for (const [resource, flags] of Object.entries(resources)) {
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "OR",
-        [">=", ["PARAM", "expiry"], Date.now() + 60000],
-        ["IS NULL", ["PARAM", "expiry"]],
+        "NOT",
+        ["<", ["PARAM", "expiry"], Date.now() + 60000]
       ]);
     }
 
@@ -272,18 +263,18 @@ for (const [resource, flags] of Object.entries(resources)) {
     );
 
     ctx.body.write(
-      Object.keys(columns).map((k) => `"${k.replace(/"/, '""')}"`) + "\n"
+      Object.keys(columns).map(k => `"${k.replace(/"/, '""')}"`) + "\n"
     );
-    await db.query(resource, filter, options, (obj) => {
-      const arr = Object.values(columns).map((exp) => {
-        const v = evaluate(exp, obj, null, (e) => {
+    await db.query(resource, filter, options, obj => {
+      const arr = Object.values(columns).map(exp => {
+        const v = evaluate(exp, obj, null, e => {
           if (Array.isArray(e)) {
             if (e[0] === "PARAM") {
               if (resource === "devices") {
                 if (e[1] === "Tags") {
                   const tags = [];
                   for (const p in obj)
-                    if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
+                    if (p.startsWith("Tags.")) tags.push(p.slice(5));
 
                   return tags.join(", ");
                 }
@@ -310,78 +301,78 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.head(`/${resource}/:id`, async (ctx) => {
+  router.head(`/${resource}/:id`, async (ctx, next) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Count ${resource}`,
       context: ctx,
-      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`,
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
     };
 
     const filter = and(authorizer.getFilter(resource, 2), [
       "=",
       ["PARAM", RESOURCE_IDS[resource]],
-      ctx.params.id,
+      ctx.params.id
     ]);
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void (ctx.status = 403);
+      return void next();
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void (ctx.status = 404);
+    if (!res.length) return void next();
 
     logger.accessInfo(log);
     ctx.body = "";
   });
 
-  router.get(`/${resource}/:id`, async (ctx) => {
+  router.get(`/${resource}/:id`, async (ctx, next) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Query ${resource}`,
       context: ctx,
-      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`,
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
     };
 
     const filter = and(authorizer.getFilter(resource, 2), [
       "=",
       ["PARAM", RESOURCE_IDS[resource]],
-      ctx.params.id,
+      ctx.params.id
     ]);
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void (ctx.status = 403);
+      return void next();
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void (ctx.status = 404);
+    if (!res.length) return void next();
 
     logger.accessInfo(log);
     ctx.body = res[0];
   });
 
   if (flags & RESOURCE_DELETE) {
-    router.delete(`/${resource}/:id(.+)`, async (ctx) => {
+    router.delete(`/${resource}/:id`, async (ctx, next) => {
       const authorizer: Authorizer = ctx.state.authorizer;
       const log = {
         message: `Delete ${resource}`,
         context: ctx,
-        id: ctx.params.id,
+        id: ctx.params.id
       };
 
       const filter = and(authorizer.getFilter(resource, 3), [
         "=",
         ["PARAM", RESOURCE_IDS[resource]],
-        ctx.params.id,
+        ctx.params.id
       ]);
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void (ctx.status = 403);
+        return void next();
       }
       const res = await db.query(resource, filter);
-      if (!res.length) return void (ctx.status = 404);
+      if (!res.length) return void next();
 
       const validate = authorizer.getValidator(resource, res[0]);
       if (!validate("delete")) {
@@ -398,19 +389,19 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 
   if (flags & RESOURCE_PUT) {
-    router.put(`/${resource}/:id`, async (ctx) => {
+    router.put(`/${resource}/:id`, async (ctx, next) => {
       const authorizer: Authorizer = ctx.state.authorizer;
       const id = ctx.params.id;
 
       const log = {
         message: `Put ${resource}`,
         context: ctx,
-        id: id,
+        id: id
       };
 
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void (ctx.status = 403);
+        return void next();
       }
 
       const obj = ctx.request.body;
@@ -437,7 +428,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 }
 
-router.put("/files/:id", async (ctx) => {
+router.put("/files/:id", async (ctx, next) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const resource = "files";
   const id = ctx.params.id;
@@ -446,19 +437,19 @@ router.put("/files/:id", async (ctx) => {
     message: `Upload ${resource}`,
     context: ctx,
     id: id,
-    metadata: null,
+    metadata: null
   };
 
   if (!authorizer.hasAccess(resource, 3)) {
     logUnauthorizedWarning(log);
-    return void (ctx.status = 403);
+    return void next();
   }
 
   const metadata = {
     fileType: ctx.request.headers["metadata.filetype"] || "",
     oui: ctx.request.headers["metadata.oui"] || "",
     productClass: ctx.request.headers["metadata.productclass"] || "",
-    version: ctx.request.headers["metadata.version"] || "",
+    version: ctx.request.headers["metadata.version"] || ""
   };
 
   const validate = authorizer.getValidator(resource, metadata);
@@ -467,12 +458,6 @@ router.put("/files/:id", async (ctx) => {
     return void (ctx.status = 403);
   }
 
-  try {
-    await db.deleteFile(id);
-  } catch (err) {
-    // File doesn't exist, ignore
-  }
-
   await db.putFile(id, metadata, ctx.req);
   log.metadata = metadata;
   logger.accessInfo(log);
@@ -480,64 +465,28 @@ router.put("/files/:id", async (ctx) => {
   ctx.body = "";
 });
 
-router.get("/uploads/blob/:id(.+)", async (ctx) => {
-  const authorizer: Authorizer = ctx.state.authorizer;
-  const resource = "uploads";
-  const id = ctx.params.id;
-
-  const log = {
-    message: `Download ${resource} blob`,
-    context: ctx,
-    id: id,
-  };
-  1;
-
-  const filter = and(authorizer.getFilter(resource, 2), [
-    "=",
-    ["PARAM", RESOURCE_IDS[resource]],
-    id,
-  ]);
-
-  if (!authorizer.hasAccess(resource, 2)) {
-    logUnauthorizedWarning(log);
-    return void (ctx.status = 403);
-  }
-
-  const res = await db.query(resource, filter);
-  if (!res[0]) return void (ctx.status = 404);
-
-  ctx.type = "application/octet-stream";
-  ctx.attachment(id.replace(/\//g, "_"));
-  ctx.body = new stream.PassThrough();
-
-  const downloadStream = await db.getUploadBlob(id);
-  downloadStream.pipe(ctx.body);
-  logger.accessInfo(log);
-});
-
-router.post("/devices/:id/tasks", async (ctx) => {
+router.post("/devices/:id/tasks", async (ctx, next) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Commit tasks",
     context: ctx,
     deviceId: ctx.params.id,
-    tasks: null,
+    tasks: null
   };
 
   const filter = and(authorizer.getFilter("devices", 3), [
     "=",
     ["PARAM", "DeviceID.ID"],
-    ctx.params.id,
+    ctx.params.id
   ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void (ctx.status = 403);
+    return void next();
   }
   const devices = await db.query("devices", filter);
-  if (!devices.length) return void (ctx.status = 404);
-  const device = devices[0];
+  if (!devices.length) return void next();
 
-  const validate = authorizer.getValidator("devices", device);
+  const validate = authorizer.getValidator("devices", devices[0]);
   for (const t of ctx.request.body) {
     if (!validate("task", t)) {
       logUnauthorizedWarning(log);
@@ -545,45 +494,19 @@ router.post("/devices/:id/tasks", async (ctx) => {
     }
   }
 
+  // TODO Pass context and timestamp
   const onlineThreshold = getConfig(
     ctx.state.configSnapshot,
-    "cwmp.deviceOnlineThreshold",
-    {},
-    Date.now(),
-    (exp) => {
-      if (!Array.isArray(exp)) return exp;
-      if (exp[0] === "PARAM") {
-        const p = device[exp[1]];
-        if (p && p.value) return p.value[0];
-      } else if (exp[0] === "FUNC") {
-        if (exp[1] === "REMOTE_ADDRESS") {
-          for (const root of ["InternetGatewayDevice", "Device"]) {
-            const p = device[`${root}.ManagementServer.ConnectionRequestURL`];
-            if (p && p.value) return url.parse(p.value[0]).host;
-          }
-          return null;
-        }
-      }
-      return exp;
-    }
-  ) as number;
-
-  const socketTimeout: number = ctx.socket["timeout"];
-
-  // Disable socket timeout while waiting for postTasks()
-  if (socketTimeout) ctx.socket.setTimeout(0);
+    "cwmp.deviceOnlineThreshold"
+  );
 
   const res = await apiFunctions.postTasks(
     ctx.params.id,
     ctx.request.body,
-    onlineThreshold,
-    device
+    onlineThreshold
   );
 
-  // Restore socket timeout
-  if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
-
-  log.tasks = res.tasks.map((t) => t._id).join(",");
+  log.tasks = res.tasks.map(t => t._id).join(",");
 
   logger.accessInfo(log);
 
@@ -591,26 +514,26 @@ router.post("/devices/:id/tasks", async (ctx) => {
   ctx.body = res.tasks;
 });
 
-router.post("/devices/:id/tags", async (ctx) => {
+router.post("/devices/:id/tags", async (ctx, next) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Update tags",
     context: ctx,
     deviceId: ctx.params.id,
-    tags: ctx.request.body,
+    tags: ctx.request.body
   };
 
   const filter = and(authorizer.getFilter("devices", 3), [
     "=",
     ["PARAM", "DeviceID.ID"],
-    ctx.params.id,
+    ctx.params.id
   ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void (ctx.status = 403);
+    return void next();
   }
   const res = await db.query("devices", filter);
-  if (!res.length) return void (ctx.status = 404);
+  if (!res.length) return void next();
 
   const validate = authorizer.getValidator("devices", res[0]);
   if (!validate("tags", ctx.request.body)) {
@@ -632,8 +555,8 @@ router.post("/devices/:id/tags", async (ctx) => {
   ctx.body = "";
 });
 
-router.get("/ping/:host", async (ctx) => {
-  return new Promise((resolve) => {
+router.get("/ping/:host", async ctx => {
+  return new Promise(resolve => {
     ping(ctx.params.host, (err, parsed) => {
       if (parsed) {
         ctx.body = parsed;
@@ -646,13 +569,13 @@ router.get("/ping/:host", async (ctx) => {
   });
 });
 
-router.put("/users/:id/password", async (ctx) => {
+router.put("/users/:id/password", async (ctx, next) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const username = ctx.params.id;
   const log = {
     message: "Change password",
     context: ctx,
-    username: username,
+    username: username
   };
 
   if (!ctx.state.user) {
@@ -671,16 +594,16 @@ router.put("/users/:id/password", async (ctx) => {
     }
   } else if (!authorizer.hasAccess("users", 3)) {
     logUnauthorizedWarning(log);
-    return void (ctx.status = 403);
+    return void next();
   }
 
   const filter = and(authorizer.getFilter("users", 3), [
     "=",
     ["PARAM", RESOURCE_IDS.users],
-    username,
+    username
   ]);
   const res = await db.query("users", filter);
-  if (!res.length) return void (ctx.status = 404);
+  if (!res.length) return void next();
 
   const newPassword = ctx.request.body.newPassword;
   if (ctx.state.user) {

lib/ui/db.ts

@@ -17,36 +17,89 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Db, GridFSBucket, ObjectID } from "mongodb";
+import { MongoClient, GridFSBucket } from "mongodb";
 import { Script } from "vm";
-import { onConnect } from "../db";
+import * as config from "../config";
 import * as mongodbFunctions from "../mongodb-functions";
 import * as expression from "../common/expression";
-import { QueryOptions, Expression } from "../types";
-import { Readable } from "stream";
-import { minimize } from "../common/boolean-expression";
+import { QueryOptions } from "../types";
+
+const CACHE_TTL = 300000;
+
+let clientPromise: Promise<MongoClient>;
 
 const RESOURCE_COLLECTION = {
-  files: "fs.files",
-  uploads: "uploads.files",
+  files: "fs.files"
 };
 
-let db: Db;
+function ensureIndexes(client): void {
+  client
+    .db()
+    .collection("cache")
+    .createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+}
 
-onConnect(async (_db) => {
-  db = _db;
+function getClient(): Promise<MongoClient> {
+  if (!clientPromise) {
+    clientPromise = new Promise((resolve, reject) => {
+      const CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
+      MongoClient.connect(
+        CONNECTION_URL,
+        { useNewUrlParser: true },
+        (err, client) => {
+          if (err) return void reject(err);
+          ensureIndexes(client);
+          resolve(client);
+        }
+      );
     });
+  }
+
+  return clientPromise;
+}
+
+export function cache<T>(key, valueGetter: () => Promise<T>, ttl): Promise<T> {
+  return new Promise((resolve, reject) => {
+    getClient()
+      .then(client => {
+        const collection = client.db().collection("cache");
+        collection.findOne({ _id: key }, (err, doc) => {
+          if (err) return void reject(err);
+          if (doc != null) return void resolve(JSON.parse(doc.value) as T);
+          valueGetter()
+            .then(res => {
+              const expire = Date.now() + (ttl || CACHE_TTL);
+              const cacheDoc = {
+                _id: key,
+                value: JSON.stringify(res),
+                expire: new Date(expire)
+              };
+              collection.updateOne(
+                { _id: key },
+                { $set: cacheDoc },
+                { upsert: true },
+                err => {
+                  if (err) reject(err);
+                  else resolve(res);
+                }
+              );
+            })
+            .catch(reject);
+        });
+      })
+      .catch(reject);
+  });
+}
 
 export function query(
-  resource: string,
-  filter: Expression,
+  resource,
+  filter,
   options?: QueryOptions,
   callback?: (doc: any) => void
 ): Promise<any[]> {
   options = options || {};
   let q;
   filter = expression.evaluate(filter, null, Date.now());
-  filter = minimize(filter, true);
 
   if (Array.isArray(filter)) {
     if (resource === "devices") {
@@ -63,12 +116,15 @@ export function query(
     }
 
     q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (!filter) {
+  } else if (filter != null && !filter) {
     return Promise.resolve([]);
   }
 
   return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
+    getClient().then(client => {
+      const collection = client
+        .db()
+        .collection(RESOURCE_COLLECTION[resource] || resource);
       const cursor = collection.find(q);
       if (options.projection) {
         let projection = options.projection;
@@ -104,23 +160,22 @@ export function query(
         cursor.toArray((err, docs) => {
           if (err) return reject(err);
           if (resource === "devices")
-          docs = docs.map((d) => mongodbFunctions.flattenDevice(d));
+            docs = docs.map(d => mongodbFunctions.flattenDevice(d));
           else if (resource === "faults")
-          docs = docs.map((d) => mongodbFunctions.flattenFault(d));
+            docs = docs.map(d => mongodbFunctions.flattenFault(d));
           else if (resource === "tasks")
-          docs = docs.map((d) => mongodbFunctions.flattenTask(d));
+            docs = docs.map(d => mongodbFunctions.flattenTask(d));
           else if (resource === "presets")
-          docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
+            docs = docs.map(d => mongodbFunctions.flattenPreset(d));
           else if (resource === "files")
-          docs = docs.map((d) => mongodbFunctions.flattenFile(d));
-        else if (resource === "uploads")
-          docs = docs.map((d) => mongodbFunctions.flattenUpload(d));
+            docs = docs.map(d => mongodbFunctions.flattenFile(d));
           return resolve(docs);
         });
       } else {
         cursor.forEach(
-        (doc) => {
-          if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc);
+          doc => {
+            if (resource === "devices")
+              doc = mongodbFunctions.flattenDevice(doc);
             else if (resource === "faults")
               doc = mongodbFunctions.flattenFault(doc);
             else if (resource === "tasks")
@@ -129,23 +184,21 @@ export function query(
               doc = mongodbFunctions.flattenPreset(doc);
             else if (resource === "files")
               doc = mongodbFunctions.flattenFile(doc);
-          else if (resource === "uploads")
-            doc = mongodbFunctions.flattenUpload(doc);
             callback(doc);
           },
-        (err) => {
+          err => {
             if (err) reject(err);
             else resolve();
           }
         );
       }
     });
+  });
 }
 
-export function count(resource: string, filter: Expression): Promise<number> {
+export function count(resource, filter): Promise<number> {
   let q;
   filter = expression.evaluate(filter, null, Date.now());
-  filter = minimize(filter, true);
 
   if (Array.isArray(filter)) {
     if (resource === "devices")
@@ -155,80 +208,97 @@ export function count(resource: string, filter: Expression): Promise<number> {
     else if (resource === "faults")
       filter = mongodbFunctions.processFaultsFilter(filter);
     q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (!filter) {
+  } else if (filter != null && !filter) {
     return Promise.resolve(0);
   }
 
   return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
+    getClient().then(client => {
+      const collection = client
+        .db()
+        .collection(RESOURCE_COLLECTION[resource] || resource);
       collection.find(q).count((err, c) => {
         if (err) reject(err);
         else resolve(c);
       });
     });
+  });
 }
 
-export async function updateDeviceTags(
-  deviceId: string,
-  tags: Record<string, boolean>
-): Promise<void> {
+export function updateDeviceTags(deviceId, tags): Promise<void> {
+  return new Promise((resolve, reject) => {
     const add = [];
     const pull = [];
 
+    const regex = /^[0-9a-zA-Z_]+$/;
     for (let [tag, onOff] of Object.entries(tags)) {
       tag = tag.trim();
-    if (onOff) add.push(tag);
-    else pull.push(tag);
+      if (onOff) {
+        if (!tag.match(regex))
+          return void reject(new Error(`Invalid tag '${tag}'`));
+        add.push(tag);
+      } else {
+        pull.push(tag);
       }
+    }
+    getClient()
+      .then(client => {
+        const collection = client.db().collection("devices");
 
-  const collection = db.collection("devices");
         const object = {};
-
         if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
         if (pull && pull.length) object["$pullAll"] = { _tags: pull };
-
-  await collection.updateOne({ _id: deviceId }, object);
+        collection.updateOne({ _id: deviceId }, object, err => {
+          if (err) return void reject(err);
+          resolve();
+        });
+      })
+      .catch(reject);
+  });
 }
 
 function putResource(resource, id, object): Promise<void> {
   return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-    collection.replaceOne({ _id: id }, object, { upsert: true }, (err) => {
+    getClient()
+      .then(client => {
+        const collection = client
+          .db()
+          .collection(RESOURCE_COLLECTION[resource] || resource);
+        collection.replaceOne({ _id: id }, object, { upsert: true }, err => {
           if (err) return void reject(err);
           resolve();
         });
+      })
+      .catch(reject);
   });
 }
 
-function deleteResource(
-  resource: string,
-  id: string | ObjectID
-): Promise<void> {
+function deleteResource(resource, id): Promise<void> {
   return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-    collection.deleteOne({ _id: id }, (err) => {
+    getClient()
+      .then(client => {
+        const collection = client
+          .db()
+          .collection(RESOURCE_COLLECTION[resource] || resource);
+        collection.deleteOne({ _id: id }, err => {
           if (err) return void reject(err);
           resolve();
         });
+      })
+      .catch(reject);
   });
 }
 
-export function putPreset(
-  id: string,
-  object: Record<string, unknown>
-): Promise<void> {
+export function putPreset(id, object): Promise<void> {
   object = mongodbFunctions.preProcessPreset(object);
   return putResource("presets", id, object);
 }
 
-export function deletePreset(id: string): Promise<void> {
+export function deletePreset(id): Promise<void> {
   return deleteResource("presets", id);
 }
 
-export function putProvision(
-  id: string,
-  object: Record<string, unknown>
-): Promise<void> {
+export function putProvision(id, object): Promise<void> {
   if (!object.script) object.script = "";
   try {
     new Script(`"use strict";(function(){\n${object.script}\n})();`);
@@ -238,14 +308,11 @@ export function putProvision(
   return putResource("provisions", id, object);
 }
 
-export function deleteProvision(id: string): Promise<void> {
+export function deleteProvision(id): Promise<void> {
   return deleteResource("provisions", id);
 }
 
-export function putVirtualParameter(
-  id: string,
-  object: Record<string, unknown>
-): Promise<void> {
+export function putVirtualParameter(id, object): Promise<void> {
   if (!object.script) object.script = "";
   try {
     new Script(`"use strict";(function(){\n${object.script}\n})();`);
@@ -255,96 +322,89 @@ export function putVirtualParameter(
   return putResource("virtualParameters", id, object);
 }
 
-export function deleteVirtualParameter(id: string): Promise<void> {
+export function deleteVirtualParameter(id): Promise<void> {
   return deleteResource("virtualParameters", id);
 }
 
-export function putConfig(
-  id: string,
-  object: Record<string, unknown>
-): Promise<void> {
+export function putConfig(id, object): Promise<void> {
   return putResource("config", id, object);
 }
 
-export function deleteConfig(id: string): Promise<void> {
+export function deleteConfig(id): Promise<void> {
   return deleteResource("config", id);
 }
 
-export function putPermission(
-  id: string,
-  object: Record<string, unknown>
-): Promise<void> {
+export function putPermission(id, object): Promise<void> {
   return putResource("permissions", id, object);
 }
 
-export function deletePermission(id: string): Promise<void> {
+export function deletePermission(id): Promise<void> {
   return deleteResource("permissions", id);
 }
 
-export function putUser(
-  id: string,
-  object: Record<string, unknown>
-): Promise<void> {
+export function putUser(id, object): Promise<void> {
   return new Promise((resolve, reject) => {
-    const collection = db.collection("users");
+    getClient()
+      .then(client => {
+        const collection = client.db().collection("users");
         // update instead of replace to keep the password if not set by user
         collection.updateOne(
           { _id: id },
           { $set: object },
           { upsert: true },
-      (err) => {
+          err => {
             if (err) return void reject(err);
             resolve();
           }
         );
+      })
+      .catch(reject);
   });
 }
 
-export function deleteUser(id: string): Promise<void> {
+export function deleteUser(id): Promise<void> {
   return deleteResource("users", id);
 }
 
-export function putFile(
-  filename: string,
-  metadata: Record<string, string>,
-  contentStream: Readable
-): Promise<void> {
+export function putFile(filename, metadata, contentStream): Promise<void> {
   return new Promise((resolve, reject) => {
-    const bucket = new GridFSBucket(db);
+    getClient()
+      .then(client => {
+        const bucket = new GridFSBucket(client.db());
         const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
-      metadata: metadata,
+          metadata: metadata
         });
         uploadStream.on("error", reject);
         contentStream.on("error", reject);
         uploadStream.on("finish", resolve);
         contentStream.pipe(uploadStream);
+      })
+      .catch(reject);
   });
 }
 
-export function deleteFile(filename: string): Promise<void> {
+export function deleteFile(filename): Promise<void> {
   return new Promise((resolve, reject) => {
-    const bucket = new GridFSBucket(db);
-    bucket.delete(filename as any, (err) => {
+    getClient()
+      .then(client => {
+        const bucket = new GridFSBucket(client.db());
+        bucket.delete(filename, err => {
           if (err) return void reject(err);
           resolve();
         });
+      })
+      .catch(reject);
   });
 }
 
-export async function deleteUpload(filename: string): Promise<void> {
-  const bucket = new GridFSBucket(db, { bucketName: "uploads" });
-  await bucket.delete(filename as any);
-}
-
-export async function getUploadBlob(filename: string): Promise<Readable> {
-  const bucket = new GridFSBucket(db, { bucketName: "uploads" });
-  return bucket.openDownloadStreamByName(filename);
-}
-
-export function deleteFault(id: string): Promise<void> {
+export function deleteFault(id): Promise<void> {
   return deleteResource("faults", id);
 }
 
-export function deleteTask(id: ObjectID): Promise<void> {
+export function deleteTask(id): Promise<void> {
   return deleteResource("tasks", id);
 }
+
+export async function disconnect(): Promise<void> {
+  if (clientPromise) await (await clientPromise).close();
+}

lib/versioned-map.ts

@@ -46,7 +46,7 @@ export default class VersionedMap<K, V> {
     return this._revision;
   }
 
-  public set revision(rev: number) {
+  public set revision(rev) {
     for (let i = this._sizeDiff.length; i <= rev; ++i)
       this._sizeDiff[i] = this._sizeDiff[i - 1];
 
@@ -193,7 +193,7 @@ export default class VersionedMap<K, V> {
       const last = v[l];
       v.splice(revision, l - revision);
 
-      if (last === NONEXISTENT && !v.some((vv) => vv !== NONEXISTENT))
+      if (last === NONEXISTENT && !v.some(vv => vv !== NONEXISTENT))
         this.map.delete(k);
     }
   }

lib/xml-parser.ts

@@ -53,13 +53,12 @@ export interface Element {
 }
 
 export function parseXmlDeclaration(buffer: Buffer): Attribute[] {
-  const encodings: BufferEncoding[] = ["utf16le", "utf8", "latin1", "ascii"];
-  for (const enc of encodings) {
+  for (const enc of ["utf16le", "utf8", "latin1", "ascii"]) {
     let str = buffer.toString(enc, 0, 150);
     if (str.startsWith("<?xml")) {
-      str = str.slice(0, str.indexOf("?>"));
+      str = str.split("\n")[0].trim();
       try {
-        return parseAttrs(str.slice(5));
+        return parseAttrs(str.slice(5, -2));
       } catch (err) {
         // Ignore
       }
@@ -91,7 +90,7 @@ export function parseAttrs(string: string): Attribute[] {
               name: name,
               namespace: namespace,
               localName: localName,
-              value: value,
+              value: value
             };
             attrs.push(e);
             name = "";
@@ -104,12 +103,10 @@ export function parseAttrs(string: string): Attribute[] {
         continue;
 
       case CHAR_COLON:
-        if (state) continue;
         if (idx >= colonIdx) colonIdx = i;
         continue;
 
       case CHAR_EQUAL:
-        if (state) continue;
         if (name) throw new Error(`Unexpected character at ${i}`);
         name = string.slice(idx, i).trim();
         // TODO validate name
@@ -131,8 +128,8 @@ export function parseAttrs(string: string): Attribute[] {
   return attrs;
 }
 
-export function decodeEntities(string: string): string {
-  return string.replace(/&[0-9a-z#]+;/gi, (match) => {
+export function decodeEntities(string): string {
+  return string.replace(/&[0-9a-z#]+;/gi, match => {
     switch (match) {
       case "&quot;":
         return '"';
@@ -164,15 +161,15 @@ export function decodeEntities(string: string): string {
   });
 }
 
-export function encodeEntities(string: string): string {
+export function encodeEntities(string): string {
   const entities = {
     "&": "&amp;",
     '"': "&quot;",
     "'": "&apos;",
     "<": "&lt;",
-    ">": "&gt;",
+    ">": "&gt;"
   };
-  return string.replace(/[&"'<>]/g, (m) => entities[m]);
+  return string.replace(/[&"'<>]/g, m => entities[m]);
 }
 
 export function parseXml(string: string): Element {
@@ -189,7 +186,7 @@ export function parseXml(string: string): Element {
     attrs: "",
     text: "",
     bodyIndex: 0,
-    children: [],
+    children: []
   };
 
   const stack: Element[] = [root];
@@ -333,7 +330,7 @@ export function parseXml(string: string): Element {
                   : "",
                 text: "",
                 bodyIndex: i + 1,
-                children: [],
+                children: []
               };
               parent.children.push(e);
               if (!selfClosing) stack.push(e);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.3",
+  "version": "1.2.0-beta.0",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -20,69 +20,69 @@
     "tools": "tools"
   },
   "dependencies": {
-    "espresso-iisojs": "^1.0.3",
-    "iconv-lite": "^0.6.2",
-    "ipaddr.js": "^2.0.0",
+    "iconv-lite": "^0.5.0",
     "jsonwebtoken": "^8.5.1",
-    "koa": "^2.13.0",
-    "koa-bodyparser": "^4.3.0",
-    "koa-compress": "^5.0.1",
-    "koa-jwt": "^4.0.0",
-    "koa-router": "^9.4.0",
+    "koa": "^2.7.0",
+    "koa-bodyparser": "^4.2.1",
+    "koa-compress": "^3.0.0",
+    "koa-jwt": "^3.6.0",
+    "koa-router": "^7.4.0",
     "koa-static": "^5.0.0",
     "later": "^1.2.0",
-    "mongodb": "^3.6.2",
-    "parsimmon": "^1.16.0",
-    "seedrandom": "^3.0.5",
-    "yaml": "^1.10.0"
+    "mongodb": "^3.2.6",
+    "parsimmon": "^1.13.0",
+    "seedrandom": "^3.0.1",
+    "yaml": "^1.6.0"
+  },
+  "optionalDependencies": {
+    "libxmljs": "^0.19.5"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^16.0.0",
-    "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^10.0.0",
-    "@rollup/plugin-typescript": "^6.0.0",
-    "@types/jsonwebtoken": "^8.5.0",
-    "@types/koa": "^2.11.6",
-    "@types/koa-compress": "^4.0.1",
-    "@types/koa-router": "^7.4.1",
+    "@types/jsonwebtoken": "^8.3.2",
+    "@types/koa": "^2.0.49",
+    "@types/koa-compress": "^2.0.9",
+    "@types/koa-router": "^7.0.42",
     "@types/koa-static": "^4.0.1",
-    "@types/mithril": "^2.0.5",
-    "@types/mongodb": "^3.5.31",
-    "@types/node": "^14.14.5",
-    "@types/parsimmon": "^1.10.3",
+    "@types/libxmljs": "^0.18.3",
+    "@types/mithril": "^2.0.0",
+    "@types/mongodb": "^3.1.31",
+    "@types/parsimmon": "^1.10.0",
     "@types/seedrandom": "^2.4.28",
-    "@types/svgo": "^1.3.3",
-    "@typescript-eslint/eslint-plugin": "^4.6.0",
-    "@typescript-eslint/parser": "^4.6.0",
-    "ava": "^3.13.0",
-    "codemirror": "^5.58.2",
+    "@types/svgo": "^1.2.1",
+    "@typescript-eslint/eslint-plugin": "^1.13.0",
+    "@typescript-eslint/parser": "^1.13.0",
+    "ava": "^2.2.0",
+    "codemirror": "^5.48.2",
     "cssnano": "^4.1.10",
-    "eslint": "^7.12.1",
-    "eslint-config-prettier": "^6.14.0",
+    "eslint": "^6.1.0",
+    "eslint-config-prettier": "^6.0.0",
     "esm": "^3.2.25",
-    "jsbi": "^3.1.4",
-    "mithril": "^2.0.4",
-    "postcss": "^8.1.4",
-    "postcss-import": "^13.0.0",
+    "mithril": "^2.0.3",
+    "postcss": "^7.0.17",
+    "postcss-import": "^12.0.1",
     "postcss-preset-env": "^6.7.0",
-    "prettier": "^2.1.2",
-    "rollup": "^2.32.1",
-    "rollup-plugin-terser": "^7.0.2",
-    "source-map-loader": "^1.1.1",
-    "svgo": "^1.3.2",
-    "ts-node": "^9.0.0",
-    "typescript": "^4.0.5",
-    "webpack": "^5.2.0"
+    "prettier": "^1.18.2",
+    "rollup": "^1.17.0",
+    "rollup-plugin-json": "^4.0.0",
+    "rollup-plugin-replace": "^2.2.0",
+    "rollup-plugin-terser": "^5.1.1",
+    "rollup-plugin-typescript": "^1.0.1",
+    "source-map-loader": "^0.2.4",
+    "svgo": "^1.3.0",
+    "ts-node": "^8.3.0",
+    "typescript": "^3.5.3",
+    "webpack": "^4.38.0"
   },
   "engines": {
-    "node": ">=12.13.0"
+    "node": ">=8.14.0"
   },
   "scripts": {
     "test": "ava --verbose --color",
-    "lint-eslint": "eslint --fix \"bin/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/*.ts\"",
+    "lint-ln": "find bin -type f -exec sh -c 'ln -fs \"$(sh -c pwd)/$1\" \"bin/.${1#bin/}.ts\"' _ {} \\;; find tools -type f -exec sh -c 'ln -fs \"$(sh -c pwd)/$1\" \"tools/.${1#tools/}.ts\"' _ {} \\;",
+    "lint-eslint": "eslint --fix \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
     "lint-prettier": "prettier --parser typescript --write \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
     "lint-tsc": "tsc --noEmit",
-    "lint": "npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
+    "lint": "npm run lint-ln && npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
     "build": "node -r esm -r ts-node/register/transpile-only build/build"
   }
 }

test/auth.ts

@@ -2,7 +2,7 @@ import ava from "ava";
 import { randomBytes } from "crypto";
 import * as auth from "../lib/auth";
 
-ava("digest", (t) => {
+ava("digest", t => {
   const username = "test";
   const password = "test";
   const uri = "/";
@@ -14,7 +14,7 @@ ava("digest", (t) => {
   const challenges = [
     `Digest realm="${realm}",nonce="${nonce}"`,
     `Digest realm="${realm}",nonce="${nonce}",qop="auth"`,
-    `Digest realm="${realm}",nonce="${nonce}",qop="auth-int"`,
+    `Digest realm="${realm}",nonce="${nonce}",qop="auth-int"`
   ];
 
   t.plan(challenges.length);

test/common.ts

@@ -1,7 +1,35 @@
 import ava from "ava";
 import * as common from "../lib/common";
 
-ava("generateDeviceId", (t) => {
+ava("typeOf", t => {
+  const cases = [
+    [undefined, common.UNDEFINED_TYPE],
+    [null, common.NULL_TYPE],
+    [true, common.BOOLEAN_TYPE],
+    [false, common.BOOLEAN_TYPE],
+    [0, common.NUMBER_TYPE],
+    [1, common.NUMBER_TYPE],
+    [-1, common.NUMBER_TYPE],
+    [0.0, common.NUMBER_TYPE],
+    [1.0, common.NUMBER_TYPE],
+    [1.1, common.NUMBER_TYPE],
+    [-1.0, common.NUMBER_TYPE],
+    [-1.1, common.NUMBER_TYPE],
+    ["", common.STRING_TYPE],
+    ["TEST", common.STRING_TYPE],
+    [[], common.ARRAY_TYPE],
+    [[1, 2, 3], common.ARRAY_TYPE],
+    [{}, common.OBJECT_TYPE],
+    [{ test: 1 }, common.OBJECT_TYPE],
+    [/[^A-Za-z0-9_]/g, common.REGEXP_TYPE],
+    [new RegExp("//"), common.REGEXP_TYPE],
+    [new Date(), common.DATE_TYPE]
+  ];
+  t.plan(cases.length);
+  for (const c of cases) t.is(common.typeOf(c[0]), c[1]);
+});
+
+ava("generateDeviceId", t => {
   const space = [" ", "%20"];
   const special = [";", "%3B"];
   const cases = [
@@ -9,31 +37,30 @@ ava("generateDeviceId", (t) => {
       {
         ProductClass: "TestProductClass",
         OUI: "TestOUI",
-        SerialNumber: "TestSerialNumber",
+        SerialNumber: "TestSerialNumber"
       },
-      "TestOUI-TestProductClass-TestSerialNumber",
+      "TestOUI-TestProductClass-TestSerialNumber"
     ],
     [
       {
         OUI: "TestOUI",
-        SerialNumber: "TestSerialNumber",
+        SerialNumber: "TestSerialNumber"
       },
-      "TestOUI-TestSerialNumber",
+      "TestOUI-TestSerialNumber"
     ],
     [
       {
         OUI: `TestOUIWith${space[0]}_${special[0]}2912`,
-        SerialNumber: `TestSerialNumberWith${space[0]}_${special[0]}2912`,
+        SerialNumber: `TestSerialNumberWith${space[0]}_${special[0]}2912`
       },
-      `TestOUIWith${space[1]}_${special[1]}2912-TestSerialNumberWith${space[1]}_${special[1]}2912`,
-    ],
+      `TestOUIWith${space[1]}_${special[1]}2912-TestSerialNumberWith${space[1]}_${special[1]}2912`
+    ]
   ];
   t.plan(cases.length);
-  for (const c of cases)
-    t.is(common.generateDeviceId(c[0] as Record<string, string>), c[1]);
+  for (const c of cases) t.is(common.generateDeviceId(c[0]), c[1]);
 });
 
-ava("escapeRegExp", (t) => {
+ava("escapeRegExp", t => {
   t.is(
     common.escapeRegExp("\\ ^ $ * + ? . ( ) | { } [ ]"),
     "\\\\ \\^ \\$ \\* \\+ \\? \\. \\( \\) \\| \\{ \\} \\[ \\]"

test/device.ts

@@ -5,7 +5,7 @@ import PathSet from "../lib/common/path-set";
 import VersionedMap from "../lib/versioned-map";
 import { Attributes } from "../lib/types";
 
-ava("getAliasDeclarations", (t) => {
+ava("getAliasDeclarations", t => {
   const path = Path.parse("a.[aa:10,bb.[aaa:100].cc:1].b");
   const decs = device.getAliasDeclarations(path, 99);
 
@@ -22,64 +22,64 @@ ava("getAliasDeclarations", (t) => {
   }
 });
 
-ava("unpack", (t) => {
+ava("unpack", t => {
   const now = Date.now();
   const deviceData = {
     paths: new PathSet(),
     timestamps: new VersionedMap<Path, number>(),
     attributes: new VersionedMap<Path, Attributes>(),
     trackers: new Map<Path, { [name: string]: number }>(),
-    changes: new Set<string>(),
+    changes: new Set<string>()
   };
 
   device.set(deviceData, Path.parse("a.1.b"), now, {
-    value: [now, ["b", "xsd:string"]],
+    value: [now, ["b", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.1.c"), now, {
-    value: [now, ["c", "xsd:string"]],
+    value: [now, ["c", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.1.a.1.a"), now, {
-    value: [now, ["", "xsd:string"]],
+    value: [now, ["", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.1.a.1.b"), now, {
-    value: [now, ["b1", "xsd:string"]],
+    value: [now, ["b1", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.1.a.1.c"), now, {
-    value: [now, ["c1", "xsd:string"]],
+    value: [now, ["c1", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.1.a.2.a"), now, {
-    value: [now, ["", "xsd:string"]],
+    value: [now, ["", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.1.a.2.b"), now, {
-    value: [now, ["b2", "xsd:string"]],
+    value: [now, ["b2", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.1.a.2.c"), now, {
-    value: [now, ["c2", "xsd:string"]],
+    value: [now, ["c2", "xsd:string"]]
   });
 
   device.set(deviceData, Path.parse("a.2.b"), now, {
-    value: [now, ["b", "xsd:string"]],
+    value: [now, ["b", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.2.c"), now, {
-    value: [now, ["c", "xsd:string"]],
+    value: [now, ["c", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.2.a.1.a"), now, {
-    value: [now, ["", "xsd:string"]],
+    value: [now, ["", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.2.a.1.b"), now, {
-    value: [now, ["b1", "xsd:string"]],
+    value: [now, ["b1", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.2.a.1.c"), now, {
-    value: [now, ["c1", "xsd:string"]],
+    value: [now, ["c1", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.2.a.2.a"), now, {
-    value: [now, ["", "xsd:string"]],
+    value: [now, ["", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.2.a.2.b"), now, {
-    value: [now, ["c1", "xsd:string"]],
+    value: [now, ["c1", "xsd:string"]]
   });
   device.set(deviceData, Path.parse("a.2.a.2.c"), now, {
-    value: [now, ["b1", "xsd:string"]],
+    value: [now, ["b1", "xsd:string"]]
   });
 
   let unpacked: Path[];

test/mongodb-functions.ts

@@ -1,13 +1,13 @@
 import ava from "ava";
 import * as mongodbFunctions from "../lib/mongodb-functions";
-import { stringify, parse } from "../lib/common/expression-parser";
+import { stringify } from "../lib/common/expression-parser";
 
-ava("convertOldPrecondition", (t) => {
+ava("mongoQueryToFilter", t => {
   const tests = [
     [{}, "true"],
     [{ test: "test" }, 'test = "test"'],
     [{ test: { $eq: "test" } }, 'test = "test"'],
-    [{ test: { $ne: "test" } }, 'test <> "test" OR test IS NULL'],
+    [{ test: { $ne: "test" } }, 'test <> "test"'],
     [{ test: { $gte: "test" } }, 'test >= "test"'],
     [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
     [{ "Tags.test": false }, "Tags.test IS NULL"],
@@ -22,21 +22,21 @@ ava("convertOldPrecondition", (t) => {
     [{ _tags: { $eq: "test" } }, "Tags.test IS NOT NULL"],
     [
       { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" AND (test <> "test" OR test IS NULL)',
+      'test = "test" AND test <> "test"'
     ],
     [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
     [
       { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" OR (test <> "test" OR test IS NULL)',
+      'test = "test" OR test <> "test"'
     ],
     [
       { test: { $gte: "test1", $ne: "test2" } },
-      'test >= "test1" AND (test <> "test2" OR test IS NULL)',
+      'test >= "test1" AND test <> "test2"'
     ],
     [
       { test: "test", test2: { $ne: "test2" } },
-      'test = "test" AND (test2 <> "test2" OR test2 IS NULL)',
-    ],
+      'test = "test" AND test2 <> "test2"'
+    ]
   ];
 
   const shouldFailTests = [
@@ -45,88 +45,18 @@ ava("convertOldPrecondition", (t) => {
     [{ "Tags.test": { $gt: true } }, "Invalid tag query"],
     [{ _tags: [] }, "Invalid type"],
     [{ _tags: { $gt: "test" } }, "Invalid tag query"],
-    [{ $nor: [] }, "Operator $nor not supported"],
+    [{ $nor: [] }, "Operator $nor not supported"]
   ];
 
   t.plan(tests.length + 2 * shouldFailTests.length);
   for (const test of tests)
-    t.is(stringify(mongodbFunctions.convertOldPrecondition(test[0])), test[1]);
+    t.is(stringify(mongodbFunctions.mongoQueryToFilter(test[0])), test[1]);
 
   for (const test of shouldFailTests) {
     const func = (): void => {
-      mongodbFunctions.convertOldPrecondition(
-        test[0] as Record<string, unknown>
-      );
+      mongodbFunctions.mongoQueryToFilter(test[0]);
     };
-    const error = t.throws(func, { instanceOf: Error });
+    const error = t.throws(func, Error);
     t.is(error.message, test[1]);
   }
 });
-
-ava("filterToMongoQuery", async (t) => {
-  const queries: [string, Record<string, unknown>][] = [
-    ["true", {}],
-    ["Tags.tag1 = true", { _tags: "tag1" }],
-    ["Tags.tag1 <> false", { _tags: "tag1" }],
-    ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }],
-    ["Tags.tag1 = 123", { "Tags.tag1": 123 }],
-    ["Param1 = 'value1'", { "Param1._value": "value1" }],
-    ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }],
-    ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
-    ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
-    ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }],
-    ["LOWER(Param1) LIKE 'value'", { "Param1._value": /^value$/i }],
-    [
-      "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')",
-      {
-        $or: [
-          { "Param1._value": { $nin: ["value2", null] } },
-          {
-            $and: [
-              { "Param2._value": { $nin: ["value1", null] } },
-              { "Param1._value": { $gte: "value2" } },
-            ],
-          },
-        ],
-      },
-    ],
-    [
-      "Param1 <> 'value2' OR Param1 IS NULL",
-      {
-        $or: [
-          { "Param1._value": { $nin: ["value2", null] } },
-          { "Param1._value": null },
-        ],
-      },
-    ],
-  ];
-
-  for (const [expStr, expect] of queries) {
-    const exp = parse(expStr);
-    const query = mongodbFunctions.filterToMongoQuery(
-      mongodbFunctions.processDeviceFilter(exp)
-    );
-    t.deepEqual(query, expect);
-  }
-
-  const failQueries: [any, string][] = [
-    ["Param1 = Param2", "Invalid RHS operand of = clause"],
-    ["Param1 LIKE Param2", "Invalid RHS operand of LIKE clause"],
-    ["1 LIKE '1'", "Invalid LHS operand of LIKE clause"],
-    ["1 = 2", "Invalid LHS operand of = clause"],
-    ["1 IS NULL", "Invalid LHS operand of IS NULL clause"],
-    ["false", "Primitives are not valid queries"],
-    ["UPPER(Param1) LIKE 'value'", "Invalid RHS operand of LIKE clause"],
-  ];
-
-  for (const [expStr, err] of failQueries) {
-    const exp = parse(expStr);
-    t.throws(
-      () =>
-        mongodbFunctions.filterToMongoQuery(
-          mongodbFunctions.processDeviceFilter(exp)
-        ),
-      { message: err }
-    );
-  }
-});

test/path-set.ts

@@ -2,7 +2,7 @@ import ava from "ava";
 import Path from "../lib/common/path";
 import PathSet from "../lib/common/path-set";
 
-ava("depth", (t) => {
+ava("depth", t => {
   const pathSet = new PathSet();
   t.is(pathSet.depth, 0);
   pathSet.add(Path.parse(""));
@@ -11,14 +11,14 @@ ava("depth", (t) => {
   t.is(pathSet.depth, 2);
 });
 
-ava("add", (t) => {
+ava("add", t => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a"));
   pathSet.add(Path.parse("a"));
   t.is(pathSet.find(Path.parse("a"), true, true, 99).length, 1);
 });
 
-ava("get", (t) => {
+ava("get", t => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a.*"));
   pathSet.add(Path.parse("a.a"));
@@ -28,7 +28,7 @@ ava("get", (t) => {
   t.is(pathSet.get(Path.parse("*.a")), null);
 });
 
-ava("find", (t) => {
+ava("find", t => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a"));
   pathSet.add(Path.parse("a.*"));
@@ -37,32 +37,32 @@ ava("find", (t) => {
   pathSet.add(Path.parse("*.*"));
 
   t.deepEqual(
-    pathSet.find(Path.parse(""), true, true, 1).map((p) => p.toString()),
+    pathSet.find(Path.parse(""), true, true, 1).map(p => p.toString()),
     ["a"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse(""), false, false, 2).map((p) => p.toString()),
+    pathSet.find(Path.parse(""), false, false, 2).map(p => p.toString()),
     ["a", "a.*", "a.a", "*.a", "*.*"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), false, false).map((p) => p.toString()),
+    pathSet.find(Path.parse("a.*"), false, false).map(p => p.toString()),
     ["a.*"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), false, true).map((p) => p.toString()),
+    pathSet.find(Path.parse("a.*"), false, true).map(p => p.toString()),
     ["a.*", "a.a"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), true, false).map((p) => p.toString()),
+    pathSet.find(Path.parse("a.*"), true, false).map(p => p.toString()),
     ["a.*", "*.*"]
   );
 
   t.deepEqual(
-    pathSet.find(Path.parse("a.*"), true, true).map((p) => p.toString()),
+    pathSet.find(Path.parse("a.*"), true, true).map(p => p.toString()),
     ["a.*", "a.a", "*.a", "*.*"]
   );
 });

test/path.ts

@@ -1,13 +1,13 @@
 import ava from "ava";
 import Path from "../lib/common/path";
 
-ava("toString", (t) => {
+ava("toString", t => {
   const path1 = Path.parse('abc.[ abc : 123 , 123 : abc , 123: " abc "].123');
   const path2 = Path.parse('abc.[123:  " abc ",abc:123,123:abc].123');
   t.is(path1.toString(), path2.toString());
 });
 
-ava("slice", (t) => {
+ava("slice", t => {
   const path = Path.parse("a.*.b.[x:y].c");
   const sliced = path.slice(1, -1);
   t.is(sliced.toString(), '*.b.[x:"y"]');
@@ -15,7 +15,7 @@ ava("slice", (t) => {
   t.is(sliced.wildcard, 0b1);
 });
 
-ava("concat", (t) => {
+ava("concat", t => {
   const path1 = Path.parse("a");
   const path2 = Path.parse("*.[a:b]");
   const concat = path1.concat(path2);

test/xml-parser.ts

@@ -2,10 +2,10 @@ import ava from "ava";
 import {
   parseXmlDeclaration,
   decodeEntities,
-  parseXml,
+  parseXml
 } from "../lib/xml-parser";
 
-ava("parseXmlDeclaration", (t) => {
+ava("parseXmlDeclaration", t => {
   const buf = Buffer.from(
     '<?xml version="1.0" encoding="UTF-8"?>\n<soap-env:Envelope />'
   );
@@ -15,25 +15,25 @@ ava("parseXmlDeclaration", (t) => {
       name: "version",
       namespace: "",
       localName: "version",
-      value: "1.0",
+      value: "1.0"
     },
     {
       name: "encoding",
       namespace: "",
       localName: "encoding",
-      value: "UTF-8",
-    },
+      value: "UTF-8"
+    }
   ]);
 });
 
-ava("decodeEntities", (t) => {
+ava("decodeEntities", t => {
   t.is(
     decodeEntities("&&amp;&lt;&gt;&quot;&apos;&gt;&#167;&#xd842;&#xDFB7;;"),
     "&&<>\"'>§𠮷;"
   );
 });
 
-ava("parse", (t) => {
+ava("parse", t => {
   const xml =
     '<?xml version="1.0"?>\n<a-b:c><d f="1<g>"/><!-- comment --><h >i</h></a-b:c>';
   const parsed = parseXml(xml);
@@ -60,7 +60,7 @@ ava("parse", (t) => {
             attrs: 'f="1<g>"',
             text: "",
             bodyIndex: 42,
-            children: [],
+            children: []
           },
           {
             name: "h",
@@ -69,10 +69,10 @@ ava("parse", (t) => {
             attrs: "",
             text: "i",
             bodyIndex: 62,
-            children: [],
-          },
-        ],
-      },
-    ],
+            children: []
+          }
+        ]
+      }
+    ]
   });
 });

tools/dump-data-model

@@ -1,3 +1,5 @@
+#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
+
 /**
  * Copyright 2013-2019  GenieACS Inc.
  *
@@ -58,7 +60,7 @@ connect()
   .then(() => {
     devicesCollection.findOne(
       {
-        _id: process.argv[2],
+        _id: process.argv[2]
       },
       {},
       (err, device) => {
@@ -77,6 +79,6 @@ connect()
       }
     );
   })
-  .catch((err) => {
+  .catch(err => {
     throw err;
   });

tsconfig.json

@@ -14,11 +14,11 @@
     "allowJs": true
   },
   "include": [
-      "./bin/*.ts",
+      "./bin/.*.ts",
       "./lib/**/*",
       "./ui/**/*",
       "./test/**/*",
       "./build/**/*",
-      "./tools/*.ts"
+      "./tools/.*.ts"
   ]
 }

ui/admin-menu.ts

@@ -22,7 +22,7 @@ import { ClosureComponent, Component } from "mithril";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       const active = { [vnode.attrs["page"]]: "active" };
       const tabs = [];
 
@@ -97,7 +97,7 @@ const component: ClosureComponent = (): Component => {
       }
 
       return m("nav#side-menu", m("ul", tabs));
-    },
+    }
   };
 };
 

ui/app.ts

@@ -37,14 +37,13 @@ import * as usersPage from "./users-page";
 import Authorizer from "../lib//common/authorizer";
 import * as notifications from "./notifications";
 import { contextifyComponent } from "./components";
-import { PermissionSet, UiConfig } from "../lib/types";
 
 declare global {
   interface Window {
     authorizer: Authorizer;
-    permissionSets: PermissionSet[];
+    permissionSets: any;
     username: string;
-    clientConfig: { ui: UiConfig };
+    clientConfig: any;
     configSnapshot: string;
     genieacsVersion: string;
   }
@@ -59,7 +58,7 @@ const adminPages = [
   "files",
   "config",
   "users",
-  "permissions",
+  "permissions"
 ];
 
 let state;
@@ -71,7 +70,7 @@ let fulfillTimeout = null;
 function fulfill(): void {
   clearTimeout(fulfillTimeout);
   fulfillTimeout = setTimeout(() => {
-    store.fulfill(lastRenderTimestamp, pageVisitTimestamp, (updated) => {
+    store.fulfill(lastRenderTimestamp, pageVisitTimestamp).then(updated => {
       if (updated) m.redraw();
     });
   }, 100);
@@ -89,7 +88,7 @@ function pagify(pageName, page): RouteResolver {
       attrs["page"] = pageName;
       return m(layout, attrs, p);
     },
-    onmatch: null,
+    onmatch: null
   };
 
   component.onmatch = (args, requestedPath) => {
@@ -100,16 +99,16 @@ function pagify(pageName, page): RouteResolver {
       return null;
     }
 
-    return new Promise((resolve) => {
+    return new Promise(resolve => {
       page
         .init(args)
-        .then((st) => {
+        .then(st => {
           if (!st) return void m.route.set("/");
           state = st;
           resolve();
           fulfill();
         })
-        .catch((err) => {
+        .catch(err => {
           if (!window.username && err.message.indexOf("authorized") >= 0) {
             notifications.push("error", err.message);
             m.route.set("/login", { continue: requestedPath });
@@ -133,7 +132,7 @@ function redirectAdminPage(): RouteResolver {
         }
       }
       return null;
-    },
+    }
   };
   return component;
 }
@@ -155,5 +154,5 @@ m.route(document.body, "/overview", {
   "/admin/files": pagify("files", filesPage),
   "/admin/config": pagify("config", configPage),
   "/admin/users": pagify("users", usersPage),
-  "/admin/permissions": pagify("permissions", permissionsPage),
+  "/admin/permissions": pagify("permissions", permissionsPage)
 });

ui/autocomplete-compnent.ts

@@ -17,13 +17,11 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-type AutocompleteCallback = (
+export default class Autocomplete {
+  private callback: (
     value: string,
     callback: (suggestions: { value: string; tip?: string }[]) => void
   ) => void;
-
-export default class Autocomplete {
-  private callback: AutocompleteCallback;
   private element: HTMLInputElement;
   private hideTimeout: NodeJS.Timeout;
   private visible: boolean;
@@ -31,7 +29,7 @@ export default class Autocomplete {
   private selection: number;
   private container: HTMLElement;
 
-  public constructor(className: string, callback: AutocompleteCallback) {
+  public constructor(className, callback) {
     this.callback = callback;
     this.element = null;
     this.hideTimeout = null;
@@ -46,7 +44,7 @@ export default class Autocomplete {
     this.container.className = className;
   }
 
-  public attach(el: HTMLInputElement): void {
+  public attach(el): void {
     el.setAttribute("autocomplete", "off");
 
     el.addEventListener("focus", () => {
@@ -64,7 +62,7 @@ export default class Autocomplete {
       this.hide();
     });
 
-    el.addEventListener("keydown", (e) => {
+    el.addEventListener("keydown", e => {
       if (this.element !== el) return;
       if (e.key === "Escape") {
         if (this.visible) this.hide();
@@ -110,7 +108,7 @@ export default class Autocomplete {
   private update(): void {
     const el = this.element;
 
-    this.callback(el.value, (suggestions) => {
+    this.callback(el.value, suggestions => {
       if (this.element !== el) return;
       this.default = null;
 
@@ -156,7 +154,7 @@ export default class Autocomplete {
 
         const t = document.createTextNode(suggestion.value);
         e.appendChild(t);
-        e.addEventListener("mousedown", (ev) => {
+        e.addEventListener("mousedown", ev => {
           ev.preventDefault();
           el.value = suggestion.value;
           if (this.element === el) this.update();

ui/change-password-component.ts

@@ -24,7 +24,7 @@ import { m } from "./components";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       const onPasswordChange = vnode.attrs["onPasswordChange"];
       const enforceAuth = !vnode.attrs["noAuth"];
       const username = vnode.attrs["username"];
@@ -40,19 +40,19 @@ const component: ClosureComponent = (): Component => {
             type: "text",
             value: vnode.state["username"],
             disabled: !!username,
-            oninput: (e) => {
+            oninput: e => {
               vnode.state["username"] = e.target.value;
             },
-            oncreate: (_vnode) => {
+            oncreate: _vnode => {
               (_vnode.dom as HTMLSelectElement).focus();
-            },
+            }
           })
-        ),
+        )
       ];
 
       let fields = {
         newPassword: "New password",
-        confirmPassword: "Confirm password",
+        confirmPassword: "Confirm password"
       };
       if (enforceAuth)
         fields = Object.assign({ authPassword: "Your password" }, fields);
@@ -66,9 +66,9 @@ const component: ClosureComponent = (): Component => {
               name: f,
               type: "password",
               value: vnode.state[f],
-              oninput: (e) => {
+              oninput: e => {
                 vnode.state[f] = e.target.value;
-              },
+              }
             })
           )
         );
@@ -77,7 +77,7 @@ const component: ClosureComponent = (): Component => {
       const submit = m(
         "button.primary",
         {
-          type: "submit",
+          type: "submit"
         },
         "Change password"
       ) as VnodeDOM;
@@ -89,7 +89,7 @@ const component: ClosureComponent = (): Component => {
         m(
           "form",
           {
-            onsubmit: (e) => {
+            onsubmit: e => {
               e.redraw = false;
               e.preventDefault();
               if (
@@ -120,19 +120,19 @@ const component: ClosureComponent = (): Component => {
                     if (onPasswordChange) onPasswordChange();
                     (submit.dom as HTMLFormElement).disabled = false;
                   })
-                  .catch((err) => {
+                  .catch(err => {
                     notifications.push("error", err.message);
                     (submit.dom as HTMLFormElement).disabled = false;
                   });
               }
-            },
+            }
           },
           form
-        ),
+        )
       ];
 
       return m("div.put-form", children);
-    },
+    }
   };
 };
 

ui/code-editor-component.ts

@@ -23,11 +23,11 @@ import { codeMirror } from "./dynamic-loader";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       return m("textarea", {
         name: vnode.attrs["id"],
         value: vnode.attrs["value"],
-        oncreate: (_vnode) => {
+        oncreate: _vnode => {
           const editor = codeMirror.fromTextArea(_vnode.dom, {
             mode: vnode.attrs["mode"],
             lineNumbers: true,
@@ -40,21 +40,21 @@ const component: ClosureComponent = (): Component => {
               "Cmd-Enter": () => {
                 if (vnode.attrs["onSubmit"])
                   vnode.attrs["onSubmit"](_vnode.dom);
-              },
-            },
+              }
+            }
           });
 
           if (vnode.attrs["onChange"]) {
-            editor.on("change", (e) => {
+            editor.on("change", e => {
               vnode.attrs["onChange"](e.getValue());
             });
           }
 
           if (vnode.attrs["focus"]) editor.focus();
           if (vnode.attrs["onReady"]) vnode.attrs["onReady"](editor);
-        },
+        }
       });
-    },
+    }
   };
 };
 

ui/components.ts

@@ -24,7 +24,7 @@ import m, {
   ComponentTypes,
   Lifecycle,
   ClosureComponent,
-  Vnode,
+  Vnode
 } from "mithril";
 import parameter from "./components/parameter";
 import parameterList from "./components/parameter-list";
@@ -39,7 +39,6 @@ import tags from "./components/tags";
 import ping from "./components/ping";
 import deviceLink from "./components/device-link";
 import longTextComponent from "./long-text-component";
-import deviceUploads from "./components/device-uploads";
 
 const comps = {
   parameter,
@@ -54,8 +53,7 @@ const comps = {
   tags,
   ping,
   "device-link": deviceLink,
-  "long-text": longTextComponent,
-  "device-uploads": deviceUploads,
+  "long-text": longTextComponent
 };
 
 const contextifiedComponents = new WeakMap<ComponentTypes, ComponentTypes>();
@@ -98,7 +96,7 @@ const M = new Proxy(m, {
   get: (target, prop) => {
     if (prop === "context") return contextFn;
     else return Reflect.get(target, prop);
-  },
+  }
 }) as MC;
 
 function contextFn(context, ...argumentsList): Vnode {
@@ -134,7 +132,7 @@ export function contextifyComponent(component: ComponentTypes): ComponentTypes {
         return res;
       };
     } else if (!component.prototype || !component.prototype.view) {
-      c = (initialNode) => {
+      c = initialNode => {
         const state = (component as ClosureComponent)(initialNode);
         const view = state.view;
         state.view = function(vnode) {

ui/components/all-parameters.ts

@@ -30,73 +30,40 @@ function escapeRegExp(str): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
 
-const keysByDepth: WeakMap<Record<string, unknown>, string[][]> = new WeakMap();
-
-function orderKeysByDepth(device: Record<string, unknown>): string[][] {
-  if (keysByDepth.has(device)) return keysByDepth.get(device);
-  const res: string[][] = [];
-  for (const key of Object.keys(device)) {
-    let count = 0;
-    for (
-      let i = key.lastIndexOf(".", key.length - 2);
-      i >= 0;
-      i = key.lastIndexOf(".", i - 1)
-    )
-      ++count;
-    while (res.length <= count) res.push([]);
-    res[count].push(key);
-  }
-  keysByDepth.set(device, res);
-  return res;
-}
-
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       const device = vnode.attrs["device"];
 
       const search = m("input", {
         type: "text",
         placeholder: "Search parameters",
-        oninput: (e) => {
+        oninput: e => {
           vnode.state["searchString"] = e.target.value;
           e.redraw = false;
           clearTimeout(vnode.state["timeout"]);
           vnode.state["timeout"] = setTimeout(m.redraw, 500);
-        },
+        }
       });
 
       const instanceRegex = /\.[0-9]+$/;
       let re;
       if (vnode.state["searchString"]) {
-        const keywords = vnode.state["searchString"]
-          .split(" ")
-          .filter((s) => s);
+        const keywords = vnode.state["searchString"].split(" ").filter(s => s);
         if (keywords.length)
-          re = new RegExp(keywords.map((s) => escapeRegExp(s)).join(".*"), "i");
+          re = new RegExp(keywords.map(s => escapeRegExp(s)).join(".*"), "i");
       }
 
-      const filteredKeys: string[] = [];
-      const allKeys = orderKeysByDepth(device);
-      let count = 0;
-      for (const keys of allKeys) {
-        let c = 0;
-        for (const k of keys) {
-          const p = device[k];
-          const str = p.value && p.value[0] ? `${k} ${p.value[0]}` : k;
-          if (re && !re.test(str)) continue;
-          ++c;
-          if (count < 100) filteredKeys.push(k);
-        }
-        count += c;
-      }
-
-      filteredKeys.sort();
-
-      const rows = filteredKeys.map((k) => {
+      const rows = Object.keys(device)
+        .sort()
+        .map(k => {
           const p = device[k];
           const val = [];
-        const attrs = { key: k };
+          const attrs = {};
+          if (re) {
+            const str = p.value && p.value[0] ? `${k} ${p.value[0]}` : k;
+            if (!re.test(str)) attrs["style"] = "display: none;";
+          }
 
           if (p.object === false) {
             val.push(
@@ -116,9 +83,9 @@ const component: ClosureComponent = (): Component => {
                       taskQueue.queueTask({
                         name: "deleteObject",
                         device: device["DeviceID.ID"].value[0],
-                      objectName: k,
+                        objectName: k
                       });
-                  },
+                    }
                   },
                   getIcon("delete-instance")
                 )
@@ -133,9 +100,9 @@ const component: ClosureComponent = (): Component => {
                       taskQueue.queueTask({
                         name: "addObject",
                         device: device["DeviceID.ID"].value[0],
-                      objectName: k,
+                        objectName: k
                       });
-                  },
+                    }
                   },
                   getIcon("add-instance")
                 )
@@ -152,9 +119,9 @@ const component: ClosureComponent = (): Component => {
                   taskQueue.queueTask({
                     name: "getParameterValues",
                     device: device["DeviceID.ID"].value[0],
-                  parameterNames: [k],
+                    parameterNames: [k]
                   });
-              },
+                }
               },
               getIcon("refresh")
             )
@@ -173,25 +140,18 @@ const component: ClosureComponent = (): Component => {
         m(
           "a.download-csv",
           {
-            href: `api/devices/${encodeURIComponent(
+            href: `/api/devices/${encodeURIComponent(
               device["DeviceID.ID"].value[0]
             )}.csv`,
             download: "",
-            style: "float: right;",
+            style: "float: right;"
           },
           "Download"
         ),
         search,
-        m(
-          ".parameter-list",
-          m("table", m("tbody", rows)),
-          m(
-            "m",
-            `Displaying ${filteredKeys.length} out of ${count} parameters.`
-          )
-        )
+        m(".parameter-list", m("table", m("tbody", rows)))
       );
-    },
+    }
   };
 };
 

ui/components/container.ts

@@ -23,7 +23,7 @@ import * as store from "../store";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       if (vnode.attrs["filter"]) {
         if (
           !store.evaluateExpression(
@@ -34,13 +34,13 @@ const component: ClosureComponent = (): Component => {
           return null;
       }
 
-      const children = Object.values(vnode.attrs["components"]).map((c) => {
+      const children = Object.values(vnode.attrs["components"]).map(c => {
         if (typeof c !== "object") return `${c}`;
         return m(c["type"], c);
       });
       if (vnode.attrs["element"]) return m(vnode.attrs["element"], children);
       else return children;
-    },
+    }
   };
 };
 

ui/components/device-actions.ts

@@ -22,11 +22,10 @@ import { m } from "../components";
 import * as taskQueue from "../task-queue";
 import * as notifications from "../notifications";
 import * as store from "../store";
-import * as overlay from "../overlay";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       const device = vnode.attrs["device"];
 
       const buttons = [];
@@ -39,9 +38,9 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.queueTask({
                 name: "reboot",
-                device: device["DeviceID.ID"].value[0],
+                device: device["DeviceID.ID"].value[0]
               });
-            },
+            }
           },
           "Reboot"
         )
@@ -55,9 +54,9 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.queueTask({
                 name: "factoryReset",
-                device: device["DeviceID.ID"].value[0],
+                device: device["DeviceID.ID"].value[0]
               });
-            },
+            }
           },
           "Reset"
         )
@@ -71,38 +70,14 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageDownload({
                 name: "download",
-                devices: [device["DeviceID.ID"].value[0]],
+                device: device["DeviceID.ID"].value[0]
               });
-            },
+            }
           },
           "Push file"
         )
       );
 
-      buttons.push(
-        m(
-          "button.primary",
-          {
-            title: "Fetch files from device",
-            onclick: () => {
-              const deviceId = device["DeviceID.ID"].value[0];
-              overlay.open(() => {
-                const dev = store.fetch("devices", [
-                  "=",
-                  ["PARAM", "DeviceID.ID"],
-                  deviceId,
-                ]);
-                return [
-                  m("h3", `Uploads from ${deviceId}`),
-                  m("device-uploads", { device: dev.value[0] }),
-                ];
-              });
-            },
-          },
-          "Uploads"
-        )
-      );
-
       buttons.push(
         m(
           "button.primary",
@@ -118,17 +93,17 @@ const component: ClosureComponent = (): Component => {
                   notifications.push("success", `${deviceId}: Device deleted`);
                   m.route.set("/devices");
                 })
-                .catch((err) => {
+                .catch(err => {
                   notifications.push("error", err.message);
                 });
-            },
+            }
           },
           "Delete"
         )
       );
 
       return m(".actions-bar", buttons);
-    },
+    }
   };
 };
 

ui/components/device-faults.ts

@@ -25,13 +25,13 @@ import { getIcon } from "../icons";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       const device = vnode.attrs["device"];
       const deviceId = device["DeviceID.ID"].value[0];
       const faults = store.fetch("faults", [
         "AND",
         [">", ["PARAM", "_id"], `${deviceId}:`],
-        ["<", ["PARAM", "_id"], `${deviceId}:zzzz`],
+        ["<", ["PARAM", "_id"], `${deviceId}:zzzz`]
       ]);
 
       const headers = [
@@ -39,8 +39,8 @@ const component: ClosureComponent = (): Component => {
         "Code",
         "Message",
         "Retries",
-        "Timestamp",
-      ].map((l) => m("th", l));
+        "Timestamp"
+      ].map(l => m("th", l));
       const thead = m("thead", m("tr", headers));
 
       const rows = [];
@@ -57,7 +57,7 @@ const component: ClosureComponent = (): Component => {
               "button",
               {
                 title: "Delete fault",
-                onclick: (e) => {
+                onclick: e => {
                   e.redraw = false;
                   store
                     .deleteResource("faults", f["_id"])
@@ -66,24 +66,21 @@ const component: ClosureComponent = (): Component => {
                       store.fulfill(Date.now(), Date.now());
                       m.redraw();
                     })
-                    .catch((err) => {
+                    .catch(err => {
                       notifications.push("error", err.message);
                       store.fulfill(Date.now(), Date.now());
                     });
-                },
+                }
               },
               getIcon("remove")
             )
-          ),
+          )
         ]);
       }
 
       let tbody;
       if (rows.length) {
-        tbody = m(
-          "tbody",
-          rows.map((r) => m("tr", r))
-        );
+        tbody = m("tbody", rows.map(r => m("tr", r)));
       } else {
         tbody = m(
           "tbody",
@@ -92,7 +89,7 @@ const component: ClosureComponent = (): Component => {
       }
 
       return m("table.table", thead, tbody);
-    },
+    }
   };
 };
 

ui/components/device-link.ts

@@ -22,12 +22,12 @@ import { m } from "../components";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       let deviceId;
       if (vnode.attrs["device"])
         deviceId = vnode.attrs["device"]["DeviceID.ID"].value[0];
 
-      const children = Object.values(vnode.attrs["components"]).map((c) => {
+      const children = Object.values(vnode.attrs["components"]).map(c => {
         if (typeof c !== "object") return `${c}`;
         const attrs = Object.assign({}, vnode.attrs, c);
         return m(attrs["type"], attrs);
@@ -41,7 +41,7 @@ const component: ClosureComponent = (): Component => {
       } else {
         return children;
       }
-    },
+    }
   };
 };
 

ui/components/device-uploads.ts

@@ -1,155 +0,0 @@
-/**
- * Copyright 2013-2020  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ClosureComponent, Component } from "mithril";
-import { m } from "../components";
-import * as store from "../store";
-import * as notifications from "../notifications";
-import { getIcon } from "../icons";
-import * as taskQueue from "../task-queue";
-
-const component: ClosureComponent = (): Component => {
-  return {
-    view: (vnode) => {
-      const device = vnode.attrs["device"];
-      const deviceId = device["DeviceID.ID"].value[0];
-      const files = store.fetch("uploads", [
-        "AND",
-        [">", ["PARAM", "_id"], `${deviceId}/`],
-        ["<", ["PARAM", "_id"], `${deviceId}/zzzz`],
-      ]);
-
-      const uploads: Record<string, any> = {};
-
-      for (const [k, v] of Object.entries(device)) {
-        if (k.startsWith("Uploads.")) {
-          const parts = k.split(".");
-          if (parts.length !== 3) continue;
-          uploads[parts[1]] = uploads[parts[1]] || {};
-          uploads[parts[1]][parts[2]] = v["value"][0];
-        }
-      }
-
-      const headers = ["Filename", "Type", "Timestamp", "Status"].map((l) =>
-        m("th", l)
-      );
-      const thead = m("thead", m("tr", headers));
-
-      let minTime = 999999;
-      const timestamp = store.getTimestamp();
-      const rows = [];
-      const uploadsSorted = Object.values(uploads).sort(
-        (a, b) => a["Upload"] - b["Upload"]
-      );
-
-      for (const u of Object.values(uploadsSorted)) {
-        if (!u["Upload"]) continue;
-        const ready = u["LastUpload"] >= u["Upload"];
-        if (!ready) {
-          minTime = Math.min(minTime, u["Upload"] - timestamp);
-          rows.push([
-            m("td", u["FileName"]),
-            m("td", u["FileType"]),
-            m("td", new Date(u["Upload"]).toLocaleString()),
-            m("td", "Waiting for upload"),
-          ]);
-        } else {
-          const filePath = `${deviceId}/${u["FileName"]}`;
-          if (!Array.isArray(files.value)) continue;
-          if (!files.value.some((f) => f["_id"] === filePath)) continue;
-
-          const deleteButton = m(
-            "button",
-            {
-              title: "Delete file",
-              onclick: (e) => {
-                e.redraw = false;
-                store
-                  .deleteResource("uploads", filePath)
-                  .then(() => {
-                    notifications.push("success", "File deleted");
-                    store.fulfill(Date.now(), Date.now());
-                    m.redraw();
-                  })
-                  .catch((err) => {
-                    notifications.push("error", err.message);
-                    store.fulfill(Date.now(), Date.now());
-                  });
-              },
-            },
-            getIcon("delete-instance")
-          );
-
-          rows.push([
-            m("td", u["FileName"]),
-            m("td", u["FileType"]),
-            m("td", new Date(u["LastUpload"]).toLocaleString()),
-            m(
-              "td",
-              m(
-                "a",
-                {
-                  href: `api/uploads/blob/${encodeURIComponent(filePath)}`,
-                },
-                "Ready"
-              )
-            ),
-            m("td", deleteButton),
-          ]);
-        }
-      }
-
-      if (minTime < 60000) {
-        setTimeout(() => {
-          store.fulfill(0, timestamp + 5000);
-        }, 10000);
-      }
-
-      const addButton = m(
-        "button",
-        {
-          title: "Fetch a new file",
-          onclick: () => {
-            taskQueue.stageUpload({
-              name: "upload",
-              devices: [deviceId],
-            });
-          },
-        },
-        getIcon("add-instance")
-      );
-
-      const empty = !rows.length;
-
-      rows.push([m("td", { colspan: headers.length }), m("td", addButton)]);
-
-      const tbody = m(
-        "tbody",
-        rows.map((r) => m("tr", r)),
-        empty
-          ? m("tr.empty", m("td", { colspan: headers.length }, "No uploads"))
-          : null
-      );
-
-      return m("table.table", thead, tbody);
-    },
-  };
-};
-
-export default component;

ui/components/overview-dot.ts

@@ -26,9 +26,9 @@ const CHARTS = config.ui.overview.charts;
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       const device = vnode.attrs["device"];
-      const chart = CHARTS[vnode.attrs["chart"]] as Record<string, unknown>;
+      const chart = CHARTS[vnode.attrs["chart"]];
       if (!chart) return null;
       for (const slice of Object.values(chart.slices)) {
         const filter = slice["filter"];
@@ -39,20 +39,20 @@ const component: ClosureComponent = (): Component => {
               width: "1em",
               height: "1em",
               xmlns: "http://www.w3.org/2000/svg",
-              "xmlns:xlink": "http://www.w3.org/1999/xlink",
+              "xmlns:xlink": "http://www.w3.org/1999/xlink"
             },
             m("circle", {
               cx: "0.5em",
               cy: "0.5em",
               r: "0.4em",
-              fill: slice["color"],
+              fill: slice["color"]
             })
           );
           return m("span.overview-dot", dot, `${slice["label"]}`);
         }
       }
       return null;
-    },
+    }
   };
 };
 

ui/components/parameter-list.ts

@@ -22,14 +22,14 @@ import { m } from "../components";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       const device = vnode.attrs["device"];
 
-      const rows = Object.values(vnode.attrs["parameters"]).map((parameter) => {
+      const rows = Object.values(vnode.attrs["parameters"]).map(parameter => {
         const p = m.context(
           {
             device: device,
-            parameter: parameter["parameter"],
+            parameter: parameter["parameter"]
           },
           parameter["type"] || "parameter",
           parameter
@@ -38,11 +38,11 @@ const component: ClosureComponent = (): Component => {
         return m(
           "tr",
           {
-            onupdate: (vn) => {
+            onupdate: vn => {
               (vn.dom as HTMLElement).style.display = (p as VnodeDOM).dom
                 ? ""
                 : "none";
-            },
+            }
           },
           m("th", parameter["label"]),
           m("td", p)
@@ -50,7 +50,7 @@ const component: ClosureComponent = (): Component => {
       });
 
       return m("table.parameter-list", rows);
-    },
+    }
   };
 };
 

ui/components/parameter-table.ts

@@ -26,21 +26,21 @@ import { getIcon } from "../icons";
 
 const component: ClosureComponent = (): Component => {
   return {
-    oninit: (vnode) => {
+    oninit: vnode => {
       const obj = vnode.attrs["parameter"];
       if (!Array.isArray(obj) || obj[0] !== "PARAM")
         throw new Error("Object must be a parameter path");
       vnode.state["object"] = obj[1];
       vnode.state["parameters"] = Object.values(vnode.attrs["childParameters"]);
     },
-    view: (vnode) => {
+    view: vnode => {
       const device = vnode.attrs["device"];
       const object = store.evaluateExpression(vnode.state["object"], device);
       const parameters = vnode.state["parameters"];
 
-      if (typeof object !== "string" || !device[object]) return null;
+      if (!device[object]) return null;
 
-      const instances: Set<string> = new Set();
+      const instances = new Set();
       const prefix = `${object}.`;
       for (const p in device) {
         if (p.startsWith(prefix)) {
@@ -50,7 +50,7 @@ const component: ClosureComponent = (): Component => {
         }
       }
 
-      const headers = Object.values(parameters).map((p) => m("th", p["label"]));
+      const headers = Object.values(parameters).map(p => m("th", p["label"]));
 
       const thead = m("thead", m("tr", headers));
 
@@ -59,7 +59,7 @@ const component: ClosureComponent = (): Component => {
         let filter =
           vnode.attrs["filter"] != null ? vnode.attrs["filter"] : true;
 
-        filter = expressionParser.map(filter, (e) => {
+        filter = expressionParser.map(filter, e => {
           if (Array.isArray(e) && e[0] === "PARAM")
             return ["PARAM", ["||", i, ".", e[1]]];
           return e;
@@ -67,8 +67,8 @@ const component: ClosureComponent = (): Component => {
 
         if (!store.evaluateExpression(filter, device)) continue;
 
-        const row = parameters.map((p) => {
-          const param = expressionParser.map(p.parameter, (e) => {
+        const row = parameters.map(p => {
+          const param = expressionParser.map(p.parameter, e => {
             if (Array.isArray(e) && e[0] === "PARAM")
               return ["PARAM", ["||", i, ".", e[1]]];
             return e;
@@ -78,13 +78,13 @@ const component: ClosureComponent = (): Component => {
             m.context(
               {
                 device: device,
-                parameter: param,
+                parameter: param
               },
               p.type || "parameter",
               Object.assign({}, p, {
                 device: device,
                 parameter: param,
-                label: null,
+                label: null
               })
             )
           );
@@ -102,9 +102,9 @@ const component: ClosureComponent = (): Component => {
                     taskQueue.queueTask({
                       name: "deleteObject",
                       device: device["DeviceID.ID"].value[0],
-                      objectName: i,
+                      objectName: i
                     });
-                  },
+                  }
                 },
                 getIcon("delete-instance")
               )
@@ -135,9 +135,9 @@ const component: ClosureComponent = (): Component => {
                     taskQueue.queueTask({
                       name: "addObject",
                       device: device["DeviceID.ID"].value[0],
-                      objectName: object,
+                      objectName: object
                     });
-                  },
+                  }
                 },
                 getIcon("add-instance")
               )
@@ -150,7 +150,7 @@ const component: ClosureComponent = (): Component => {
       if (vnode.attrs["label"]) label = m("h2", vnode.attrs["label"]);
 
       return [label, m("table.table", thead, m("tbody", rows))];
-    },
+    }
   };
 };
 

ui/components/parameter.ts

@@ -28,49 +28,31 @@ import { getIcon } from "../icons";
 
 const evaluateParam = memoize((exp, obj, now: number) => {
   let timestamp = now;
-  exp = expression.evaluate(exp, null, now, (e) => {
-    if (!Array.isArray(e)) return e;
-    for (let i = 1; i < e.length; ++i) {
-      if (
-        Array.isArray(e[i]) &&
-        e[i][0] === "PARAM" &&
-        !Array.isArray(e[i][1])
-      ) {
-        let v = null;
-        const p = obj[e[i][1]];
-        if (p && p.value) {
-          v = p.value[0];
+  const params = new Set();
+  const value = expression.evaluate(exp, obj, now, e => {
+    if (Array.isArray(e)) {
+      if (e[0] === "PARAM") {
+        params.add(e[1]);
+        if (!Array.isArray(e[1])) {
+          const p = obj[e[1]];
+          if (p && p.valueTimestamp)
             timestamp = Math.min(timestamp, p.valueTimestamp);
         }
-        e = e.slice();
-        e[i] = v;
-      }
       }
       if (e[0] === "FUNC" && e[1] === "DATE_STRING" && !Array.isArray(e[2]))
         return new Date(e[2]).toLocaleString();
+    }
     return e;
   });
 
-  let parameter = null;
-  let value = null;
-
-  if (!Array.isArray(exp)) {
-    value = exp;
-  } else if (exp[0] === "PARAM") {
-    const p = obj[exp[1]];
-    if (p && p.value) {
-      timestamp = p.valueTimestamp;
-      value = p.value[0];
-      parameter = exp[1];
-    }
-  }
-
+  let parameter = params.size === 1 ? params.values().next().value : null;
+  if (Array.isArray(parameter)) parameter = null;
   return { value, timestamp, parameter };
 });
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       const device = vnode.attrs["device"];
 
       const { value, timestamp, parameter } = evaluateParam(
@@ -90,16 +72,16 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageSpv({
                 name: "setParameterValues",
-                devices: [device["DeviceID.ID"].value[0]],
+                device: device["DeviceID.ID"].value[0],
                 parameterValues: [
                   [
                     parameter,
                     device[parameter].value[0],
-                    device[parameter].value[1],
-                  ],
-                ],
+                    device[parameter].value[1]
+                  ]
+                ]
               });
-            },
+            }
           },
           getIcon("edit")
         );
@@ -111,7 +93,7 @@ const component: ClosureComponent = (): Component => {
         "span",
         {
           class: "parameter-value",
-          onmouseover: (e) => {
+          onmouseover: e => {
             e.redraw = false;
             // Don't update any child element
             if (e.target === (el as VnodeDOM).dom) {
@@ -119,12 +101,12 @@ const component: ClosureComponent = (): Component => {
               const localeString = new Date(timestamp).toLocaleString();
               e.target.title = `${localeString} (${timeAgo(now - timestamp)})`;
             }
-          },
+          }
         },
         el,
         edit
       );
-    },
+    }
   };
 };
 

ui/components/ping.ts

@@ -27,35 +27,30 @@ const component: ClosureComponent = (vn): Component => {
   let interval: ReturnType<typeof setInterval>;
   let host: string;
 
-  const refresh = (): void => {
-    if (!host) {
-      const dom = (vn as VnodeDOM).dom;
-      if (dom) dom.innerHTML = "";
-      return;
-    }
-
+  const refresh = async (): Promise<void> => {
     let status = "";
-    store
-      .ping(host)
-      .then((res) => {
+    if (host) {
+      try {
+        const res = await store.ping(host);
         if (res["avg"] != null) status = `${Math.trunc(res["avg"])} ms`;
         else status = "Unreachable";
-      })
-      .catch(() => {
-        status = "Error!";
-        clearInterval(interval);
-      })
-      .finally(() => {
+      } catch (err) {
+        setTimeout(() => {
+          throw err;
+        }, 0);
+      }
+    }
+
     const dom = (vn as VnodeDOM).dom;
+
     if (dom) dom.innerHTML = `Pinging ${host}: ${status}`;
-      });
   };
 
   return {
     onremove: () => {
       clearInterval(interval);
     },
-    view: (vnode) => {
+    view: vnode => {
       const device = vnode.attrs["device"];
       let param =
         device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"];
@@ -78,7 +73,7 @@ const component: ClosureComponent = (vn): Component => {
       }
 
       return m("div", `Pinging ${host}:`);
-    },
+    }
   };
 };
 

ui/components/summon-button.ts

@@ -25,26 +25,26 @@ import * as notifications from "../notifications";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       const device = vnode.attrs["device"];
 
       return m(
         "button.primary",
         {
           title: "Initiate session and refresh basic parameters",
-          onclick: (e) => {
+          onclick: e => {
             e.target.disabled = true;
             const params = Object.values(vnode.attrs["parameters"])
-              .map((exp) => {
+              .map(exp => {
                 if (!Array.isArray(exp) || exp[0] !== "PARAM") return null;
-                return store.evaluateExpression(exp[1], device) as string;
+                return store.evaluateExpression(exp[1], device);
               })
-              .filter((exp) => !!exp);
+              .filter(exp => !!exp);
 
             const task = {
               name: "getParameterValues",
               parameterNames: params,
-              device: device["DeviceID.ID"].value[0],
+              device: device["DeviceID.ID"].value[0]
             };
 
             taskQueue
@@ -79,16 +79,12 @@ const component: ClosureComponent = (): Component => {
               .then(() => {
                 e.target.disabled = false;
                 store.fulfill(0, Date.now());
-              })
-              .catch((err) => {
-                e.target.disabled = false;
-                notifications.push("error", err.message);
               });
-          },
+          }
         },
         "Summon"
       );
-    },
+    }
   };
 };
 

ui/components/tags.ts

@@ -22,38 +22,32 @@ import { m } from "../components";
 import * as notifications from "../notifications";
 import * as store from "../store";
 import { getIcon } from "../icons";
-import { decodeTag } from "../../lib/common";
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       const device = vnode.attrs["device"];
       let writable = true;
       if (vnode.attrs["writable"] != null) writable = vnode.attrs["writable"];
 
       const tags = [];
       for (const p of Object.keys(device))
-        if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
+        if (p.startsWith("Tags.")) tags.push(p.slice(5));
 
       tags.sort();
 
-      if (!writable) {
-        return m(
-          ".tags",
-          tags.map((t) => m("span.tag", t))
-        );
-      }
+      if (!writable) return m(".tags", tags.map(t => m("span.tag", t)));
 
       return m(
         ".tags",
-        tags.map((tag) =>
+        tags.map(tag =>
           m(
             "span.tag",
             tag,
             m(
               "button",
               {
-                onclick: (e) => {
+                onclick: e => {
                   e.target.disabled = true;
                   const deviceId = device["DeviceID.ID"].value[0];
                   store
@@ -66,14 +60,14 @@ const component: ClosureComponent = (): Component => {
                       );
                       store.fulfill(0, Date.now());
                     })
-                    .catch((err) => {
+                    .catch(err => {
                       e.target.disabled = false;
                       notifications.push(
                         "error",
                         `${deviceId}: ${err.message}`
                       );
                     });
-                },
+                }
               },
               getIcon("remove")
             )
@@ -85,7 +79,7 @@ const component: ClosureComponent = (): Component => {
           m(
             "button",
             {
-              onclick: (e) => {
+              onclick: e => {
                 e.target.disabled = true;
                 const deviceId = device["DeviceID.ID"].value[0];
                 const tag = prompt(`Enter tag to assign to device:`);
@@ -101,17 +95,17 @@ const component: ClosureComponent = (): Component => {
                     notifications.push("success", `${deviceId}: Tags updated`);
                     store.fulfill(0, Date.now());
                   })
-                  .catch((err) => {
+                  .catch(err => {
                     e.target.disabled = false;
                     notifications.push("error", `${deviceId}: ${err.message}`);
                   });
-              },
+              }
             },
             getIcon("add")
           )
         )
       );
-    },
+    }
   };
 };
 

ui/config-functions.ts

@@ -27,7 +27,7 @@ interface Diff {
   remove: string[];
 }
 
-export function flattenConfig(config: Record<string, unknown>): any {
+export function flattenConfig(config): any {
   const flatten = {};
   const recuresive = (obj: any, root: string): void => {
     for (const [k, v] of Object.entries(obj)) {
@@ -42,35 +42,6 @@ export function flattenConfig(config: Record<string, unknown>): any {
   return flatten;
 }
 
-// Order keys such that nested objects come last
-function orderKeys(config: any): number {
-  let res = 1;
-  if (typeof config !== "object") return res;
-  if (Array.isArray(config)) {
-    for (const c of config) res += orderKeys(c);
-    return res;
-  }
-
-  const weights: [string, number][] = Object.entries(config).map(([k, v]) => [
-    k,
-    orderKeys(v),
-  ]);
-
-  weights.sort((a, b) => {
-    if (a[1] !== b[1]) return a[1] - b[1];
-    if (b[0] > a[0]) return -1;
-    else return 1;
-  });
-
-  for (const [k, w] of weights) {
-    res += w;
-    const v = config[k];
-    delete config[k];
-    config[k] = v;
-  }
-  return res;
-}
-
 export function structureConfig(config: Config[]): any {
   config.sort((a, b) => (a._id > b._id ? 1 : a._id < b._id ? -1 : 0));
   const _config = {};
@@ -122,18 +93,13 @@ export function structureConfig(config: Config[]): any {
     return object;
   };
 
-  const res = toArray(_config);
-  orderKeys(res);
-  return res;
+  return toArray(_config);
 }
 
-export function diffConfig(
-  current: Record<string, unknown>,
-  target: Record<string, unknown>
-): Diff {
+export function diffConfig(current, target): Diff {
   const diff = {
     add: [],
-    remove: [],
+    remove: []
   };
 
   for (const [k, v] of Object.entries(target))

ui/config-page.ts

@@ -30,7 +30,7 @@ import { getIcon } from "./icons";
 
 const attributes = [
   { id: "_id", label: "Key" },
-  { id: "value", label: "Value", type: "textarea" },
+  { id: "value", label: "Value", type: "textarea" }
 ];
 
 interface ValidationErrors {
@@ -52,13 +52,13 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
         object.value = stringify(parse(object.value || ""));
       } catch (err) {
         return void resolve({
-          value: "Config value must be valid expression",
+          value: "Config value must be valid expression"
         });
       }
 
       store
         .resourceExists("config", id)
-        .then((exists) => {
+        .then(exists => {
           if (exists && isNew) {
             store.fulfill(0, Date.now());
             return void resolve({ _id: "Config already exists" });
@@ -90,7 +90,7 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
           store.fulfill(0, Date.now());
           resolve();
         })
-        .catch((err) => {
+        .catch(err => {
           store.fulfill(0, Date.now());
           reject(err);
         });
@@ -102,14 +102,14 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
 
 const formData = {
   resource: "config",
-  attributes: attributes,
+  attributes: attributes
 };
 
 function escapeRegExp(str): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
 
-export function init(): Promise<Record<string, unknown>> {
+export function init(): Promise<{}> {
   if (!window.authorizer.hasAccess("config", 2)) {
     return Promise.reject(
       new Error("You are not authorized to view this page")
@@ -131,9 +131,9 @@ function renderTable(confsResponse, searchString): Children {
 
   let regex;
   if (searchString) {
-    const keywords = searchString.split(" ").filter((s) => s);
+    const keywords = searchString.split(" ").filter(s => s);
     if (keywords.length)
-      regex = new RegExp(keywords.map((s) => escapeRegExp(s)).join(".*"), "i");
+      regex = new RegExp(keywords.map(s => escapeRegExp(s)).join(".*"), "i");
   }
 
   const rows = [];
@@ -147,16 +147,16 @@ function renderTable(confsResponse, searchString): Children {
       {
         title: "Edit config value",
         onclick: () => {
-          let cb: () => Children = null;
-          const comp = m(
+          const cb = (): Children => {
+            return m(
               putFormComponent,
               Object.assign(
                 {
                   base: conf,
                   actionHandler: (action, object) => {
-                  return new Promise((resolve) => {
+                    return new Promise(resolve => {
                       putActionHandler(action, object, false)
-                      .then((errors) => {
+                        .then(errors => {
                           const ErrorList = errors ? Object.values(errors) : [];
                           if (ErrorList.length) {
                             for (const err of ErrorList)
@@ -166,24 +166,19 @@ function renderTable(confsResponse, searchString): Children {
                           }
                           resolve();
                         })
-                      .catch((err) => {
+                        .catch(err => {
                           notifications.push("error", err.message);
                           resolve();
                         });
                     });
-                },
+                  }
                 },
                 formData
               )
             );
-          cb = () => comp;
-          overlay.open(
-            cb,
-            () =>
-              !comp.state["current"]["modified"] ||
-              confirm("You have unsaved changes. Close anyway?")
-          );
-        },
+          };
+          overlay.open(cb);
+        }
       },
       getIcon("edit")
     );
@@ -195,10 +190,10 @@ function renderTable(confsResponse, searchString): Children {
         onclick: () => {
           if (!confirm(`Deleting ${conf._id} config. Are you sure?`)) return;
 
-          putActionHandler("delete", conf).catch((err) => {
+          putActionHandler("delete", conf).catch(err => {
             throw err;
           });
-        },
+        }
       },
       getIcon("remove")
     );
@@ -224,18 +219,18 @@ function renderTable(confsResponse, searchString): Children {
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       document.title = "Config - GenieACS";
 
       const search = m("input", {
         type: "text",
         placeholder: "Search config",
-        oninput: (e) => {
+        oninput: e => {
           vnode.state["searchString"] = e.target.value;
           e.redraw = false;
           clearTimeout(vnode.state["timeout"]);
           vnode.state["timeout"] = setTimeout(m.redraw, 250);
-        },
+        }
       });
 
       const confs = store.fetch("config", true);
@@ -249,15 +244,15 @@ export const component: ClosureComponent = (): Component => {
           {
             title: "Create new config",
             onclick: () => {
-              let cb: () => Children = null;
-              const comp = m(
+              const cb = (): Children => {
+                return m(
                   putFormComponent,
                   Object.assign(
                     {
                       actionHandler: (action, object) => {
-                      return new Promise((resolve) => {
+                        return new Promise(resolve => {
                           putActionHandler(action, object, true)
-                          .then((errors) => {
+                            .then(errors => {
                               const errorList = errors
                                 ? Object.values(errors)
                                 : [];
@@ -269,24 +264,19 @@ export const component: ClosureComponent = (): Component => {
                               }
                               resolve();
                             })
-                          .catch((err) => {
+                            .catch(err => {
                               notifications.push("error", err.message);
                               resolve();
                             });
                         });
-                    },
+                      }
                     },
                     formData
                   )
                 );
-              cb = () => comp;
-              overlay.open(
-                cb,
-                () =>
-                  !comp.state["current"]["modified"] ||
-                  confirm("You have unsaved changes. Close anyway?")
-              );
-            },
+              };
+              overlay.open(cb);
+            }
           },
           "New config"
         );
@@ -296,7 +286,7 @@ export const component: ClosureComponent = (): Component => {
           { name: "charts", prefix: "ui.overview.charts.", data: [] },
           { name: "filters", prefix: "ui.filters.", data: [] },
           { name: "index page", prefix: "ui.index.", data: [] },
-          { name: "device page", prefix: "ui.device.", data: [] },
+          { name: "device page", prefix: "ui.device.", data: [] }
         ];
 
         if (confs.fulfilled) {
@@ -317,12 +307,12 @@ export const component: ClosureComponent = (): Component => {
               "button",
               {
                 onclick: () => {
-                  let cb: () => Children = null;
-                  const comp = m(
+                  const cb = (): Children => {
+                    return m(
                       uiConfigComponent,
                       Object.assign(
                         {
-                        onUpdate: (errs: string[]) => {
+                          onUpdate: errs => {
                             const errors = errs ? Object.values(errs) : [];
                             if (errors.length) {
                               for (const err of errors)
@@ -339,23 +329,18 @@ export const component: ClosureComponent = (): Component => {
                             }
                             store.fulfill(0, Date.now());
                           },
-                        onError: (err) => {
+                          onError: err => {
                             notifications.push("error", err.message);
                             store.fulfill(0, Date.now());
                             overlay.close(cb);
-                        },
+                          }
                         },
                         attrs
                       )
                     );
-                  cb = () => comp;
-                  overlay.open(
-                    cb,
-                    () =>
-                      !comp.state["modified"] ||
-                      confirm("You have unsaved changes. Close anyway?")
-                  );
-                },
+                  };
+                  overlay.open(cb);
+                }
               },
               `Edit ${sub.name}`
             )
@@ -374,8 +359,8 @@ export const component: ClosureComponent = (): Component => {
             renderTable(confs, vnode.state["searchString"])
           ),
           m(".actions-bar", [newConfig].concat(subs))
-        ),
+        )
       ];
-    },
+    }
   };
 };

ui/css/app.css

@@ -106,7 +106,7 @@ body {
 
   & > .drawer-wrapper {
     position: fixed;
-    z-index: 101;
+    z-index: 10;
     width: 720px;
     margin-left: -360px;
     top: 0;
@@ -178,8 +178,6 @@ body {
       }
 
       & > .queue {
-        max-height: 400px;
-        overflow-y: auto;
         & > .pending {
           animation: 0.5s linear 0s infinite alternate pending-blink;
           border-radius: 0.4em;
@@ -462,7 +460,7 @@ input {
   margin-bottom: 0.5em;
   & > input {
     font-size: 90%;
-    width: 35em;
+    width: 30em;
     margin: 2px;
     padding: 2px;
   }
@@ -564,7 +562,7 @@ span.na {
     font-family: monospace;
     margin-bottom: 2px;
     padding: 4px;
-    width: 700px;
+    width: 600px;
   }
 
   & > .parameter-list {

ui/device-page.ts

@@ -22,9 +22,7 @@ import { m } from "./components";
 import config from "./config";
 import * as store from "./store";
 
-export function init(
-  args: Record<string, unknown>
-): Promise<Record<string, unknown>> {
+export function init(args): Promise<{}> {
   if (!window.authorizer.hasAccess("devices", 2)) {
     return Promise.reject(
       new Error("You are not authorized to view this page")
@@ -33,13 +31,13 @@ export function init(
 
   return Promise.resolve({
     deviceId: args.id,
-    deviceFilter: ["=", ["PARAM", "DeviceID.ID"], args.id],
+    deviceFilter: ["=", ["PARAM", "DeviceID.ID"], args.id]
   });
 }
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       document.title = `${vnode.attrs["deviceId"]} - Devices - GenieACS`;
 
       const dev = store.fetch("devices", vnode.attrs["deviceFilter"]).value;
@@ -51,6 +49,6 @@ export const component: ClosureComponent = (): Component => {
         cmps.push(m.context({ device: dev[0] }, c["type"], c));
 
       return [m("h1", vnode.attrs["deviceId"]), cmps];
-    },
+    }
   };
 };

ui/devices-page.ts

@@ -23,7 +23,7 @@ import config from "./config";
 import indexTableComponent from "./index-table-component";
 import filterComponent from "./filter-component";
 import * as store from "./store";
-import { queueTask, stageDownload } from "./task-queue";
+import * as taskQueue from "./task-queue";
 import * as notifications from "./notifications";
 import { parse, stringify } from "../lib/common/expression-parser";
 import { evaluate, extractParams } from "../lib/common/expression";
@@ -35,7 +35,7 @@ const PAGE_SIZE = config.ui.pageSize || 10;
 
 const memoizedParse = memoize(parse);
 const memoizedJsonParse = memoize(JSON.parse);
-const memoizedGetSortable = memoize((p) => {
+const memoizedGetSortable = memoize(p => {
   const expressionParams = extractParams(p);
   if (expressionParams.length === 1) {
     const param = evaluate(expressionParams[0]);
@@ -47,23 +47,21 @@ const memoizedGetSortable = memoize((p) => {
 const getDownloadUrl = memoize((filter, indexParameters) => {
   const columns = {};
   for (const p of indexParameters) columns[p.label] = stringify(p.parameter);
-  return `api/devices.csv?${m.buildQueryString({
+  return `/api/devices.csv?${m.buildQueryString({
     filter: stringify(filter),
-    columns: JSON.stringify(columns),
+    columns: JSON.stringify(columns)
   })}`;
 });
 
-const unpackSmartQuery = memoize((query) => {
-  return expressionParser.map(query, (e) => {
+const unpackSmartQuery = memoize(query => {
+  return expressionParser.map(query, e => {
     if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "Q")
       return smartQuery.unpack("devices", e[2], e[3]);
     return e;
   });
 });
 
-export function init(
-  args: Record<string, unknown>
-): Promise<Record<string, unknown>> {
+export function init(args): Promise<{}> {
   return new Promise((resolve, reject) => {
     if (!window.authorizer.hasAccess("devices", 2))
       return void reject(new Error("You are not authorized to view this page"));
@@ -74,14 +72,14 @@ export function init(
     if (!indexParameters.length) {
       indexParameters.push({
         label: "ID",
-        parameter: ["PARAM", "DeviceID.ID"],
+        parameter: ["PARAM", "DeviceID.ID"]
       });
     }
     resolve({ filter, indexParameters, sort });
   });
 }
 
-function renderActions(selected: Set<string>): Children {
+function renderActions(selected): Children {
   const buttons = [];
 
   buttons.push(
@@ -91,12 +89,13 @@ function renderActions(selected: Set<string>): Children {
         title: "Reboot selected devices",
         disabled: !selected.size,
         onclick: () => {
-          const tasks = [...selected].map((s) => ({
+          for (const d of selected) {
+            taskQueue.queueTask({
               name: "reboot",
-            device: s,
-          }));
-          queueTask(...tasks);
-        },
+              device: d
+            });
+          }
+        }
       },
       "Reboot"
     )
@@ -109,34 +108,18 @@ function renderActions(selected: Set<string>): Children {
         title: "Factory reset selected devices",
         disabled: !selected.size,
         onclick: () => {
-          const tasks = [...selected].map((s) => ({
+          for (const d of selected) {
+            taskQueue.queueTask({
               name: "factoryReset",
-            device: s,
-          }));
-          queueTask(...tasks);
-        },
+              device: d
+            });
+          }
+        }
       },
       "Reset"
     )
   );
 
-  buttons.push(
-    m(
-      "button.critical",
-      {
-        title: "Push a firmware or a config file",
-        disabled: !selected.size,
-        onclick: () => {
-          stageDownload({
-            name: "download",
-            devices: [...selected],
-          });
-        },
-      },
-      "Push file"
-    )
-  );
-
   buttons.push(
     m(
       "button.primary",
@@ -156,13 +139,13 @@ function renderActions(selected: Set<string>): Children {
                 notifications.push("success", `${id}: Deleted`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               })
-              .catch((err) => {
+              .catch(err => {
                 notifications.push("error", `${id}: ${err.message}`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
           if (--counter === 0) store.fulfill(0, Date.now());
-        },
+        }
       },
       "Delete"
     )
@@ -188,13 +171,13 @@ function renderActions(selected: Set<string>): Children {
                 notifications.push("success", `${id}: Tags updated`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               })
-              .catch((err) => {
+              .catch(err => {
                 notifications.push("error", `${id}: ${err.message}`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
           if (--counter === 0) store.fulfill(0, Date.now());
-        },
+        }
       },
       "Tag"
     )
@@ -222,13 +205,13 @@ function renderActions(selected: Set<string>): Children {
                 notifications.push("success", `${id}: Tags updated`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               })
-              .catch((err) => {
+              .catch(err => {
                 notifications.push("error", `${id}: ${err.message}`);
                 if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
           if (--counter === 0) store.fulfill(0, Date.now());
-        },
+        }
       },
       "Untag"
     )
@@ -239,7 +222,7 @@ function renderActions(selected: Set<string>): Children {
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       document.title = "Devices - GenieACS";
       const attributes = vnode.attrs["indexParameters"];
 
@@ -290,7 +273,7 @@ export const component: ClosureComponent = (): Component => {
 
       const devs = store.fetch("devices", filter, {
         limit: vnode.state["showCount"] || PAGE_SIZE,
-        sort: sort,
+        sort: sort
       });
       const count = store.count("devices", filter);
 
@@ -319,7 +302,7 @@ export const component: ClosureComponent = (): Component => {
           {
             href: `#!/devices/${encodeURIComponent(
               device["DeviceID.ID"].value[0]
-            )}`,
+            )}`
           },
           "Show"
         );
@@ -336,8 +319,8 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing devices"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs),
+        m(indexTableComponent, attrs)
       ];
-    },
+    }
   };
 };

ui/drawer-component.ts

@@ -22,23 +22,12 @@ import m, {
   Child,
   ClosureComponent,
   Component,
-  VnodeDOM,
+  VnodeDOM
 } from "mithril";
+import * as taskQueue from "./task-queue";
 import * as store from "./store";
 import * as notifications from "./notifications";
 import { getIcon } from "./icons";
-import {
-  clear,
-  commit,
-  deleteTask,
-  getQueue,
-  getStaging,
-  QueueTask,
-  queueTask,
-  StageTask,
-} from "./task-queue";
-
-const invalid: WeakSet<StageTask> = new WeakSet();
 
 function mparam(param): Children {
   return m("span.parameter", { title: param }, `${param}`);
@@ -48,11 +37,11 @@ function mval(val): Children {
   return m("span.value", { title: val }, `${val}`);
 }
 
-function renderStagingSpv(task: StageTask, queueFunc, cancelFunc): Children {
-  function keydown(e: KeyboardEvent): void {
-    if (e.key === "Enter") queueFunc();
-    else if (e.key === "Escape") cancelFunc();
-    else e["redraw"] = false;
+function renderStagingSpv(task, queueFunc, cancelFunc): Children {
+  function keydown(e): void {
+    if (e.keyCode === 13) queueFunc();
+    else if (e.keyCode === 27) cancelFunc();
+    else e.redraw = false;
   }
 
   let input;
@@ -61,18 +50,18 @@ function renderStagingSpv(task: StageTask, queueFunc, cancelFunc): Children {
       "select",
       {
         value: task.parameterValues[0][1].toString(),
-        onchange: (e) => {
+        onchange: e => {
           e.redraw = false;
           task.parameterValues[0][1] = input.dom.value;
         },
         onkeydown: keydown,
-        oncreate: (vnode) => {
+        oncreate: vnode => {
           (vnode.dom as HTMLSelectElement).focus();
-        },
+        }
       },
       [
         m("option", { value: "true" }, "true"),
-        m("option", { value: "false" }, "false"),
+        m("option", { value: "false" }, "false")
       ]
     );
   } else {
@@ -83,42 +72,31 @@ function renderStagingSpv(task: StageTask, queueFunc, cancelFunc): Children {
         ? "number"
         : "text",
       value: task.parameterValues[0][1],
-      oninput: (e) => {
+      oninput: e => {
         e.redraw = false;
         task.parameterValues[0][1] = input.dom.value;
       },
       onkeydown: keydown,
-      oncreate: (vnode) => {
+      oncreate: vnode => {
         (vnode.dom as HTMLInputElement).focus();
         (vnode.dom as HTMLInputElement).select();
         // Need to prevent scrolling on focus because
         // we're animating height and using overflow: hidden
         (vnode.dom.parentNode.parentNode as Element).scrollTop = 0;
-      },
+      }
     });
   }
 
   return [m("span", "Editing ", mparam(task.parameterValues[0][0])), input];
 }
 
-function renderStagingDownload(task: StageTask): Children {
-  if (!task.fileName || !task.fileType) invalid.add(task);
-  else invalid.delete(task);
+function renderStagingDownload(task): Children {
+  task.invalid = !task.fileName || !task.fileType;
   const files = store.fetch("files", true);
-  let oui = "";
-  let productClass = "";
-  for (const d of task.devices) {
-    const parts = d.split("-");
-    if (oui === "") oui = parts[0];
-    else if (oui !== parts[0]) oui = null;
-    if (parts.length === 3) {
-      if (productClass === "") productClass = parts[1];
-      else if (productClass !== parts[1]) productClass = null;
-    }
-  }
-
-  if (oui) oui = decodeURIComponent(oui);
-  if (productClass) productClass = decodeURIComponent(productClass);
+  const idParts = task.device.split("-");
+  const oui = decodeURIComponent(idParts[0]);
+  const productClass =
+    idParts.length === 3 ? decodeURIComponent(idParts[1]) : "";
 
   const typesList = [
     "",
@@ -126,8 +104,8 @@ function renderStagingDownload(task: StageTask): Children {
     "2 Web Content",
     "3 Vendor Configuration File",
     "4 Tone File",
-    "5 Ringer File",
-  ].map((t) =>
+    "5 Ringer File"
+  ].map(t =>
     m(
       "option",
       { disabled: !t, value: t, selected: (task.fileType || "") === t },
@@ -139,14 +117,14 @@ function renderStagingDownload(task: StageTask): Children {
     .concat(
       files.value
         .filter(
-          (f) =>
+          f =>
             (!f["metadata.oui"] || f["metadata.oui"] === oui) &&
             (!f["metadata.productClass"] ||
               f["metadata.productClass"] === productClass)
         )
-        .map((f) => f._id)
+        .map(f => f._id)
     )
-    .map((f) =>
+    .map(f =>
       m(
         "option",
         { disabled: !f, value: f, selected: (task.fileName || "") === f },
@@ -159,7 +137,7 @@ function renderStagingDownload(task: StageTask): Children {
     m(
       "select",
       {
-        onchange: (e) => {
+        onchange: e => {
           const f = e.target.value;
           task.fileName = f;
           task.fileType = "";
@@ -167,7 +145,7 @@ function renderStagingDownload(task: StageTask): Children {
             if (file._id === f) task.fileType = file["metadata.fileType"];
         },
         disabled: files.fulfilling,
-        style: "width: 350px",
+        style: "width: 350px"
       },
       filesList
     ),
@@ -175,142 +153,22 @@ function renderStagingDownload(task: StageTask): Children {
     m(
       "select",
       {
-        onchange: (e) => {
+        onchange: e => {
           task.fileType = e.target.value;
-        },
+        }
       },
       typesList
-    ),
+    )
   ];
 }
 
-const uploadTypesForDevice: WeakMap<
-  Record<string, unknown>,
-  Record<string, string>
-> = new WeakMap();
-function getUploadTypesForDevice(
-  device: Record<string, unknown>
-): Record<string, string> {
-  if (uploadTypesForDevice.has(device)) return uploadTypesForDevice.get(device);
-  const instances: Set<string> = new Set();
-  const filenames: Record<string, string> = {};
-  const versions: Record<string, string> = {};
-
-  if (device) {
-    for (const [k, v] of Object.entries(device)) {
-      if (/DeviceInfo\.VendorConfigFile\.[0-9]+(\.Name|\.Version)?$/.test(k)) {
-        const p = k.split(".");
-        if (p[p.length - 1] === "Name") {
-          const n = `3 Vendor Configuration File ${p[p.length - 2]}`;
-          instances.add(n);
-          if (v["value"]) filenames[n] = v["value"][0];
-        } else if (p[p.length - 1] === "Version") {
-          const n = `3 Vendor Configuration File ${p[p.length - 2]}`;
-          instances.add(n);
-          if (v["value"]) versions[n] = v["value"][0];
-        } else {
-          const n = `3 Vendor Configuration File ${p[p.length - 1]}`;
-          instances.add(n);
-        }
-      } else if (/DeviceInfo\.VendorLogFile\.[0-9]+(\.Name)?$/.test(k)) {
-        const p = k.split(".");
-        if (p[p.length - 1] === "Name") {
-          const n = `4 Vendor Log File ${p[p.length - 2]}`;
-          instances.add(n);
-          if (v["value"]) filenames[n] = v["value"][0];
-        } else {
-          const n = `4 Vendor Log File ${p[p.length - 1]}`;
-          instances.add(n);
-        }
-      }
-    }
-  }
-
-  if (!instances.size) {
-    instances.add("1 Vendor Configuration File");
-    instances.add("2 Vendor Log File");
-  }
-
-  const res: Record<string, string> = {};
-
-  for (const i of instances) {
-    let n = filenames[i];
-    const v = versions[i];
-    if (!n) n = i.replace(/\s+/g, "_") + ".txt";
-    if (v) n = `${n}-${v}`;
-    res[i] = n;
-  }
-
-  uploadTypesForDevice.set(device, res);
-  return res;
-}
-
-function renderStagingUpload(task: StageTask): Children {
-  if (!task.fileName || !task.fileType) invalid.add(task);
-  else invalid.delete(task);
-
-  const commonTypes: Record<string, number> = {};
-
-  const filenames: Record<string, string> = {};
-
-  for (const d of task.devices) {
-    const dev = store.fetch("devices", ["=", ["PARAM", "DeviceID.ID"], d])
-      .value[0];
-    if (dev) {
-      const u = getUploadTypesForDevice(dev);
-      Object.assign(filenames, u);
-      for (const k of Object.keys(u))
-        commonTypes[k] = (commonTypes[k] || 0) + 1;
-    }
-  }
-
-  const typesList = [""];
-  typesList.push(
-    ...Object.entries(commonTypes)
-      .filter(([, c]) => c === task.devices.length)
-      .map(([k]) => k)
-      .sort()
-  );
-
-  return [
-    "Fetch ",
-    m(
-      "select",
-      {
-        onchange: (e) => {
-          task.fileType = e.target.value;
-          task.fileName = filenames[task.fileType];
-        },
-      },
-      typesList.map((t) =>
-        m(
-          "option",
-          { disabled: !t, value: t, selected: (task.fileType || "") === t },
-          t
-        )
-      )
-    ),
-    " as ",
-    m("input", {
-      onchange: (e) => {
-        task.fileName = e.target.value;
-      },
-      value: task.fileName || "",
-    }),
-  ];
-}
-
-function renderStaging(staging: Set<StageTask>): Child[] {
+function renderStaging(staging): Child[] {
   const elements: Child[] = [];
 
   for (const s of staging) {
     const queueFunc = (): void => {
       staging.delete(s);
-      for (const d of s.devices) {
-        const t = Object.assign({ device: d }, s);
-        delete t.devices;
-        queueTask(t);
-      }
+      taskQueue.queueTask(s);
     };
     const cancelFunc = (): void => {
       staging.delete(s);
@@ -320,11 +178,10 @@ function renderStaging(staging: Set<StageTask>): Child[] {
     if (s.name === "setParameterValues")
       elms = renderStagingSpv(s, queueFunc, cancelFunc);
     else if (s.name === "download") elms = renderStagingDownload(s);
-    else if (s.name === "upload") elms = renderStagingUpload(s);
 
     const queue = m(
       "button.primary",
-      { title: "Queue task", onclick: queueFunc, disabled: invalid.has(s) },
+      { title: "Queue task", onclick: queueFunc, disabled: s.invalid },
       "Queue"
     );
     const cancel = m(
@@ -338,7 +195,7 @@ function renderStaging(staging: Set<StageTask>): Child[] {
   return elements;
 }
 
-function renderQueue(queue: Set<QueueTask>): Child[] {
+function renderQueue(queue): Child[] {
   const details: Child[] = [];
   const devices: { [deviceId: string]: any[] } = {};
   for (const t of queue) {
@@ -358,8 +215,8 @@ function renderQueue(queue: Set<QueueTask>): Child[] {
             {
               title: "Retry this task",
               onclick: () => {
-                queueTask(t);
-              },
+                taskQueue.queueTask(t);
+              }
             },
             getIcon("retry")
           )
@@ -372,8 +229,8 @@ function renderQueue(queue: Set<QueueTask>): Child[] {
           {
             title: "Remove this task",
             onclick: () => {
-              deleteTask(t);
-            },
+              taskQueue.deleteTask(t);
+            }
           },
           getIcon("remove")
         )
@@ -440,14 +297,6 @@ function renderQueue(queue: Set<QueueTask>): Child[] {
             m(".actions", actions)
           )
         );
-      } else if (t.name === "upload") {
-        details.push(
-          m(
-            `div.${t.status}`,
-            `Fetch file ${t.fileName} (${t.fileType})`,
-            m(".actions", actions)
-          )
-        );
       } else {
         details.push(m(`div.${t.status}`, t.name, m(".actions", actions)));
       }
@@ -475,18 +324,18 @@ function renderNotifications(notifs): Child[] {
         {
           class: n.type,
           style: "position: absolute;opacity: 0",
-          oncreate: (vnode) => {
+          oncreate: vnode => {
             (vnode.dom as HTMLDivElement).style.opacity = "1";
           },
-          onbeforeremove: (vnode) => {
+          onbeforeremove: vnode => {
             (vnode.dom as HTMLDivElement).style.opacity = "0";
-            return new Promise((resolve) => {
+            return new Promise(resolve => {
               setTimeout(() => {
                 resolve();
               }, 500);
             });
           },
-          key: n.timestamp,
+          key: n.timestamp
         },
         m("div", buttons, n.message)
       )
@@ -497,9 +346,9 @@ function renderNotifications(notifs): Child[] {
 
 const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
-      const queue = getQueue();
-      const staging = getStaging();
+    view: vnode => {
+      const queue = taskQueue.getQueue();
+      const staging = taskQueue.getStaging();
       const notifs = notifications.getNotifications();
 
       let drawerElement, statusElement;
@@ -545,10 +394,11 @@ const component: ClosureComponent = (): Component => {
               title: "Commit queued tasks",
               disabled: !statusCount.queued,
               onclick: () => {
-                const tasks = Array.from(getQueue()).filter(
-                  (t) => t["status"] === "queued"
+                const tasks = Array.from(taskQueue.getQueue()).filter(
+                  t => t["status"] === "queued"
                 );
-                commit(
+                taskQueue
+                  .commit(
                     tasks,
                     (deviceId, err, connectionRequestStatus, tasks2) => {
                       if (err) {
@@ -591,11 +441,8 @@ const component: ClosureComponent = (): Component => {
                   )
                   .then(() => {
                     store.fulfill(0, Date.now());
-                  })
-                  .catch((err) => {
-                    notifications.push("error", err.message);
                   });
-              },
+              }
             },
             "Commit"
           ),
@@ -603,8 +450,8 @@ const component: ClosureComponent = (): Component => {
             "button",
             {
               title: "Clear tasks",
-              onclick: clear,
-              disabled: !queueElements.length,
+              onclick: taskQueue.clear,
+              disabled: !queueElements.length
             },
             "Clear"
           )
@@ -640,31 +487,31 @@ const component: ClosureComponent = (): Component => {
           {
             key: "drawer",
             style: "opacity: 0;height: 0;",
-            oncreate: (vnode2) => {
+            oncreate: vnode2 => {
               vnode.state["mouseIn"] = false;
               (vnode2.dom as HTMLDivElement).style.opacity = "1";
               resizeDrawer();
             },
-            onmouseover: (e) => {
+            onmouseover: e => {
               vnode.state["mouseIn"] = true;
               resizeDrawer();
               e.redraw = false;
             },
-            onmouseleave: (e) => {
+            onmouseleave: e => {
               vnode.state["mouseIn"] = false;
               resizeDrawer();
               e.redraw = false;
             },
             onupdate: resizeDrawer,
-            onbeforeremove: (vnode2) => {
+            onbeforeremove: vnode2 => {
               (vnode2.dom as HTMLDivElement).onmouseover = null;
               (vnode2.dom as HTMLDivElement).onmouseleave = null;
               (vnode2.dom as HTMLDivElement).style.opacity = "0";
               (vnode2.dom as HTMLDivElement).style.height = "0";
-              return new Promise((resolve) => {
+              return new Promise(resolve => {
                 setTimeout(resolve, 500);
               });
-            },
+            }
           },
           statusElement,
           stagingElements.length ? stagingElements : m(".queue", queueElements)
@@ -680,12 +527,12 @@ const component: ClosureComponent = (): Component => {
             key: "notifications",
             style: "position: relative;",
             onupdate: repositionNotifications,
-            oncreate: repositionNotifications,
+            oncreate: repositionNotifications
           },
           notificationElements
         )
       );
-    },
+    }
   };
 };
 

ui/dynamic-loader.ts

@@ -32,7 +32,7 @@ function onError(): void {
       {
         Reload: () => {
           window.location.reload();
-        },
+        }
       }
     );
   }
@@ -47,14 +47,14 @@ export function loadCodeMirror(): Promise<void> {
       import(
         /* webpackChunkName: "codemirror" */ "codemirror/mode/javascript/javascript"
       ),
-      import(/* webpackChunkName: "codemirror" */ "codemirror/mode/yaml/yaml"),
+      import(/* webpackChunkName: "codemirror" */ "codemirror/mode/yaml/yaml")
     ];
     Promise.all(promises)
-      .then((modules) => {
-        codeMirror = modules[0].default;
+      .then(modules => {
+        codeMirror = modules[0];
         resolve();
       })
-      .catch((err) => {
+      .catch(err => {
         onError();
         reject(err);
       });
@@ -66,11 +66,11 @@ export function loadYaml(): Promise<void> {
 
   return new Promise((resolve, reject) => {
     import(/* webpackChunkName: "yaml" */ "yaml")
-      .then((module) => {
+      .then(module => {
         yaml = module;
         resolve();
       })
-      .catch((err) => {
+      .catch(err => {
         onError();
         reject(err);
       });

ui/error-page.ts

@@ -25,6 +25,6 @@ export const component: ClosureComponent = (): Component => {
     view: function(vnode) {
       document.title = "Error! - GenieACS";
       return m("p.error", vnode.attrs["error"]);
-    },
+    }
   };
 };

ui/faults-page.ts

@@ -41,33 +41,31 @@ const attributes = [
   { id: "message", label: "Message" },
   { id: "detail", label: "Detail" },
   { id: "retries", label: "Retries" },
-  { id: "timestamp", label: "Timestamp" },
+  { id: "timestamp", label: "Timestamp" }
 ];
 
-const getDownloadUrl = memoize((filter) => {
+const getDownloadUrl = memoize(filter => {
   const cols = {};
   for (const attr of attributes) {
     cols[attr.label] =
       attr.id === "timestamp" ? `DATE_STRING(${attr.id})` : attr.id;
   }
 
-  return `api/faults.csv?${m.buildQueryString({
+  return `/api/faults.csv?${m.buildQueryString({
     filter: stringify(filter),
-    columns: JSON.stringify(cols),
+    columns: JSON.stringify(cols)
   })}`;
 });
 
-const unpackSmartQuery = memoize((query) => {
-  return map(query, (e) => {
+const unpackSmartQuery = memoize(query => {
+  return map(query, e => {
     if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "Q")
       return smartQuery.unpack("faults", e[2], e[3]);
     return e;
   });
 });
 
-export function init(
-  args: Record<string, unknown>
-): Promise<Record<string, unknown>> {
+export function init(args): Promise<{}> {
   if (!window.authorizer.hasAccess("faults", 2)) {
     return Promise.reject(
       new Error("You are not authorized to view this page")
@@ -87,7 +85,7 @@ export function init(
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       document.title = "Faults - GenieACS";
 
       function showMore(): void {
@@ -132,7 +130,7 @@ export const component: ClosureComponent = (): Component => {
 
       const faults = store.fetch("faults", filter, {
         limit: vnode.state["showCount"] || PAGE_SIZE,
-        sort: sort,
+        sort: sort
       });
       const count = store.count("faults", filter);
 
@@ -167,13 +165,13 @@ export const component: ClosureComponent = (): Component => {
       attrs["downloadUrl"] = downloadUrl;
 
       if (window.authorizer.hasAccess("faults", 3)) {
-        attrs["actionsCallback"] = (selected: Set<string>): Children => {
+        attrs["actionsCallback"] = (selected): Children => {
           return m(
             "button.primary",
             {
               disabled: selected.size === 0,
               title: "Delete selected faults",
-              onclick: (e) => {
+              onclick: e => {
                 e.redraw = false;
                 e.target.disabled = true;
 
@@ -181,22 +179,22 @@ export const component: ClosureComponent = (): Component => {
                   return;
 
                 Promise.all(
-                  Array.from(selected).map((id) =>
+                  Array.from(selected).map(id =>
                     store.deleteResource("faults", id)
                   )
                 )
-                  .then((res) => {
+                  .then(res => {
                     notifications.push(
                       "success",
                       `${res.length} faults deleted`
                     );
                     store.fulfill(0, Date.now());
                   })
-                  .catch((err) => {
+                  .catch(err => {
                     notifications.push("error", err.message);
                     store.fulfill(0, Date.now());
                   });
-              },
+              }
             },
             "Delete"
           );
@@ -211,8 +209,8 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing faults"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs),
+        m(indexTableComponent, attrs)
       ];
-    },
+    }
   };
 };

ui/files-page.ts

@@ -51,23 +51,23 @@ const attributes: {
       "2 Web Content",
       "3 Vendor Configuration File",
       "4 Tone File",
-      "5 Ringer File",
-    ],
+      "5 Ringer File"
+    ]
   },
   { id: "metadata.oui", label: "OUI" },
   { id: "metadata.productClass", label: "Product Class" },
-  { id: "metadata.version", label: "Version" },
+  { id: "metadata.version", label: "Version" }
 ];
 
 const formData = {
   resource: "files",
   attributes: attributes
     .slice(1) // remove _id from new object form
-    .concat([{ id: "file", label: "File", type: "file" }]),
+    .concat([{ id: "file", label: "File", type: "file" }])
 };
 
-const unpackSmartQuery = memoize((query) => {
-  return map(query, (e) => {
+const unpackSmartQuery = memoize(query => {
+  return map(query, e => {
     if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "Q")
       return smartQuery.unpack("files", e[2], e[3]);
     return e;
@@ -91,7 +91,7 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
 
       store
         .resourceExists("files", id)
-        .then((exists) => {
+        .then(exists => {
           if (exists) {
             store.fulfill(0, Date.now());
             return void resolve({ file: "File already exists" });
@@ -99,7 +99,7 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
           const headers = Object.assign(
             {
               "Content-Type": "application/octet-stream",
-              Accept: "application/octet-stream",
+              Accept: "application/octet-stream"
             },
             object
           );
@@ -108,9 +108,9 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
             .xhrRequest({
               method: "PUT",
               headers: headers,
-              url: `api/files/${encodeURIComponent(id)}`,
-              serialize: (body) => body, // Identity function to prevent JSON.parse on blob data
-              body: file,
+              url: `/api/files/${encodeURIComponent(id)}`,
+              serialize: body => body, // Identity function to prevent JSON.parse on blob data
+              body: file
             })
             .then(() => {
               notifications.push(
@@ -129,18 +129,16 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
   });
 }
 
-const getDownloadUrl = memoize((filter) => {
+const getDownloadUrl = memoize(filter => {
   const cols = {};
   for (const attr of attributes) cols[attr.label] = attr.id;
-  return `api/files.csv?${m.buildQueryString({
+  return `/api/files.csv?${m.buildQueryString({
     filter: stringify(filter),
-    columns: JSON.stringify(cols),
+    columns: JSON.stringify(cols)
   })}`;
 });
 
-export function init(
-  args: Record<string, unknown>
-): Promise<Record<string, unknown>> {
+export function init(args): Promise<{}> {
   if (!window.authorizer.hasAccess("files", 2)) {
     return Promise.reject(
       new Error("You are not authorized to view this page")
@@ -154,7 +152,7 @@ export function init(
 
 export const component: ClosureComponent = (): Component => {
   return {
-    view: (vnode) => {
+    view: vnode => {
       document.title = "Files - GenieACS";
 
       function showMore(): void {
@@ -197,7 +195,7 @@ export const component: ClosureComponent = (): Component => {
 
       const files = store.fetch("files", filter, {
         limit: vnode.state["showCount"] || PAGE_SIZE,
-        sort: sort,
+        sort: sort
       });
 
       const count = store.count("files", filter);
@@ -214,22 +212,22 @@ export const component: ClosureComponent = (): Component => {
       attrs["downloadUrl"] = downloadUrl;
 
       if (window.authorizer.hasAccess("files", 3)) {
-        attrs["actionsCallback"] = (selected: Set<string>): Children => {
+        attrs["actionsCallback"] = (selected): Children => {
           return [
             m(
               "button.primary",
               {
                 title: "Create new file",
                 onclick: () => {
-                  let cb: () => Children = null;
-                  const comp = m(
+                  const cb = (): Children => {
+                    return m(
                       putFormComponent,
                       Object.assign(
                         {
                           actionHandler: (action, object) => {
-                          return new Promise((resolve) => {
+                            return new Promise(resolve => {
                               putActionHandler(action, object)
-                              .then((errors) => {
+                                .then(errors => {
                                   const errorList = errors
                                     ? Object.values(errors)
                                     : [];
@@ -241,24 +239,19 @@ export const component: ClosureComponent = (): Component => {
                                   }
                                   resolve();
                                 })
-                              .catch((err) => {
+                                .catch(err => {
                                   notifications.push("error", err.message);
                                   resolve();
                                 });
                             });
-                        },
+                          }
                         },
                         formData
                       )
                     );
-                  cb = () => comp;
-                  overlay.open(
-                    cb,
-                    () =>
-                      !comp.state["current"]["modified"] ||
-                      confirm("You have unsaved changes. Close anyway?")
-                  );
-                },
+                  };
+                  overlay.open(cb);
+                }
               },
               "New"
             ),
@@ -267,7 +260,7 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Delete selected files",
                 disabled: !selected.size,
-                onclick: (e) => {
+                onclick: e => {
                   if (
                     !confirm(`Deleting ${selected.size} files. Are you sure?`)
                   )
@@ -276,25 +269,25 @@ export const component: ClosureComponent = (): Component => {
                   e.redraw = false;
                   e.target.disabled = true;
                   Promise.all(
-                    Array.from(selected).map((id) =>
+                    Array.from(selected).map(id =>
                       store.deleteResource("files", id)
                     )
                   )
-                    .then((res) => {
+                    .then(res => {
                       notifications.push(
                         "success",
                         `${res.length} files deleted`
                       );
                       store.fulfill(0, Date.now());
                     })
-                    .catch((err) => {
+                    .catch(err => {
                       notifications.push("error", err.message);
                       store.fulfill(0, Date.now());
                     });
-                },
+                }
               },
               "Delete"
-            ),
+            )
           ];
         };
       }
@@ -307,8 +300,8 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing files"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs),
+        m(indexTableComponent, attrs)
       ];
-    },
+    }
   };
 };