1.2.5

Lookbehind in regex is not supported in Safari browsers. This implementation
should be more efficient so why not.

.eslintignore

@@ -1,2 +0,0 @@
-!/bin/.*.ts
-!/tools/.*.ts

.eslintrc.json

@@ -8,19 +8,30 @@
     "project": "./tsconfig.json"
   },
   "plugins": ["@typescript-eslint"],
-  "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended", "prettier", "prettier/@typescript-eslint"],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
   "rules": {
     "no-shadow": ["error", { "allow": ["err"] }],
     "handle-callback-err": "error",
     "prefer-arrow-callback": "error",
     "curly": ["error", "multi-or-nest", "consistent"],
     "no-buffer-constructor": "error",
-    "prefer-const": ["error", {"destructuring": "all"}],
+    "prefer-const": ["error", { "destructuring": "all" }],
-    "eqeqeq": ["error", "always", {"null": "ignore"}],
+    "eqeqeq": ["error", "always", { "null": "ignore" }],
     "no-return-await": "error",
     "@typescript-eslint/no-explicit-any": 0,
-    "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
+    "@typescript-eslint/no-use-before-define": [
-    "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
+      "error",
+      { "functions": false }
+    ],
+    "@typescript-eslint/explicit-function-return-type": [
+      "error",
+      { "allowExpressions": true }
+    ],
+    "@typescript-eslint/no-floating-promises": "error",
     "require-atomic-updates": 0,
     "no-prototype-builtins": 0
   }

.gitignore

@@ -1,6 +1,4 @@
 *~
 node_modules
 dist
-bin/.*.ts
-tools/.*.ts
 docs/_build

.npmignore

@@ -1,25 +0,0 @@
-*~
-.npmignore
-
-config/config.json
-config/cwmp.key
-config/cwmp.crt
-config/cwmp.ca-bundle
-config/nbi.key
-config/nbi.crt
-config/nbi.ca-bundle
-config/fs.key
-config/fs.crt
-config/fs.ca-bundle
-config/ext
-
-debug/*
-!debug/.gitkeep
-
-dist
-.eslintrc.json
-.eslintignore
-ava.config.js
-tsconfig.json
-bin/.*
-tools/.*

.prettierignore

@@ -0,0 +1,2 @@
+dist
+npm-shrinkwrap.json

CHANGELOG.md

@@ -1,70 +1,143 @@
 # Change Log
 
+## 1.2.5 (2021-03-12)
+
+- Support specifying custom types when uploading files.
+- Fix JS compatibility issue with Safari browser.
+
+## 1.2.4 (2021-02-24)
+
+- The data model state of a CPE is no longer forgotten after unsuccessful
+  session termination (e.g. timeout). This addresses a number of undesired side
+  effects that arise when a CPE does not terminate the session properly.
+- Executing tasks that take a long time to complete (e.g. refreshing the entire
+  data model) no longer shows a timeout error while the task is still being
+  processed.
+- New function `ROUND()` available to expressions. It works similar to the
+  function by the same name in SQLite and PostgreSQL.
+- Log access events for `genieacs-nbi` service.
+- Pipe stdout/stderr from extension scripts to the `genieacs-cwmp` process log.
+- Parameter values of type `xsd:dateTime` are now displayed in the UI and CSV
+  downloads as a date string rather than a numeric value.
+- Add file download link in the files listing page.
+- Display spinner loading animation throughout the UI.
+- Display GenieACS version and build number underneath the logo.
+- New option to specify how many parameters are displayed at a time in the
+  all-parameters component. Simply set `limit` property in the component config.
+- Reduce overly strong Brotli compression level which was causing significant
+  page load slowdown when Brotli is used.
+- Retire dump-data-model tool. `genieacs-sim` can now use a CSV file as its data
+  model.
+- Reduce the number of concurrent database connections from each process.
+- Remove dependency on 'hostInfo' MongoDB command which is a privileged action.
+  It is now possible to use shared MongoDB instances with limited privileged.
+- Fix bug in NBI where querying files returns 404 error.
+- Fix ping not working for devices with an IPv6 address.
+- Fix an elusive memory leak in `genieacs-fs` that slowly eats up memory and can
+  go unnoticed for long periods of time.
+- Fix a rare edge case where a `declare()` call to set a parameter value may not
+  work as intended if the parameter was originally received as part of the
+  Inform message.
+- A number of other fixes and stability improvements.
+
+## 1.2.3 (2020-10-26)
+
+- New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
+  report writable parameters as non-writable. When set to true, the scripts will
+  no longer respect the 'writable' attribute of the CPE parameters and will send
+  a SetParamteerValues, AddObject, or DeleteObject request anyway.
+- Tags no longer restrict what characters are allowed. Any character other than
+  alphanumeric characters, hyphen, or underscore is now encoded in the data
+  model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
+- Ask for a confirmation before closing a pop-up dialog with unsaved changes.
+- Better XML validation to avoid crashes caused by invalid CPE requests.
+- Fix confusing 404 error message when the user attempts to modify a resource
+  when they don't have the necessary permissions.
+- Fix a rare issue where genieacs-cwmp stops accepting new connections after
+  running for a few weeks.
+- Fix exception when IS NULL operator is used in certain situations.
+
+## 1.2.2 (2020-10-03)
+
+- Added button to push files to selected devices from device listing page.
+- A few minor UI improvements.
+- Fix exception that can happen and persist after a Download request.
+- Fix validation bug preventing running refreshObject task on data model root.
+- Fix invalid arguments fault in refresh preset configuration when upgrading
+  from v1.1.
+
+## 1.2.1 (2020-09-08)
+
+- Fix bug causing faults to not be displayed in the UI.
+- Fix bug where deleting objects does not get reflected immediately in the UI.
+- Improve conversion between filters written in the expression format and
+  MongoDB queries. There should now be fewer edge cases where the two are not
+  equisatisfiable.
+
 ## 1.2.0 (2020-09-01)
 
 - Support GetParameterAttributes and SetParameterAttributes TR-069 methods.
 - Support CASE statement and COALESCE function in expressions.
 - Provision arguments can now be a list of expressions that are dynamically
-evaluated.
+  evaluated.
 - Support Forwarded HTTP header to display in the logs the correct IP of CPEs
-behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
+  behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
 - Config expressions can now access all available device parameters, not only
-serial number, product class, and OUI.
+  serial number, product class, and OUI.
-- Use relative URLs throughout the UI to allow serving from a subdirectory
+- Use relative URLs throughout the UI to allow serving from a subdirectory using
-using a reverse proxy.
+  a reverse proxy.
 - Make Date.parse() and Date.UTC() available to provision scripts.
 - libxmljs has been entirely removed in favor of our bespoke XML parser.
-- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now
+- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now used
-used to determine the TCP connection timeout.
+  to determine the TCP connection timeout.
-- The all-parameters component now limits the number of parameters displayed
+- The all-parameters component now limits the number of parameters displayed for
-for better performance.
+  better performance.
 - The process genieacs-cwmp is now much less likely to throw exceptions as a
-result of invalid requests from CPE.
+  result of invalid requests from CPE.
 - A large number of bug fixes and stability improvements.
 
 ## 1.2.0-beta.0 (2019-07-30)
 
 - A brand new UI superseding genieacs-gui.
 - New initialization wizard on first run.
-- New expression/query language used in search filters and preset
+- New expression/query language used in search filters and preset preconditions.
-preconditions.
 - CPE -> ACS authentication is now supported.
 - New config option (CWMP_KEEP_ALIVE_TIMEOUT) to specify how long to wait for a
-reply from the CPE before closing the TCP connection.
+  reply from the CPE before closing the TCP connection.
 - Debug logging has been reimplemented utilizing YAML format for logs.
 - Handle 9005 faults (Invalid Parameter Name) gracefully by attempting to
-rediscover the path of the missing parameter recursively.
+  rediscover the path of the missing parameter recursively.
 - declare() statements not followed by an explicit commit() are now deferred
-until all currently active scripts have been executed.
+  until all currently active scripts have been executed.
 - FS_HOSTNAME now defaults to the server's hostname or IP.
 - The API now validates the structure of task objects before saving.
 - New XML parser implementation for better performance. You can revert to the
-old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
+  old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
-v10.
+  v10.
 - Performance optimizations. While performance has improved for the majority of
-use cases, there may be situations where performance has degraded. It's
+  use cases, there may be situations where performance has degraded. It's
-recommended to revisit your hardware requirements.
+  recommended to revisit your hardware requirements.
 - Connection request authentication no longer uses 'auth.js' file. Instead, the
-connection request authentication behavior can now be customized using an
+  connection request authentication behavior can now be customized using an
-'expression'.
+  'expression'.
 - The config file (config.json) has been deprecated. System configuration (e.g.
-listen ports, worker count) are now recommended to be passed as environment.
+  listen ports, worker count) are now recommended to be passed as environment.
-variables. Other general configuration options are stored in the database so as
+  variables. Other general configuration options are stored in the database so
-to not require service restart for changes to take effect.
+  as to not require service restart for changes to take effect.
 - Optional redis dependency has been removed completely.
 - Tags now allow only alphanumeric characters and underscore.
-- Supported versions of NodeJs and MongoDB are 10.x and up and 2.6 and up
+- Supported versions of Node.js and MongoDB are 10.x and up and 2.6 and up
-respectively.
+  respectively.
 
 ## 1.1.3 (2018-10-23)
 
 - New config option (MAX_COMMIT_ITERATIONS) to avoid max commit iterations
-faults for more complex scripts.
+  faults for more complex scripts.
 - Support base64 and hexBinary parameter types.
 - Strict parsing of number values in queries (e.g. "123abc" no longer accepted
-as 123).
+  as 123).
 - Mixing $ne and $not operators is not allowed. Now it throws an error instead
-of returning incorrect results.
+  of returning incorrect results.
 - When a task expires, any associated fault is also deleted.
 - API now accepts 'timeout' argument when posting a task.
 - A number of stability fixes.
@@ -74,66 +147,68 @@ of returning incorrect results.
 - A large number of bug fixes as well as stability and performance improvements.
 - Three security vulnerabilities disclosed by Maximilian Hils have been patched.
 - New config option UDP_CONNECTION_REQUEST_PORT to specify binding port for UDP
-connection requests.
+  connection requests.
 - New config option DATETIME_MILLISECONDS to strip milliseconds from dateTime
-values.
+  values.
 - New config option BOOLEAN_LITERAL to use 1/0 or true/false for boolean values.
 - Parameter values that cannot be parsed according to the reported type now show
-a warning message.
+  a warning message.
 - Virtual parameter scripts now use the variable 'args' instead of the special
-'TIMESTAMPS' and 'VALUES' variables. The content of the args array is: {declare
+  'TIMESTAMPS' and 'VALUES' variables. The content of the args array is:
-timestamps}, {declare values}, {current timestamps}, {current values}.
+  {declare timestamps}, {declare values}, {current timestamps}, {current
+  values}.
 - Virtual parameter value types are now inferred from the JavaScript type if the
-returned value attribute is not a value-type pair.
+  returned value attribute is not a value-type pair.
 - Show a fault when a virtual parameter script doesn't return the required
-attributes.
+  attributes.
 - Redis is now optional (and disabled by default), reducing the complexity of
-scalable deployments.
+  scalable deployments.
 - Better detection of cyclical presets resulting in fewer faults for complex
-provisioning scripts.
+  provisioning scripts.
 - Math.random() is now deterministic on per-device basis. A function has been
-added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
+  added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
 - Overload spikes are now handled gracefully by refusing to accept new sessions
-temporarily when under abnormal load.
+  temporarily when under abnormal load.
 - Added log messages for session timeouts, connection drops, and XML parsing
-errors.
+  errors.
 - Date.now() now takes an optional argument to specify "time steps" (in
-milliseconds). This can be used to ensure a group of parameters are all
+  milliseconds). This can be used to ensure a group of parameters are all
-refreshed at the same time intervals.
+  refreshed at the same time intervals.
 - Only the non-default configuration options are now logged at process start.
 - Faults caused by errors from extensions now show a cleaner stack trace.
 - Exit main process if there are too many worker crashes (e.g. when DB is down).
 - Updated dependencies and included a lockfile to ensure installations get the
-exact dependencies it was tested against.
+  exact dependencies it was tested against.
 
 ## 1.1.1 (2017-03-23)
 
 - Avoid crashing when connection request credentials are missing.
 - Show a warning instead of crashing when failing to parse parameter values
-according to the expected value type.
+  according to the expected value type.
 - Add missing "Registered" event.
 - Fix bug where in certain cases many more instances than declared are created.
 - Fix parameter discovery bug when declared path timestamp is 1 or is not set.
 - Fix preset precondition failing when testing against datetime parameters and
-certain other parameters like \_deviceId.\_ProductClass.
+  certain other parameters like \_deviceId.\_ProductClass.
 
 ## 1.1.0 (2017-03-10)
 
 - Provisions enable implementing dynamic device configuration or complex device
-provisioning work flow using arbitrary scripts.
+  provisioning work flow using arbitrary scripts.
 - Virtual parameters are user-defined parameters whose values are evaluated from
-a custom script.
+  a custom script.
 - Extensions are sandboxed Node.js scripts that are accessible from provision
-and virtual parameter scripts to facilitate integration with external entities.
+  and virtual parameter scripts to facilitate integration with external
-- Support for UDP/STUN based connection requests for reaching devices behind
+  entities.
-NAT (TR-069 Annex G).
+- Support for UDP/STUN based connection requests for reaching devices behind NAT
+  (TR-069 Annex G).
 - Presets can now be scheduled using a cron-like expression.
 - Presets can now be tied to specific device events (e.g. boot).
-- Presets precondition queries no longer support "$or" or other MongoDB logical
+- Presets precondition queries no longer support "\$or" or other MongoDB logical
-operators.
+  operators.
 - Faults are no longer a part of tasks but are now first class objects.
 - Presets are now assigned to channels. A fault in one channel only blocks
-presets in that channel.
+  presets in that channel.
 - New API CRUD functions for provisions, virtual parameters, and faults.
 - New config options for XML output.
 - API responses now include "GenieACS-Version" header.
@@ -142,19 +217,19 @@ presets in that channel.
 - Supported Node.js versions are 6.x and 7.x.
 - Supported MongoDB versions are 2.6 through 3.4.
 - Expect performance differences due to major under the hood changes. Some
-operations are faster and some are slower. Overall performance is improved.
+  operations are faster and some are slower. Overall performance is improved.
 - GenieACS will no longer fetch the entire device data model upon first contact
-but will instead only fetch the parameters it needs to fulfill the presets.
+  but will instead only fetch the parameters it needs to fulfill the presets.
 - Logs have been overhauled and split into two streams: process log (stderr) and
-access log (stdout). Also added config options to dump logs to files rather than
+  access log (stdout). Also added config options to dump logs to files rather
-standard streams.
+  than standard streams.
 - Connection request authentication credentials are picked up from the device
-data model if available. config/auth.js is still supported as a fallback and now
+  data model if available. config/auth.js is still supported as a fallback and
-supports an optional callback argument.
+  now supports an optional callback argument.
 - Custom commands have been removed. Use virtual parameters and/or extensions.
 - Aliases and value normalizers (config/parameters.json) have been removed. Use
-virtual parameters.
+  virtual parameters.
 - The API /devices/<device_id>/preset has been removed.
 - Rarely used RequestDownload method no longer supported.
 - The TR-069 client simulator has moved to its own repo at
-https://github.com/zaidka/genieacs-sim
+  https://github.com/zaidka/genieacs-sim

CONTRIBUTING.md

@@ -5,8 +5,8 @@ tracker is strictly for bug reports.
 
 # Issues
 
-We use GitHub issues to track bugs. Please ensure your bug description is
+We use GitHub issues to track bugs. Please ensure your bug description is clear
-clear and has sufficient instructions to be able to reproduce the issue.
+and has sufficient instructions to be able to reproduce the issue.
 
 If you face interoperability issues with your CPE, more often than not it's a
 device related issue. Please consult the [forum] before submitting an Issue.
@@ -26,13 +26,14 @@ By submitting a pull request to this repository, you acknowledge that, while
 maintaining copyright, you grant Zaid Abdulla a perpetual, worldwide, non-
 exclusive, no-charge, royalty-free, irrevocable license to reproduce, prepare
 derivative works of, publicly display, publicly perform, sublicense, and
-distribute your contributions and such derivative works under the AGPLv3
+distribute your contributions and such derivative works under the AGPLv3 license
-license or any other license terms, including, but not limited to, proprietary
+or any other license terms, including, but not limited to, proprietary or
-or commercial license terms.
+commercial license terms.
 
-You confirm that you own or have rights to distribute and sublicense the
+You confirm that you own or have rights to distribute and sublicense the source
-source code contained therein, and that your content does not infringe upon
+code contained therein, and that your content does not infringe upon the
-the intellectual property rights of a third party.
+intellectual property rights of a third party.
 
-[commit-guidelines]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
+[commit-guidelines]:
+  http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
 [forum]: https://forum.genieacs.com

README.md

@@ -17,10 +17,10 @@ supported versions are:
 
 Install GenieACS from NPM:
 
-    sudo npm install -g --unsafe-perm genieacs
+    sudo npm install -g genieacs
 
 To build from source instead, clone this repo or download the source archive
-then *cd* into the source directory then run:
+then _cd_ into the source directory then run:
 
     npm install
     npm run build
@@ -30,8 +30,8 @@ source):
 
 ### genieacs-cwmp
 
-This is the service that the CPEs will communicate with. It listens on port
+This is the service that the CPEs will communicate with. It listens on port 7547
-7547 by default. Configure the ACS URL in your devices accordingly.
+by default. Configure the ACS URL in your devices accordingly.
 
 You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
 as a dummy TR-069 simulator if you don't have a CPE at hand.
@@ -50,7 +50,7 @@ such. It listens on port 7567 by default.
 ### genieacs-ui
 
 This serves the web based user interface. It listens on port 3000 by default.
-You must pass *--ui-jwt-secret* argument to supply the secret key used for
+You must pass _--ui-jwt-secret_ argument to supply the secret key used for
 signing browser cookies:
 
     genieacs-ui --ui-jwt-secret secret
@@ -64,14 +64,13 @@ a complete installation guide for production deployments.
 
 ## Support
 
-The [forum](https://forum.genieacs.com) is a good place to get guidance and
+The [forum](https://forum.genieacs.com) is a good place to get guidance and help
-help from the community. Head on over and join the conversation!
+from the community. Head on over and join the conversation!
 
 For commercial support options, please visit
 [genieacs.com](https://genieacs.com/support/).
 
 ## License
 
-Copyright 2013-2020 GenieACS Inc. GenieACS is released under the [AGPLv3
+Copyright 2013-2020 GenieACS Inc. GenieACS is released under the
-license
+[AGPLv3 license terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).
-terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

ava.config.js

@@ -1,5 +1,4 @@
 export default {
-  "extensions": ["ts"],
+  extensions: ["ts"],
-  "require": ["esm", "ts-node/register/transpile-only"]
+  require: ["esm", "ts-node/register/transpile-only"],
 };
-

bin/genieacs-cwmp.ts

@@ -24,7 +24,6 @@ import * as server from "../lib/server";
 import * as cwmp from "../lib/cwmp";
 import * as db from "../lib/db";
 import * as extensions from "../lib/extensions";
-import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("cwmp", VERSION);
@@ -36,14 +35,13 @@ function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
-    cache.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -77,9 +75,16 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
+  const key = config.get("CWMP_SSL_KEY") as string;
-    key: config.get("CWMP_SSL_KEY") as string,
+  const cert = config.get("CWMP_SSL_CERT") as string;
-    cert: config.get("CWMP_SSL_CERT") as string,
+
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    onConnection: cwmp.onConnection,
+    timeout: 30000,
+    keepAliveTimeout: 0,
   };
 
   process.on("uncaughtException", (err) => {
@@ -92,16 +97,10 @@ if (!cluster.worker) {
     server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
-  const initPromise = Promise.all([db.connect(), cache.connect()])
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(
+      server.start(options, cwmp.listener);
-        SERVICE_PORT,
-        SERVICE_ADDRESS,
-        ssl,
-        cwmp.listener,
-        cwmp.onConnection,
-        0
-      );
     })
     .catch((err) => {
       setTimeout(() => {

bin/genieacs-fs.ts

@@ -23,7 +23,6 @@ import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import { listener } from "../lib/fs";
 import * as db from "../lib/db";
-import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("fs", VERSION);
@@ -33,11 +32,9 @@ const SERVICE_PORT = config.get("FS_PORT") as number;
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
-  Promise.all([
+  Promise.all([db.disconnect(), cluster.worker.disconnect()]).catch(
-    db.disconnect(),
+    exitWorkerUngracefully
-    cache.disconnect(),
+  );
-    cluster.worker.disconnect(),
-  ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
@@ -73,9 +70,13 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
+  const key = config.get("FS_SSL_KEY") as string;
-    key: config.get("FS_SSL_KEY") as string,
+  const cert = config.get("FS_SSL_CERT") as string;
-    cert: config.get("FS_SSL_CERT") as string,
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
   let stopping = false;
@@ -93,12 +94,13 @@ if (!cluster.worker) {
 
   const _listener = (req, res): void => {
     if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
+    void listener(req, res);
   };
 
-  const initPromise = Promise.all([db.connect(), cache.connect()])
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+      server.start(options, _listener);
     })
     .catch((err) => {
       setTimeout(() => {

bin/genieacs-nbi.ts

@@ -24,7 +24,6 @@ import * as server from "../lib/server";
 import { listener } from "../lib/nbi";
 import * as db from "../lib/db";
 import * as extensions from "../lib/extensions";
-import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("nbi", VERSION);
@@ -36,14 +35,13 @@ function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
-    cache.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -77,9 +75,13 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
+  const key = config.get("NBI_SSL_KEY") as string;
-    key: config.get("NBI_SSL_KEY") as string,
+  const cert = config.get("NBI_SSL_CERT") as string;
-    cert: config.get("NBI_SSL_CERT") as string,
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
   let stopping = false;
@@ -100,9 +102,10 @@ if (!cluster.worker) {
     listener(req, res);
   };
 
-  const initPromise = Promise.all([db.connect(), cache.connect()])
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+      server.start(options, _listener);
     })
     .catch((err) => {
       setTimeout(() => {

bin/genieacs-ui.ts

@@ -23,9 +23,7 @@ import * as cluster from "../lib/cluster";
 import * as server from "../lib/server";
 import { listener } from "../lib/ui";
 import * as extensions from "../lib/extensions";
-import * as db from "../lib/ui/db";
+import * as db from "../lib/db";
-import * as db2 from "../lib/db";
-import * as cache from "../lib/cache";
 import { version as VERSION } from "../package.json";
 
 logger.init("ui", VERSION);
@@ -37,15 +35,13 @@ function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
   Promise.all([
     db.disconnect(),
-    db2.disconnect(),
-    cache.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
   ]).catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
+  extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -79,9 +75,13 @@ if (!cluster.worker) {
     cluster.stop();
   });
 } else {
-  const ssl = {
+  const key = config.get("UI_SSL_KEY") as string;
-    key: config.get("UI_SSL_KEY") as string,
+  const cert = config.get("UI_SSL_CERT") as string;
-    cert: config.get("UI_SSL_CERT") as string,
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
   };
 
   let stopping = false;
@@ -102,9 +102,10 @@ if (!cluster.worker) {
     listener(req, res);
   };
 
-  const initPromise = Promise.all([db2.connect(), cache.connect()])
+  const initPromise = db
+    .connect()
     .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
+      server.start(options, _listener);
     })
     .catch((err) => {
       setTimeout(() => {

build/build.ts

@@ -19,19 +19,17 @@
 
 import * as path from "path";
 import * as fs from "fs";
-import { promisify } from "util";
 import { rollup, WarningHandler } from "rollup";
 import rollupJson from "@rollup/plugin-json";
 import typescript from "@rollup/plugin-typescript";
 import { terser } from "rollup-plugin-terser";
 import nodeResolve from "@rollup/plugin-node-resolve";
 import commonjs from "@rollup/plugin-commonjs";
-import webpack from "webpack";
 import postcss from "postcss";
 import postcssImport from "postcss-import";
 import postcssPresetEnv from "postcss-preset-env";
 import cssnano from "cssnano";
-import SVGO from "svgo";
+import { optimize, extendDefaultPlugins } from "svgo";
 import * as xmlParser from "../lib/xml-parser";
 
 const MODE = process.env["NODE_ENV"] || "production";
@@ -44,7 +42,7 @@ const BUILD_METADATA = new Date()
 const INPUT_DIR = path.resolve(__dirname, "..");
 const OUTPUT_DIR = path.resolve(__dirname, "../dist");
 
-const externals = [
+const builtins = [
   "path",
   "fs",
   "cluster",
@@ -55,33 +53,13 @@ const externals = [
   "zlib",
   "crypto",
   "util",
-  "mongodb",
   "vm",
-  "later",
-  "parsimmon",
-  "seedrandom",
   "querystring",
   "child_process",
   "dgram",
   "url",
-  "iconv-lite",
+  "readline",
-  "koa",
-  "koa-router",
-  "koa-compress",
-  "koa-bodyparser",
-  "koa-jwt",
-  "koa-static",
-  "jsonwebtoken",
   "stream",
-  "mithril",
-  "parsimmon",
-  "yaml",
-  "codemirror",
-  "codemirror/mode/javascript/javascript",
-  "codemirror/mode/yaml/yaml",
-  "ipaddr.js",
-  "jsbi",
-  "yebool",
 ];
 
 function rmDirSync(dirPath): void {
@@ -131,7 +109,7 @@ function generateSymbol(id: string, svgStr: string): string {
   return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
 }
 
-async function init(): Promise<void> {
+async function init(): Promise<string[]> {
   // Delete any old output directory
   rmDirSync(OUTPUT_DIR);
 
@@ -139,13 +117,13 @@ async function init(): Promise<void> {
   fs.mkdirSync(OUTPUT_DIR);
   fs.mkdirSync(OUTPUT_DIR + "/bin");
   fs.mkdirSync(OUTPUT_DIR + "/public");
-  fs.mkdirSync(OUTPUT_DIR + "/tools");
 
   // Create package.json
   const packageJson = JSON.parse(
     fs.readFileSync(path.resolve(INPUT_DIR, "package.json")).toString()
   );
   delete packageJson["devDependencies"];
+  delete packageJson["private"];
   packageJson["scripts"] = {
     install: packageJson["scripts"].install,
     configure: packageJson["scripts"].configure,
@@ -166,6 +144,7 @@ async function init(): Promise<void> {
     path.resolve(OUTPUT_DIR, "npm-shrinkwrap.json"),
     JSON.stringify(npmShrinkwrapJson, null, 2)
   );
+  return Object.keys(packageJson["dependencies"]);
 }
 
 async function copyStatic(): Promise<void> {
@@ -203,41 +182,7 @@ async function generateCss(): Promise<void> {
   fs.writeFileSync(cssOutPath, cssOut.css);
 }
 
-async function generateToolsJs(): Promise<void> {
+async function generateBackendJs(externals: string[]): Promise<void> {
-  for (const bin of ["dump-data-model"]) {
-    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}.ts`);
-    const outputFile = path.resolve(OUTPUT_DIR, `tools/${bin}`);
-    const bundle = await rollup({
-      input: inputFile,
-      external: externals,
-      acorn: {
-        allowHashBang: true,
-      },
-      plugins: [
-        typescript({
-          tsconfig: "./tsconfig.json",
-          include: [`tools/${bin}.ts`, "lib/**/*.ts"],
-        }),
-        MODE === "production" ? terser() : null,
-      ],
-    });
-
-    await bundle.write({
-      format: "cjs",
-      preferConst: true,
-      sourcemap: "inline",
-      sourcemapExcludeSources: true,
-      banner: "#!/usr/bin/env node",
-      file: outputFile,
-    });
-
-    // Mark as executable
-    const mode = fs.statSync(outputFile).mode;
-    fs.chmodSync(outputFile, mode | 73);
-  }
-}
-
-async function generateBackendJs(): Promise<void> {
   for (const bin of [
     "genieacs-cwmp",
     "genieacs-ext",
@@ -249,7 +194,7 @@ async function generateBackendJs(): Promise<void> {
     const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
     const bundle = await rollup({
       input: inputFile,
-      external: externals,
+      external: [...builtins, ...externals],
       acorn: {
         allowHashBang: true,
       },
@@ -293,31 +238,36 @@ async function generateBackendJs(): Promise<void> {
   }
 }
 
-async function generateFrontendJs(): Promise<void> {
+async function generateFrontendJs(externals: string[]): Promise<void> {
   const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
-  const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
+  const outputDir = path.resolve(OUTPUT_DIR, "public");
 
-  const inlineDeps = ["mithril", "parsimmon", "jsbi", "yebool"];
+  const inlineDeps = ["parsimmon", "espresso-iisojs"];
   const bundle = await rollup({
     input: inputFile,
-    external: externals.filter((e) => !inlineDeps.includes(e)),
+    external: [
+      ...builtins,
+      ...externals.filter((e) => !inlineDeps.includes(e)),
+    ],
     plugins: [
       rollupJson({ preferConst: true }),
       {
         name: "",
         resolveId: function (importee, importer) {
-          if (importee.endsWith("/bigint"))
+          if (importee.endsWith("/package.json")) {
-            return this.resolve(importee + "-jsbi", importer);
+            const p = path.resolve(path.dirname(importer), importee);
-          else if (importee === "yebool")
+            if (p === path.resolve(INPUT_DIR, "package.json"))
-            return this.resolve("yebool/dist/yebool-jsbi.mjs", importer);
+              return path.resolve(OUTPUT_DIR, "package.json");
+          }
           return null;
         },
       },
       typescript({ tsconfig: "./tsconfig.json" }),
       nodeResolve(),
       commonjs(),
+      MODE === "production" ? terser() : null,
     ],
-    inlineDynamicImports: true,
+    preserveEntrySignatures: false,
     treeshake: {
       propertyReadSideEffects: false,
       moduleSideEffects: false,
@@ -329,47 +279,33 @@ async function generateFrontendJs(): Promise<void> {
   });
 
   await bundle.write({
+    manualChunks: (id) => {
+      if (id.includes("node_modules/codemirror")) return "codemirror";
+      else if (id.includes("node_modules/yaml")) return "yaml";
+      return "app";
+    },
     preferConst: true,
     format: "es",
-    sourcemap: "inline",
+    sourcemap: true,
     sourcemapExcludeSources: true,
-    file: outputFile,
+    dir: outputDir,
   });
-
-  const webpackConf = {
-    mode: MODE,
-    entry: outputFile,
-    resolve: {
-      aliasFields: ["module"],
-    },
-    devtool: "nosources-source-map",
-    module: {
-      rules: [
-        {
-          test: /\.js$/,
-          use: ["source-map-loader"],
-          enforce: "pre",
-        },
-      ],
-    },
-    output: {
-      path: path.resolve(OUTPUT_DIR, "public"),
-      filename: "app.js",
-    },
-  };
-
-  const stats = await promisify(webpack)(webpackConf);
-  process.stdout.write(stats.toString({ colors: true }) + "\n");
 }
 
 async function generateIconsSprite(): Promise<void> {
-  const svgo = new SVGO({ plugins: [{ removeViewBox: false }] });
   const symbols = [];
   const iconsDir = path.resolve(INPUT_DIR, "ui/icons");
   for (const file of fs.readdirSync(iconsDir)) {
     const id = path.parse(file).name;
     const filePath = path.join(iconsDir, file);
-    const { data } = await svgo.optimize(fs.readFileSync(filePath).toString());
+    const { data } = await optimize(fs.readFileSync(filePath).toString(), {
+      plugins: extendDefaultPlugins([
+        {
+          name: "removeViewBox",
+          active: false,
+        },
+      ]),
+    });
     symbols.push(generateSymbol(id, data));
   }
   fs.writeFileSync(
@@ -379,14 +315,13 @@ async function generateIconsSprite(): Promise<void> {
 }
 
 init()
-  .then(() => {
+  .then((externals) => {
     Promise.all([
       copyStatic(),
       generateCss(),
       generateIconsSprite(),
-      generateToolsJs(),
+      generateBackendJs(externals),
-      generateBackendJs(),
+      generateFrontendJs(externals),
-      generateFrontendJs(),
     ])
       .then(() => {
         // Ignore

build/spellcheck-dict.pws

@@ -0,0 +1,167 @@
+personal_ws-1.1 en 0
+genieacs
+GenieACS
+javascript
+sudo
+config
+cwmp
+CPE
+systemctl
+nbi
+CWMP
+NBI
+ACS
+SSL
+xsd
+args
+DeviceID
+GENIEACS
+env
+param
+arg
+AUTH
+igd
+TCP
+toctree
+auth
+EnvironmentFile
+JSON
+json
+parameterValues
+setParameterValues
+stderr
+stdout
+attr
+cfg
+CPEs
+ExecStart
+faq
+FileType
+GenieACS's
+JWT
+maxdepth
+npm
+objectName
+refreshObject
+SerialNumber
+systemd
+usr
+WantedBy
+yaml
+chown
+Config
+getParameterValues
+https
+InternetGatewayDevice
+mkdir
+OUI
+sql
+TODO
+UDP
+addObject
+AGPLv
+const
+cpe
+cron
+deleteObject
+FactoryReset
+factoryReset
+FileName
+fileType
+Github
+hostname
+HTTPS
+jwt
+latlong
+NPM
+oui
+parameterNames
+ProductClass
+productClass
+repo
+SSID
+sublicense
+TLS
+WANIPConnection
+WPA
+YourProvisionName
+abc
+Abdulla
+accessors
+AddObject
+AddressingType
+api
+APIs
+chmod
+cwmp's
+dateext
+DATETIME
+dateTime
+datetime
+declaratively
+delaycompress
+DeleteObject
+deviceId
+DHCP
+dir
+distro
+equisatisfiable
+ExternalIPAddress
+failover
+favor
+func
+GetParameterAttributes
+getPassword
+gte
+gui
+hexBinary
+Hils
+HOSTNAME
+init
+journalctl
+journald
+LastFileName
+LastFileType
+lastInform
+LIBXMLJS
+libxmljs
+literalinclude
+lockfile
+logrotate
+lte
+mipsbe
+normalizers
+pre
+quickstart
+README
+redis
+reimplemented
+RequestDownload
+RESTful
+rollout
+sandboxed
+scalable
+SetParameterAttributes
+SetParamteerValues
+SIGINT
+SIGTERM
+skipWritableCheck
+stringify
+subdirectory
+TargetFileName
+useradd
+VirtualParameters
+vparam
+xml
+YAML
+Zaid
+cacheExpire
+iss
+http
+statusCode
+rawData
+pos
+Brotli
+CSV
+hostInfo
+IPv

build/spellcheck.sh

@@ -0,0 +1,12 @@
+#! /bin/sh
+
+cd "$(dirname "$0")"
+
+FILES=`ls ../docs/*.rst ../docs/*.js ../*.md`
+
+for FILE in $FILES
+do
+    echo $FILE
+    cat $FILE | aspell list --lang=en --add-extra-dicts=./spellcheck-dict.pws --ignore 2
+    echo
+done

docs/api-reference.rst

@@ -1,7 +1,7 @@
 API Reference
 =============
 
-GenieACS expoeses a rich RESTful API through its NBI component. This document
+GenieACS exposes a rich RESTful API through its NBI component. This document
 serves as a reference for the available APIs.
 
 This API makes use of MongoDB's query language in some of its endpoints. Refer
@@ -16,7 +16,7 @@ to MongoDB's documentation for details.
 
 .. warning::
 
-  A common pitfll is not properly percent-encoding special characters in the
+  A common pitfall is not properly percent-encoding special characters in the
   device ID or query in the URL.
 
 Endpoints
@@ -41,11 +41,11 @@ Examples
 
 .. code:: javascript
 
-  query = {"_id": "202BC1-BM632w-0000000"}
+  query = {"_id": "202BC1-BM632w-000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D'
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D'
 
 - Find a device by its MAC address:
 
@@ -77,21 +77,21 @@ Examples
 
 .. code:: javascript
 
-  query = {"device": "202BC1-BM632w-0000000"}
+  query = {"device": "202BC1-BM632w-000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-0000000%22%7D'
+  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-000000%22%7D'
 
 - Return specific parameters for a given device:
 
 .. code:: javascript
 
-  query = {"_id": "202BC1-BM632w-0000000"}
+  query = {"_id": "202BC1-BM632w-000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
+  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
 
 The ``projection`` URL param is a comma-separated list of the parameters to receive.
 
@@ -106,7 +106,7 @@ if the tasks have been queued to be executed at the next inform.
 *device_id*: The ID of the device.
 
 *connection_request*: Indicates that a connection request will be triggered to
-execute the tasks immediatly. Otherwise, the tasks will be queued and be
+execute the tasks immediately. Otherwise, the tasks will be queued and be
 processed at the next inform.
 
 The response body is the task object as it is inserted in the database. The
@@ -120,7 +120,7 @@ Examples
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
   -X POST \
   --data '{"name": "refreshObject", "objectName": ""}'
 
@@ -138,7 +138,7 @@ Examples
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
   -X POST \
   --data '{"name":"setParameterValues", "parameterValues": [["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]]}'
 
@@ -183,7 +183,7 @@ Example
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/faults/202BC1-BM632w-0000000:default' -X DELETE
+  curl -i 'http://localhost:7557/faults/202BC1-BM632w-000000:default' -X DELETE
 
 DELETE /devices/\<device_id\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -218,7 +218,7 @@ Assign the tag "testing" to a device:
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X POST
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X POST
 
 DELETE /devices/\<device_id\>/tags/\<tag\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -236,7 +236,7 @@ Remove the tag "testing" from a device:
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X DELETE
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X DELETE
 
 PUT /presets/\<preset_name\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -299,7 +299,7 @@ The following file metadata may be sent as request headers:
 - ``fileType``: For firmware images it should be "1 Firmware Upgrade Image".
   Other common types are "2 Web Content" and "3 Vendor Configuration File".
 
-- ``oui``: The OUI of the device model that this file belogs to.
+- ``oui``: The OUI of the device model that this file belongs to.
 
 - ``productClass``: The product class of the device.
 
@@ -344,7 +344,7 @@ Find files using a query.
 Tasks
 -----
 
-Find the different availabe tasks and their object structure.
+Find the different available tasks and their object structure.
 
 ``getParameterValues``
 ~~~~~~~~~~~~~~~~~~~~~~

docs/cpe-authentication.rst

@@ -3,4 +3,70 @@
 CPE Authentication
 ==================
 
+CPE to ACS
+----------
+
+.. note::
+
+  By default GenieACS will accept any incoming connection via HTTP/HTTPS and
+  respond to it.
+
+The following parameters are used to set and get (password is redacted but
+can be set) the username/password used to authenticate against the ACS:
+
+Username: ``Device.ManagementServer.Username`` or ``InternetGatewayDevice.ManagementServer.Username``
+
+Password: ``Device.ManagementServer.Password`` or ``InternetGatewayDevice.ManagementServer.Password``
+
+Enable CPE to ACS Authentication
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+CPE to ACS authentication can be configured in the web interface by using the
+`Config` option in the `Admin` tab.
+
+Go to the `Admin` -> `Config` page and click on `New config` button at the
+bottom of the page. This will open pop-up which requires you to fill in a key
+and value. The key should be ``cwmp.auth``. The value accepts a boolean.
+Setting the value to ``true`` makes it so that GenieACS accepts any incoming
+connection, setting it to ``false`` makes GenieACS deny all incoming
+connections. This can be further configured using the ``AUTH()`` and ``EXT()``
+functions.
+
+The ``AUTH()`` function
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``AUTH()`` function accepts two parameters, username and password. It
+checks the given username and password with the incoming request to determine
+whether to return true or false.
+
+Basic usage of the ``AUTH()`` function could be as follows:
+
+.. code:: sql
+
+   AUTH("fixed-username", "fixed-password")
+
+This will only accept incoming request who authenticate with
+"fixed-username" and "fixed-password".
+
+The various device parameters can be referenced from within the ``cwmp.auth``
+expression. For example:
+
+.. code:: sql
+
+   AUTH(Device.ManagementServer.Username, Device.ManagementServer.Password)
+
+The ``EXT()`` function
+~~~~~~~~~~~~~~~~~~~~~~
+
+The ``EXT()`` function makes it possible to call an :ref:`extension
+<extensions>` script from the auth expression. This can be used to fetch
+the credentials from an external source:
+
+.. code:: sql
+
+   AUTH(DeviceID.SerialNumber, EXT("authenticate", "getPassword", DeviceID.SerialNumber))
+
+ACS to CPE
+----------
+
 TODO

docs/environment-variables.rst

@@ -86,7 +86,7 @@ CWMP_ACCESS_LOG_FILE
   File to log incoming requests for genieacs-cwmp. If omitted, logs will go to
   stdout.
 
-  Default: unest
+  Default: unset
 
 NBI_WORKER_PROCESSES
   The number of worker processes to spawn for genieacs-nbi. A value of 0 means
@@ -107,7 +107,7 @@ NBI_INTERFACE
 NBI_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
 
-  Default: unest
+  Default: unset
 
 NBI_SSL_KEY
   Path to certificate key file. If omitted, non-secure HTTP will be used.

docs/ext-sample.js

@@ -1,4 +1,4 @@
-// This is an exmaple GenieACS extension to get the current latitude/longitude
+// This is an example GenieACS extension to get the current latitude/longitude
 // of the International Space Station. Why, you ask? Because why not.
 // To install, copy this file to config/ext/iss.js.
 
@@ -10,25 +10,28 @@ let cache = null;
 let cacheExpire = 0;
 
 function latlong(args, callback) {
-  if (Date.now() < cacheExpire)
+  if (Date.now() < cacheExpire) return callback(null, cache);
-    return callback(null, cache);
 
-  http.get("http://api.open-notify.org/iss-now.json", (res) => {
+  http
-    if (res.statusCode !== 200)
+    .get("http://api.open-notify.org/iss-now.json", (res) => {
-      return callback(new Error(`Request failed (status code: ${res.statusCode})`));
+      if (res.statusCode !== 200)
+        return callback(
+          new Error(`Request failed (status code: ${res.statusCode})`)
+        );
 
-    let rawData = "";
+      let rawData = "";
-    res.on("data", (chunk) => rawData += chunk);
+      res.on("data", (chunk) => (rawData += chunk));
 
-    res.on("end", () => {
+      res.on("end", () => {
-      let pos = JSON.parse(rawData)["iss_position"];
+        let pos = JSON.parse(rawData)["iss_position"];
-      cache = [+pos["latitude"], +pos["longitude"]];
+        cache = [+pos["latitude"], +pos["longitude"]];
-      cacheExpire = Date.now() + 10000;
+        cacheExpire = Date.now() + 10000;
-      callback(null, cache);
+        callback(null, cache);
+      });
+    })
+    .on("error", (err) => {
+      callback(err);
     });
-  }).on("error", (err) => {
-    callback(err);
-  });
 }
 
 exports.latlong = latlong;

docs/installation-guide.rst

@@ -4,7 +4,7 @@ Installation Guide
 This guide is for installing GenieACS on a single server on any Linux distro
 that uses *systemd* as its init system.
 
-The various GenieACS services are independant of each other and may be
+The various GenieACS services are independent of each other and may be
 installed on different servers. You may also run multiple instances of each in
 a load-balancing/failover setup.
 
@@ -34,7 +34,7 @@ Install GenieACS
 
   .. parsed-literal::
 
-    sudo npm install -g --unsafe-perm genieacs@\ |release|
+    sudo npm install -g genieacs@\ |release|
 
 .. topic:: Installing from source
 
@@ -73,6 +73,7 @@ Configure systemd
     GENIEACS_FS_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-fs-access.log
     GENIEACS_UI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-ui-access.log
     GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
+    NODE_OPTIONS=--enable-source-maps
     GENIEACS_EXT_DIR=/opt/genieacs/ext
     GENIEACS_UI_JWT_SECRET=secret
   

lib/api-functions.ts

@@ -37,7 +37,7 @@ import { evaluate } from "./common/expression";
 export async function connectionRequest(
   deviceId: string,
   device?: Record<string, { value?: [boolean | number | string, string] }>
-): Promise<void> {
+): Promise<string> {
   if (!device) {
     const res = await db.devicesCollection.findOne({ _id: deviceId });
     if (!res) throw new Error("No such device");
@@ -156,43 +156,54 @@ export async function connectionRequest(
     );
   }
 
-  try {
+  const status = await httpConnectionRequest(
-    await httpConnectionRequest(
+    connectionRequestUrl,
-      connectionRequestUrl,
+    authExp,
-      authExp,
+    CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
-      CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
+    CONNECTION_REQUEST_TIMEOUT,
-      CONNECTION_REQUEST_TIMEOUT,
+    debug,
-      debug,
+    deviceId
-      deviceId
+  );
-    );
+
-  } catch (err) {
+  if (udpProm) {
-    if (!udpProm) throw err;
     await udpProm;
+    return "";
   }
+
+  return status;
 }
 
-export async function watchTask(
+export async function awaitSessionStart(
   deviceId: string,
-  taskId: string,
+  lastInform: number,
   timeout: number
-): Promise<string> {
+): Promise<boolean> {
-  await new Promise((resolve) => setTimeout(resolve, 500));
+  const now = Date.now();
-
+  const device = await db.devicesCollection.findOne(
-  const task = await db.tasksCollection.findOne(
+    { _id: deviceId },
-    { _id: taskId },
+    { projection: { _lastInform: 1 } }
-    { projection: { _id: 1 } }
   );
-  if (!task) return "completed";
+  const li = (device["_lastInform"] as Date).getTime();
+  if (li > lastInform) return true;
+  const token = await cache.get(`cwmp_session_${deviceId}`);
+  if (token) return true;
+  if (timeout < 500) return false;
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionStart(deviceId, lastInform, timeout);
+}
 
-  const q = { _id: `${deviceId}:task_${taskId}` };
+export async function awaitSessionEnd(
-  const fault = await db.faultsCollection.findOne(q, {
+  deviceId: string,
-    projection: { _id: 1 },
+  timeout: number
-  });
+): Promise<boolean> {
-  if (fault) return "fault";
+  const now = Date.now();
-
+  const token = await cache.get(`cwmp_session_${deviceId}`);
-  if ((timeout -= 500) <= 0) return "timeout";
+  if (!token) return true;
-
+  if (timeout < 500) return false;
-  return watchTask(deviceId, taskId, timeout);
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionEnd(deviceId, timeout);
 }
 
 function sanitizeTask(task): void {
@@ -236,6 +247,10 @@ function sanitizeTask(task): void {
       break;
 
     case "refreshObject":
+      if (typeof task.objectName !== "string")
+        throw new Error("Missing 'objectName' property");
+      break;
+
     case "deleteObject":
       if (typeof task.objectName !== "string" || !task.objectName.length)
         throw new Error("Missing 'objectName' property");
@@ -281,6 +296,15 @@ function sanitizeTask(task): void {
       )
         throw new Error("Invalid 'provisions' property");
       break;
+
+    case "reboot":
+      break;
+
+    case "factoryReset":
+      break;
+
+    default:
+      throw new Error("Invalid task name");
   }
 
   return task;

lib/cache.ts

@@ -17,126 +17,102 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, Collection } from "mongodb";
+import { Collection } from "mongodb";
+import { onConnect } from "./db";
 import * as config from "./config";
 
+const CLOCK_SKEW_TOLERANCE = 30000;
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let clientPromise: Promise<MongoClient>;
+let cacheCollection: Collection;
-let mongoCollection: Collection;
-let mongoTimeOffset = 0;
 
-export async function connect(): Promise<void> {
+onConnect(async (db) => {
-  const MONGODB_CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
+  cacheCollection = db.collection("cache");
-  clientPromise = MongoClient.connect(MONGODB_CONNECTION_URL, {
+  await cacheCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
-    useNewUrlParser: true,
+});
-    useUnifiedTopology: true,
+
-  });
+export async function get(key: string): Promise<string> {
-  const db = (await clientPromise).db();
+  const res = await cacheCollection.findOne({ _id: key });
-  mongoCollection = db.collection("cache");
+  if (res) return res["value"];
-  await mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+  return null;
-  const now = Date.now();
-  const res = await db.command({ hostInfo: 1 });
-  mongoTimeOffset = res.system.currentTime.getTime() - now;
 }
 
-export async function disconnect(): Promise<void> {
+export async function del(key: string): Promise<void> {
-  if (clientPromise) await (await clientPromise).close();
+  await cacheCollection.deleteOne({ _id: key });
-}
-
-export async function get(key: string | string[]): Promise<any> {
-  const expire = new Date(Date.now() - mongoTimeOffset);
-  if (Array.isArray(key)) {
-    const res = await mongoCollection.find({ _id: { $in: key } }).toArray();
-
-    const indices = {};
-    key.forEach((v, i) => {
-      indices[v] = i;
-    });
-
-    const values = [];
-    res.forEach((r) => {
-      if (r["expire"] > expire) values[indices[r["_id"]]] = r["value"];
-    });
-
-    return values;
-  } else {
-    const res = await mongoCollection.findOne({ _id: { $in: [key] } });
-    if (res && res["expire"] > expire) return res["value"];
-    return null;
-  }
-}
-
-export async function del(key: string | string[]): Promise<void> {
-  if (Array.isArray(key))
-    await mongoCollection.deleteMany({ _id: { $in: key } });
-  else await mongoCollection.deleteOne({ _id: key });
 }
 
 export async function set(
   key: string,
-  value: string | number,
+  value: string,
   ttl: number = MAX_CACHE_TTL
 ): Promise<void> {
-  const expire = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
+  const timestamp = new Date();
-  await mongoCollection.replaceOne(
+  const expire = new Date(
+    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000
+  );
+  await cacheCollection.replaceOne(
     { _id: key },
-    { _id: key, value: value, expire: expire },
+    { _id: key, value, expire, timestamp },
     { upsert: true }
   );
 }
 
 export async function pop(key: string): Promise<any> {
-  const res = await mongoCollection.findOneAndDelete({ _id: key });
+  const res = await cacheCollection.findOneAndDelete({ _id: key });
-
+  if (res && res["value"]) return res["value"]["value"];
-  if (
-    res &&
-    res["value"] &&
-    +res["value"]["expire"] - (Date.now() - mongoTimeOffset)
-  )
-    return res["value"]["value"];
-
   return null;
 }
 
-export async function lock(
+export async function acquireLock(
   lockName: string,
-  ttl: number
+  ttl: number,
-): Promise<(extendTtl?: number) => Promise<void>> {
+  timeout = 0,
-  const token = Math.random().toString(36).slice(2);
+  token = Math.random().toString(36).slice(2)
-
+): Promise<string> {
-  async function unlockOrExtend(extendTtl?: number): Promise<void> {
-    if (!extendTtl) {
-      const res = await mongoCollection.deleteOne({
-        _id: lockName,
-        value: token,
-      });
-      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
-    } else {
-      const expire = new Date(Date.now() - mongoTimeOffset + extendTtl * 1000);
-      const res = await mongoCollection.updateOne(
-        { _id: lockName, value: token },
-        { expire: expire }
-      );
-      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
-    }
-  }
-
-  const expireTest = new Date(Date.now() - mongoTimeOffset);
-  const expireSet = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
-
   try {
-    await mongoCollection.updateOne(
+    const now = Date.now();
-      { _id: lockName, expire: { $lte: expireTest } },
+    const r = await cacheCollection.findOneAndUpdate(
-      { $set: { value: token, expire: expireSet } },
+      { _id: lockName, token },
-      { upsert: true }
+      {
+        $set: {
+          value: token,
+          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
+        },
+        $currentDate: { timestamp: true },
+      },
+      { upsert: true, returnOriginal: false }
     );
+    const v = r.value;
+    if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
+      throw new Error("Database clock skew too great");
   } catch (err) {
-    if (err && err.code === 11000) {
+    if (err.code === 11000) {
-      await new Promise((resolve) => setTimeout(resolve, 200));
+      if (timeout > 0) {
-      return lock(lockName, ttl);
+        return new Promise((resolve, reject) => {
+          const w = 50 + Math.random() * 50;
+          setTimeout(() => {
+            acquireLock(lockName, ttl, timeout - w, token).then(
+              resolve,
+              reject
+            );
+          }, w);
+        });
+      }
+      throw new Error("Failed to acquire lock");
     }
+    throw err;
   }
 
-  return unlockOrExtend;
+  return token;
+}
+
+export async function releaseLock(
+  lockName: string,
+  token: string
+): Promise<void> {
+  const res = await cacheCollection.deleteOne({
+    _id: lockName,
+    value: token,
+  });
+  if (res["result"]["n"] !== 1) throw new Error("Lock expired");
 }

lib/common.ts

@@ -47,3 +47,17 @@ export function generateDeviceId(
 export function escapeRegExp(str: string): string {
   return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
 }
+
+export function encodeTag(tag: string): string {
+  return encodeURIComponent(tag)
+    .replace(
+      /[!~*'()]/g,
+      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
+    )
+    .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")
+    .replace(/%/g, "0x");
+}
+
+export function decodeTag(tag: string): string {
+  return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%"));
+}

lib/common/bigint-jsbi.ts

@@ -1,108 +0,0 @@
-/**
- * Copyright 2013-2020  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import JSBI from "jsbi";
-
-export function and(a: JSBI, b: JSBI): JSBI {
-  return JSBI.bitwiseAnd(a, b);
-}
-
-export function or(a: JSBI, b: JSBI): JSBI {
-  return JSBI.bitwiseOr(a, b);
-}
-
-export function xor(a: JSBI, b: JSBI): JSBI {
-  return JSBI.bitwiseXor(a, b);
-}
-
-export function not(a: JSBI): JSBI {
-  return JSBI.bitwiseNot(a);
-}
-
-export function lshift(a: JSBI, b: JSBI): JSBI {
-  return JSBI.leftShift(a, b);
-}
-
-export function rshift(a: JSBI, b: JSBI): JSBI {
-  return JSBI.signedRightShift(a, b);
-}
-
-export function add(a: JSBI, b: JSBI): JSBI {
-  return JSBI.add(a, b);
-}
-
-export function sub(a: JSBI, b: JSBI): JSBI {
-  return JSBI.subtract(a, b);
-}
-
-export function mul(a: JSBI, b: JSBI): JSBI {
-  return JSBI.multiply(a, b);
-}
-
-export function div(a: JSBI, b: JSBI): JSBI {
-  return JSBI.divide(a, b);
-}
-
-export function exp(a: JSBI, b: JSBI): JSBI {
-  return JSBI.exponentiate(a, b);
-}
-
-export function rem(a: JSBI, b: JSBI): JSBI {
-  return JSBI.remainder(a, b);
-}
-
-export function toNumber(a: JSBI): number {
-  return JSBI.toNumber(a);
-}
-
-export function eq(a: JSBI, b: JSBI): boolean {
-  return JSBI.equal(a, b);
-}
-
-export function ne(a: JSBI, b: JSBI): boolean {
-  return JSBI.notEqual(a, b);
-}
-
-export function lt(a: JSBI, b: JSBI): boolean {
-  return JSBI.lessThan(a, b);
-}
-
-export function lte(a: JSBI, b: JSBI): boolean {
-  return JSBI.lessThanOrEqual(a, b);
-}
-
-export function gt(a: JSBI, b: JSBI): boolean {
-  return JSBI.greaterThan(a, b);
-}
-
-export function gte(a: JSBI, b: JSBI): boolean {
-  return JSBI.greaterThanOrEqual(a, b);
-}
-
-export function asUintN(a: number, b: JSBI): JSBI {
-  return JSBI.asUintN(a, b);
-}
-
-export function asIntN(a: number, b: JSBI): JSBI {
-  return JSBI.asIntN(a, b);
-}
-
-const _BigInt = JSBI.BigInt;
-type _bigint = JSBI;
-export { _BigInt as BigInt, _bigint as bigint };

lib/common/boolean-expression.ts

@@ -18,7 +18,7 @@
  */
 
 import * as BI from "./bigint";
-import { espresso, complement, tautology } from "yebool";
+import { espresso, complement, tautology } from "espresso-iisojs";
 import { Expression } from "../types";
 import { map } from "./expression-parser";
 import { and, evaluateCallback } from "./expression";
@@ -362,6 +362,10 @@ class VarSynth extends BoolExprSynth {
         exp = exp.slice();
         exp[0] = ">";
         this.negate = true;
+      } else if (op === "NOT LIKE") {
+        exp = exp.slice();
+        exp[0] = "LIKE";
+        this.negate = true;
       }
     }
     this.expStr = JSON.stringify(exp);
@@ -809,6 +813,7 @@ function sopToExpression(
       if (Array.isArray(expr) && expr[0] === "NOT" && Array.isArray(expr[1])) {
         const e: Expression[] = expr[1];
         if (e[0] === "IS NULL") expr = ["IS NOT NULL", ...e.slice(1)];
+        else if (e[0] === "LIKE") expr = ["NOT LIKE", ...e.slice(1)];
         else if (e[0] === "=") expr = ["<>", ...e.slice(1)];
         else if (e[0] === "<>") expr = ["=", ...e.slice(1)];
         else if (e[0] === ">") expr = ["<=", ...e.slice(1)];
@@ -830,7 +835,6 @@ function sopToExpression(
 function generateDcSetAndIsNull(
   variables: Map<number, Expression>
 ): { dcSet: number[][]; isNull: Map<number, number> } {
-  // TODO support LIKE and NOT LIKE operators
   const relations: Map<
     string,
     { op: string; rhs: boolean | number | string; var: number }[]
@@ -843,6 +847,11 @@ function generateDcSetAndIsNull(
       let r = relations.get(lhs);
       if (!r) relations.set(lhs, (r = []));
       r.push({ op: op, rhs: e[2], var: n });
+    } else if (op === "LIKE" && !Array.isArray(e[2]) && !Array.isArray(e[3])) {
+      const lhs = JSON.stringify(e[1]);
+      let r = relations.get(lhs);
+      if (!r) relations.set(lhs, (r = []));
+      r.push({ op: "", rhs: null, var: n });
     } else {
       const lhs = JSON.stringify(e);
       let r = relations.get(lhs);
@@ -1019,8 +1028,7 @@ function boolExprSynthToExpression(boolExpr: BoolExprSynth): Expression {
   const { dcSet, isNull } = generateDcSetAndIsNull(variables);
   minterms = sanitizeMinterms(minterms, isNull);
 
-  const canRaise = (idx: number, set: Set<number>): boolean =>
+  const canRaise = getCanRaiseCallback(isNull);
-    !isNull.has(idx >> 2) || !(idx & 1) || !set.has(idx ^ 3);
   minterms = espresso(minterms, dcSet, { canRaise });
   return sopToExpression(minterms, variables);
 }

lib/common/expression-parser.ts

@@ -42,7 +42,7 @@ function interpretEscapes(str): string {
 
 export function map(
   exp: Expression,
-  callback: (exp: Expression) => Expression
+  callback: (e: Expression) => Expression
 ): Expression {
   if (!Array.isArray(exp)) return callback(exp);
 
@@ -60,7 +60,7 @@ export function map(
 
 export async function mapAsync(
   exp: Expression,
-  callback: (exp: Expression) => Promise<Expression>
+  callback: (e: Expression) => Promise<Expression>
 ): Promise<Expression> {
   if (!Array.isArray(exp)) return callback(exp);
 

lib/common/expression.ts

@@ -130,6 +130,15 @@ export function evaluateCallback(exp: Expression): Expression {
     } else if (exp[1] === "LOWER") {
       if (exp[2] == null) return null;
       if (!isArray(exp[2])) return toString(exp[2]).toLowerCase();
+    } else if (exp[1] === "ROUND") {
+      const n = exp[2];
+      const p = exp.length > 3 ? exp[3] : 0;
+      if (n == null || p == null) return null;
+      if (!isArray(n) && !isArray(p)) {
+        const d = 10 ** p;
+        const m = n * d * (1 + Number.EPSILON);
+        return Math.round(m) / d;
+      }
     }
   } else if (exp[0] === "PARAM") {
     if (exp[1] == null) return null;
@@ -264,21 +273,21 @@ export function evaluateCallback(exp: Expression): Expression {
 
 export function evaluate(
   exp: Expression,
-  obj: Record<string, unknown> | ((exp: string) => Expression),
+  obj: Record<string, unknown> | ((e: string) => Expression),
   now: number,
-  cb?: (exp: Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): string | number | boolean | null;
 export function evaluate(
   exp: Expression,
-  obj?: Record<string, unknown> | ((exp: string) => Expression),
+  obj?: Record<string, unknown> | ((e: string) => Expression),
   now?: number,
-  cb?: (exp: Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): Expression;
 export function evaluate(
   exp: Expression,
-  obj?: Record<string, unknown> | ((exp: string) => Expression),
+  obj?: Record<string, unknown> | ((e: string) => Expression),
   now?: number,
-  cb?: (exp: Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): Expression {
   return map(exp, (e) => {
     if (cb) e = cb(e);
@@ -306,19 +315,19 @@ export async function evaluateAsync(
   exp: Expression,
   obj: Record<string, unknown>,
   now: number,
-  cb?: (exp: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<string | number | boolean | null>;
 export async function evaluateAsync(
   exp: Expression,
   obj?: Record<string, unknown>,
   now?: number,
-  cb?: (exp: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<Expression>;
 export async function evaluateAsync(
   exp: Expression,
   obj?: Record<string, unknown>,
   now?: number,
-  cb?: (exp: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>
 ): Promise<Expression> {
   return mapAsync(exp, async (e) => {
     if (cb) e = await cb(e);

lib/common/memoize.ts

@@ -47,9 +47,18 @@ export default function memoize<T extends (...args: any[]) => any>(func: T): T {
     if (cache1.has(key)) return cache1.get(key);
 
     let r;
-    if (cache2.has(key)) r = cache2.get(key);
+    if (cache2.has(key)) {
-    else r = func(...args);
+      cache1.set(key, (r = cache2.get(key)));
-    cache1.set(key, r);
+    } else {
+      cache1.set(key, (r = func(...args)));
+      // Evict rejected promises
+      if (r instanceof Promise) {
+        r.catch(() => {
+          cache1.delete(key);
+          cache2.delete(key);
+        });
+      }
+    }
     return r;
   }) as any;
 }

lib/common/yaml.ts

@@ -0,0 +1,218 @@
+const LINE_WIDTH = 80;
+const INDENTATION = "  ";
+
+const STRING_RESERVED = new Set([
+  "true",
+  "True",
+  "TRUE",
+  "false",
+  "False",
+  "FALSE",
+  "null",
+  "Null",
+  "NULL",
+]);
+
+function isPrintable(str: string): boolean {
+  return !/[^\t\n\x20-\x7e\x85\u{a0}-\u{d7ff}\u{e000}-\u{fffd}\u{10000}-\u{10ffff}]/u.test(
+    str
+  );
+}
+
+function stringifyKey(str: string): string {
+  if (!str || !isPrintable(str)) return JSON.stringify(str);
+  if (/^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|[\n,[\]{}]|\s$/.test(str))
+    return JSON.stringify(str);
+  return str;
+}
+
+function foldString(str: string): string[] {
+  if (str.length <= LINE_WIDTH) return [str];
+  if (str.startsWith(" ")) return [str];
+  const lines: string[] = [];
+
+  let idx = 0;
+  let cand = 0;
+  for (let i = 1; i < str.length - 1; ++i) {
+    if (str[i] !== " ") continue;
+
+    if (str[i + 1] === " ") {
+      i += 2;
+      while (str[i] === " ") ++i;
+      continue;
+    }
+
+    if (i <= idx + LINE_WIDTH) {
+      cand = i;
+      continue;
+    }
+
+    const c = cand > idx ? cand : i;
+    lines.push(str.slice(idx, c));
+    idx = c + 1;
+    cand = i;
+  }
+
+  if (cand > idx && str.length > idx + LINE_WIDTH) {
+    lines.push(str.slice(idx, cand));
+    idx = cand + 1;
+  }
+
+  lines.push(str.slice(idx));
+
+  return lines;
+}
+
+function stringifyString(str: string, res: string[], prefix1, prefix2): void {
+  if (/^\s*$/.test(str) || STRING_RESERVED.has(str) || !isPrintable(str)) {
+    res.push(prefix1 + JSON.stringify(str));
+    return;
+  }
+
+  if (!prefix2) prefix2 = INDENTATION;
+
+  const lines = str.split("\n");
+  if (lines.length > 1) {
+    let idt = "";
+    let chmp = "-";
+    if ((lines.find((l) => l) || "").startsWith(" "))
+      idt = `${INDENTATION.length}`;
+
+    if (!lines[lines.length - 1]) {
+      lines.pop();
+      if (lines[lines.length - 1]) chmp = "";
+      else chmp = "+";
+    }
+
+    if (/^\s+$/.test(lines[lines.length - 1])) {
+      res.push(prefix1 + JSON.stringify(str));
+      return;
+    }
+
+    let isFolded = false;
+    const folded = lines.map((l) => {
+      const ls = foldString(l);
+      if (ls.length > 1) isFolded = true;
+      return ls;
+    });
+
+    if (!isFolded) {
+      res.push(
+        `${prefix1}|${idt}${chmp}`,
+        ...lines.map((l) => (l ? prefix2 + l : l))
+      );
+      return;
+    }
+
+    res.push(`${prefix1}>${idt}${chmp}`);
+    res.push(...folded[0].map((f) => prefix2 + f));
+    for (let i = 1; i < folded.length; ++i) {
+      const prevLine = folded[i - 1][0];
+      if (prevLine && !folded[i - 1][0].startsWith(" ")) res.push("");
+      res.push(...folded[i].map((f) => prefix2 + f));
+    }
+    return;
+  }
+
+  if (
+    /^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|\s$/.test(str) ||
+    parseFloat(str) === +str
+  ) {
+    res.push(prefix1 + JSON.stringify(str));
+    return;
+  }
+
+  res.push(prefix1 + str);
+}
+
+function stringifyAny(
+  obj: unknown,
+  res: string[],
+  prefix1 = "",
+  prefix2 = ""
+): void {
+  if (obj == null) {
+    res.push(`${prefix1}null`);
+    return;
+  }
+  if (typeof obj === "number" || typeof obj === "boolean") {
+    res.push(`${prefix1}${JSON.stringify(obj)}`);
+    return;
+  }
+  if (obj instanceof Date) {
+    res.push(`${prefix1}${obj.toJSON()}`);
+    return;
+  }
+
+  if (typeof obj === "string") {
+    stringifyString(obj, res, prefix1, prefix2);
+    return;
+  }
+
+  if (Array.isArray(obj)) {
+    if (!obj.length) {
+      res.push(prefix1 + "[]");
+      return;
+    }
+
+    if (!prefix1 || prefix1.endsWith("- ")) {
+      stringifyAny(obj[0], res, prefix1 + "- ", prefix2 + INDENTATION);
+      prefix1 = prefix2 + "- ";
+      prefix2 = prefix2 + INDENTATION;
+      for (let i = 1; i < obj.length; ++i)
+        stringifyAny(obj[i], res, prefix1, prefix2);
+    } else {
+      res.push(prefix1);
+      prefix1 = prefix2 + "- ";
+      prefix2 = prefix2 + INDENTATION;
+      for (let i = 0; i < obj.length; ++i)
+        stringifyAny(obj[i], res, prefix1, prefix2);
+    }
+    return;
+  }
+
+  const entries = Object.entries(obj).filter((e) => e[1] !== undefined);
+
+  if (!entries.length) {
+    res.push(prefix1 + "{}");
+    return;
+  }
+
+  if (!prefix1 || prefix1.endsWith("- ")) {
+    stringifyAny(
+      entries[0][1],
+      res,
+      prefix1 + `${stringifyKey(entries[0][0])}: `,
+      prefix2 + INDENTATION
+    );
+    prefix1 = prefix2;
+    prefix2 = prefix2 + INDENTATION;
+    for (let i = 1; i < entries.length; ++i) {
+      stringifyAny(
+        entries[i][1],
+        res,
+        prefix1 + `${stringifyKey(entries[i][0])}: `,
+        prefix2
+      );
+    }
+  } else {
+    res.push(prefix1);
+    prefix1 = prefix2;
+    prefix2 = prefix2 + INDENTATION;
+    for (let i = 0; i < entries.length; ++i) {
+      stringifyAny(
+        entries[i][1],
+        res,
+        prefix1 + `${stringifyKey(entries[i][0])}: `,
+        prefix2
+      );
+    }
+  }
+}
+
+export function stringify(obj: unknown): string {
+  if (obj === undefined) return undefined;
+  const lines: string[] = [];
+  stringifyAny(obj, lines);
+  return lines.join("\n") + "\n";
+}

lib/connection-request.ts

@@ -62,7 +62,6 @@ async function extractAuth(
 
 function httpGet(
   options: http.RequestOptions,
-  timeout,
   _debug: boolean,
   deviceId: string
 ): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
@@ -77,15 +76,12 @@ function httpGet(
         }
       })
       .on("error", (err) => {
-        req.abort();
+        req.destroy();
-        reject(new Error("Device is offline"));
+        reject(err);
         if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
       })
-      .on("socket", (socket) => {
+      .on("timeout", () => {
-        socket.setTimeout(timeout);
+        req.destroy();
-        socket.on("timeout", () => {
-          req.abort();
-        });
       });
   });
 }
@@ -97,14 +93,15 @@ export async function httpConnectionRequest(
   timeout: number,
   _debug: boolean,
   deviceId: string
-): Promise<void> {
+): Promise<string> {
   const options: http.RequestOptions = parse(address);
   if (options.protocol !== "http:")
-    throw new Error("Invalid connection request URL or protocol");
+    return "Invalid connection request URL or protocol";
 
   options.agent = new http.Agent({
     maxSockets: 1,
     keepAlive: true,
+    timeout: timeout,
   });
 
   let authHeader: Record<string, string>;
@@ -115,8 +112,7 @@ export async function httpConnectionRequest(
     let opts = options;
     if (authHeader) {
       if (authHeader["method"] === "Basic") {
-        if (!allowBasicAuth)
+        if (!allowBasicAuth) return "Basic HTTP authentication not allowed";
-          throw new Error("Basic HTTP authentication not allowed");
 
         opts = Object.assign(
           {
@@ -143,17 +139,34 @@ export async function httpConnectionRequest(
           options
         );
       } else {
-        throw new Error("Unrecognized auth method");
+        return "Unrecognized auth method";
       }
     }
 
-    let res = await httpGet(opts, timeout, _debug, deviceId);
+    let res: { statusCode: number; headers: http.IncomingHttpHeaders };
+    try {
+      res = await httpGet(opts, _debug, deviceId);
+    } catch (err) {
+      // Workaround for some devices unexpectedly closing the connection
+      if (authHeader) {
+        try {
+          res = await httpGet(opts, _debug, deviceId);
+        } catch (err) {
+          return `Connection request error: ${err.message}`;
+        }
+      }
 
-    // Workaround for some devices unexpectedly closing the connection
+      if (err["code"] === "ECONNRESET" || err["code"] === "ECONNREFUSED")
-    if (res.statusCode === 0 && authHeader)
+        return "Device is offline";
-      res = await httpGet(opts, timeout, _debug, deviceId);
+
-    if (res.statusCode === 0) throw new Error("Device is offline");
+      return `Connection request error: ${err.message}`;
-    if (res.statusCode === 200 || res.statusCode === 204) return;
+    }
+
+    if (res.statusCode === 200 || res.statusCode === 204) return "";
+
+    // When a Connection Request is received for the Virtual CWMP Device and the Proxied
+    // Device is offline the CPE Proxier MUST respond with an HTTP 503 failure
+    if (res.statusCode === 503) return "Device is offline";
 
     if (res.statusCode === 401 && res.headers["www-authenticate"]) {
       authHeader = auth.parseWwwAuthenticateHeader(
@@ -161,12 +174,11 @@ export async function httpConnectionRequest(
       );
       [username, password, authExp] = await extractAuth(authExp, false);
     } else {
-      throw new Error(
+      return `Connection request error: Unexpected status code ${res.statusCode}`;
-        `Unexpected response code from device: ${res.statusCode}`
-      );
     }
   }
-  throw new Error("Incorrect connection request credentials");
+
+  return "Connection request error: Incorrect connection request credentials";
 }
 
 export async function udpConnectionRequest(
@@ -209,7 +221,7 @@ export async function udpConnectionRequest(
     const message = Buffer.from(msg);
 
     for (let i = 0; i < 3; ++i) {
-      await new Promise((resolve, reject) => {
+      await new Promise<void>((resolve, reject) => {
         client.send(message, 0, message.length, port, host, (err: Error) => {
           if (err) reject(err);
           else resolve();

lib/cwmp.ts

@@ -79,7 +79,7 @@ async function authenticate(
     sessionContext.cacheSnapshot,
     "cwmp.auth"
   );
-  if (!authExpression) return true;
+  if (authExpression == null) return true;
 
   let authentication;
 
@@ -90,9 +90,7 @@ async function authenticate(
   }
 
   if (authentication && authentication.method === "Digest") {
-    const sessionNonce = sessionsNonces.get(
+    const sessionNonce = sessionsNonces.get(sessionContext.httpRequest.socket);
-      sessionContext.httpRequest.connection
-    );
 
     if (
       !sessionNonce ||
@@ -185,7 +183,7 @@ async function writeResponse(
   }
 
   const httpResponse = sessionContext.httpResponse;
-  const connection = httpResponse.connection;
+  const connection = httpResponse.socket;
 
   httpResponse.setHeader("Content-Length", Buffer.byteLength(data));
   httpResponse.writeHead(res.code, res.headers);
@@ -198,17 +196,23 @@ async function writeResponse(
       sessionContext: sessionContext,
       message: "Connection dropped",
     });
+    await endSession(sessionContext);
   } else if (close) {
-    const isNew = await endSession(sessionContext);
+    session.clearProvisions(sessionContext);
-    if (isNew) {
+    await endSession(sessionContext);
-      logger.accessInfo({
-        sessionContext: sessionContext,
-        message: "New device registered",
-      });
-    }
   } else {
-    sessionContext.lastActivity = Date.now();
+    const now = Date.now();
+    sessionContext.lastActivity = now;
     currentSessions.set(connection, sessionContext);
+    if (now >= sessionContext.extendLock) {
+      sessionContext.extendLock = now + 10000;
+      await cache.acquireLock(
+        `cwmp_session_${sessionContext.deviceId}`,
+        sessionContext.timeout + 15000,
+        0,
+        sessionContext.sessionId
+      );
+    }
   }
 }
 
@@ -722,15 +726,34 @@ async function nextRpc(sessionContext: SessionContext): Promise<void> {
       );
       break;
     default:
-      throw new Error("Task name not recognized");
+      if (!sessionContext.doneTasks) sessionContext.doneTasks = [];
+      sessionContext.doneTasks.push(task._id);
+      sessionContext.tasks = sessionContext.tasks.filter((t) => t !== task);
+
+      logger.accessWarn({
+        sessionContext: sessionContext,
+        message: "Invalid task",
+        taskId: task._id,
+      });
   }
 
   return nextRpc(sessionContext);
 }
 
-async function endSession(sessionContext: SessionContext): Promise<boolean> {
+async function endSession(sessionContext: SessionContext): Promise<void> {
   let saveCache = sessionContext.cacheUntil != null;
 
+  if (sessionContext.provisions.length) {
+    const fault = {
+      code: "session_terminated",
+      message: "The TR-069 session was unsuccessfully terminated",
+      timestamp: sessionContext.timestamp,
+    };
+    recordFault(sessionContext, fault);
+    // No need to save retryNow
+    for (const f of Object.values(sessionContext.faults)) delete f.retryNow;
+  }
+
   const promises = [];
 
   promises.push(
@@ -793,7 +816,16 @@ async function endSession(sessionContext: SessionContext): Promise<boolean> {
   }
 
   await Promise.all(promises);
-  return sessionContext.new;
+  await cache.releaseLock(
+    `cwmp_session_${sessionContext.deviceId}`,
+    sessionContext.sessionId
+  );
+  if (sessionContext.new) {
+    logger.accessInfo({
+      sessionContext: sessionContext,
+      message: "New device registered",
+    });
+  }
 }
 
 async function sendAcsRequest(
@@ -842,8 +874,11 @@ async function sendAcsRequest(
   return writeResponse(sessionContext, res);
 }
 
-async function getSession(connection, sessionId): Promise<SessionContext> {
+async function getSession(
-  const sessionContext = currentSessions.get(connection);
+  connection: Socket,
+  sessionId: string
+): Promise<SessionContext> {
+  let sessionContext = currentSessions.get(connection);
   if (sessionContext) {
     currentSessions.delete(connection);
     return sessionContext;
@@ -855,7 +890,9 @@ async function getSession(connection, sessionId): Promise<SessionContext> {
 
   const sessionContextString = await cache.pop(`session_${sessionId}`);
   if (!sessionContextString) return null;
-  return session.deserialize(sessionContextString);
+  sessionContext = await session.deserialize(sessionContextString);
+  connection.setTimeout(sessionContext.timeout);
+  return sessionContext;
 }
 
 // Only needed to prevent tree shaking from removing the remoteAddress
@@ -872,18 +909,32 @@ export function onConnection(socket: Socket): void {
     const sessionContext = currentSessions.get(socket);
     if (!sessionContext) return;
     currentSessions.delete(socket);
+    if (sessionContext.authState !== 2) {
+      logger.accessError({
+        message: "Authentication failure",
+        sessionContext: sessionContext,
+      });
+      return;
+    }
+
     const now = Date.now();
 
     const lastActivity = sessionContext.lastActivity;
-    const timeoutMsg = logger.flatten({
+    const timeoutMsg = {
       sessionContext: sessionContext,
       message: "Session timeout",
       sessionTimestamp: sessionContext.timestamp,
-    });
+    };
 
     const timeout =
       sessionContext.lastActivity + sessionContext.timeout * 1000 - now;
-    if (timeout <= 0) return void logger.accessError(timeoutMsg);
+
+    if (timeout <= 0) {
+      logger.accessError(timeoutMsg);
+      // TODO it's possible that lock would have already been expired
+      await endSession(sessionContext);
+      return;
+    }
 
     setTimeout(async () => {
       const sessionContextString = await cache.get(
@@ -891,8 +942,10 @@ export function onConnection(socket: Socket): void {
       );
       if (!sessionContextString) return;
       const _sessionContext = await session.deserialize(sessionContextString);
-      if (_sessionContext.lastActivity === lastActivity)
+      if (_sessionContext.lastActivity === lastActivity) {
         logger.accessError(timeoutMsg);
+        await endSession(sessionContext);
+      }
     }, timeout + 1000).unref();
 
     if (sessionContext.state === 0) return;
@@ -988,7 +1041,7 @@ async function reportBadState(sessionContext: SessionContext): Promise<void> {
     sessionContext: sessionContext,
   });
   const httpResponse = sessionContext.httpResponse;
-  currentSessions.delete(httpResponse.connection);
+  currentSessions.delete(httpResponse.socket);
   const body = "Bad session state";
   httpResponse.setHeader("Content-Length", Buffer.byteLength(body));
   httpResponse.writeHead(400, { Connection: "close" });
@@ -1015,14 +1068,14 @@ async function responseUnauthorized(
       resHeaders["WWW-Authenticate"] = `Basic realm="${REALM}"`;
     } else {
       const nonce = crypto.randomBytes(16).toString("hex");
-      sessionsNonces.set(sessionContext.httpRequest.connection, nonce);
+      sessionsNonces.set(sessionContext.httpRequest.socket, nonce);
       let d = `Digest realm="${REALM}"`;
       d += ',qop="auth,auth-int"';
       d += `,nonce="${nonce}"`;
 
       resHeaders["WWW-Authenticate"] = d;
     }
-    currentSessions.set(sessionContext.httpRequest.connection, sessionContext);
+    currentSessions.set(sessionContext.httpRequest.socket, sessionContext);
   }
 
   const httpResponse = sessionContext.httpResponse;
@@ -1042,7 +1095,6 @@ async function processRequest(
 ): Promise<void> {
   for (const w of parseWarnings) {
     w.sessionContext = sessionContext;
-    w.rpc = rpc;
     logger.accessWarn(w);
   }
 
@@ -1070,6 +1122,8 @@ async function processRequest(
       );
     }
 
+    sessionContext.httpRequest.socket.setTimeout(sessionContext.timeout * 1000);
+
     if (sessionContext.debug) {
       debug.incomingHttpRequest(
         sessionContext.httpRequest,
@@ -1088,6 +1142,37 @@ async function processRequest(
       }
     }
 
+    try {
+      sessionContext.extendLock = sessionContext.timestamp + 10000;
+      await cache.acquireLock(
+        `cwmp_session_${sessionContext.deviceId}`,
+        sessionContext.timeout + 15000,
+        0,
+        sessionContext.sessionId
+      );
+    } catch (err) {
+      logger.accessError({
+        message: "CPE already in session",
+        sessionContext: sessionContext,
+      });
+
+      const _body = "CPE already in session";
+      sessionContext.httpResponse.setHeader(
+        "Content-Length",
+        Buffer.byteLength(_body)
+      );
+      sessionContext.httpResponse.writeHead(400, { Connection: "close" });
+      if (sessionContext.debug) {
+        debug.outgoingHttpResponse(
+          sessionContext.httpResponse,
+          sessionContext.deviceId,
+          _body
+        );
+      }
+      sessionContext.httpResponse.end(_body);
+      return;
+    }
+
     sessionContext.state = 1;
     sessionContext.authState = 2;
 
@@ -1116,6 +1201,7 @@ async function processRequest(
         sessionContext.authState = 1;
         return responseUnauthorized(sessionContext, false);
       } else {
+        await endSession(sessionContext);
         return responseUnauthorized(sessionContext, true);
       }
     }
@@ -1158,7 +1244,15 @@ async function processRequest(
   } else if (rpc.cpeResponse) {
     if (sessionContext.state !== 2) return reportBadState(sessionContext);
 
-    await session.rpcResponse(sessionContext, rpc.id, rpc.cpeResponse);
+    const fault = await session.rpcResponse(
+      sessionContext,
+      rpc.id,
+      rpc.cpeResponse
+    );
+    if (fault) {
+      recordFault(sessionContext, fault);
+      session.clearProvisions(sessionContext);
+    }
     return nextRpc(sessionContext);
   } else if (rpc.cpeFault) {
     if (sessionContext.state !== 2) return reportBadState(sessionContext);
@@ -1210,19 +1304,24 @@ export function listener(
       stats.concurrentRequests -= 1;
     })
     .catch((err) => {
-      currentSessions.delete(httpResponse.connection);
+      currentSessions.delete(httpResponse.socket);
-      httpResponse.writeHead(500, { Connection: "close" });
-      httpResponse.end(`${err.name}: ${err.message}`);
       stats.concurrentRequests -= 1;
       setTimeout(() => {
         throw err;
       });
+      try {
+        httpResponse.socket.unref();
+        httpResponse.writeHead(500, { Connection: "close" });
+        httpResponse.end(`${err.name}: ${err.message}`);
+      } catch (err) {
+        // Ignore
+      }
     });
 }
 
 function decodeString(buffer: Buffer, charset: string): string {
   try {
-    return buffer.toString(charset);
+    return buffer.toString(charset as BufferEncoding);
   } catch (err) {
     if (encodingExists(charset)) return decode(buffer, charset);
   }
@@ -1307,9 +1406,9 @@ async function listenerAsync(
   // Request aborted
   if (!body) return;
 
-  const newConnection = !currentSessions.has(httpRequest.connection);
+  const newConnection = !currentSessions.has(httpRequest.socket);
 
-  const sessionContext = await getSession(httpRequest.connection, sessionId);
+  const sessionContext = await getSession(httpRequest.socket, sessionId);
 
   if (sessionContext) {
     sessionContext.httpRequest = httpRequest;
@@ -1334,6 +1433,7 @@ async function listenerAsync(
         );
       }
       httpResponse.end(_body);
+      await endSession(sessionContext);
       return;
     }
     if (newConnection && sessionContext.authState !== 1)
@@ -1347,7 +1447,7 @@ async function listenerAsync(
     return;
   }
 
-  let charset;
+  let charset: string;
   if (httpRequest.headers["content-type"]) {
     const m = httpRequest.headers["content-type"].match(
       /charset=['"]?([^'"\s]+)/i
@@ -1384,6 +1484,7 @@ async function listenerAsync(
         );
         debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, msg);
       }
+      await endSession(sessionContext);
     } else {
       const cacheSnapshot = await localCache.getCurrentSnapshot();
       const d = !!localCache.getConfig(
@@ -1409,7 +1510,7 @@ async function listenerAsync(
   }
 
   const parseWarnings = [];
-  let rpc;
+  let rpc: SoapMessage;
   try {
     rpc = soap.request(
       bodyStr,
@@ -1440,6 +1541,7 @@ async function listenerAsync(
           error.message
         );
       }
+      await endSession(sessionContext);
     } else {
       const cacheSnapshot = await localCache.getCurrentSnapshot();
       const d = !!localCache.getConfig(
@@ -1492,8 +1594,10 @@ async function listenerAsync(
         return e;
       }
     );
-    if (d)
+    if (d) {
-      debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, _body);
+      debug.incomingHttpRequest(httpRequest, null, bodyStr);
+      debug.outgoingHttpResponse(httpResponse, null, _body);
+    }
     httpResponse.end(_body);
     return;
   }
@@ -1514,7 +1618,6 @@ async function listenerAsync(
   _sessionContext.httpRequest = httpRequest;
   _sessionContext.httpResponse = httpResponse;
   _sessionContext.sessionId = crypto.randomBytes(8).toString("hex");
-  httpRequest.connection.setTimeout(_sessionContext.timeout * 1000);
 
   const {
     tasks: dueTasks,

lib/db.ts

@@ -17,9 +17,9 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, ObjectID, Collection } from "mongodb";
+import { MongoClient, ObjectID, Collection, Db } from "mongodb";
 import { get } from "./config";
-import { escapeRegExp } from "./common";
+import { encodeTag, escapeRegExp } from "./common";
 import { parse } from "./common/expression-parser";
 import {
   DeviceData,
@@ -45,7 +45,6 @@ export let tasksCollection: Collection,
   configCollection: Collection;
 
 let clientPromise: Promise<MongoClient>;
-export let client: MongoClient;
 
 function compareAccessLists(list1: string[], list2: string[]): boolean {
   if (list1.length !== list2.length) return false;
@@ -53,15 +52,24 @@ function compareAccessLists(list1: string[], list2: string[]): boolean {
   return true;
 }
 
+const onConnectCallbacks: ((db: Db) => Promise<void>)[] = [];
+
+export function onConnect(callback: (db: Db) => Promise<void>): void {
+  onConnectCallbacks.push(callback);
+}
+
 export async function connect(): Promise<void> {
   clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
     useNewUrlParser: true,
     useUnifiedTopology: true,
   });
 
-  client = await clientPromise;
+  const client = await clientPromise;
   const db = client.db();
+  await Promise.all(onConnectCallbacks.map((c) => c(db)));
+}
 
+onConnect(async (db) => {
   tasksCollection = db.collection("tasks");
   await tasksCollection.createIndex({ device: 1, timestamp: 1 });
 
@@ -76,7 +84,7 @@ export async function connect(): Promise<void> {
   permissionsCollection = db.collection("permissions");
   usersCollection = db.collection("users");
   configCollection = db.collection("config");
-}
+});
 
 export async function disconnect(): Promise<void> {
   if (clientPromise) await (await clientPromise).close();
@@ -123,7 +131,12 @@ export async function fetchDevice(
   const device = await devicesCollection.findOne({ _id: id });
   if (!device) return null;
 
-  function storeParams(obj, path: string, pathLength: number, ts): void {
+  function storeParams(
+    obj,
+    path: string,
+    pathLength: number,
+    ts: number
+  ): void {
     if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
     if (obj["_attributesTimestamp"])
       obj["_attributesTimestamp"] = +obj["_attributesTimestamp"];
@@ -154,19 +167,22 @@ export async function fetchDevice(
     if (obj["_accessList"] != null)
       attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
 
-    res.push([Path.parse(path.slice(0, -1)), t, attrs]);
+    res.push([Path.parse(path), t, attrs]);
 
     for (const [k, v] of Object.entries(obj)) {
       if (!k.startsWith("_")) {
         obj["_object"] = true;
-        storeParams(v, path + k + ".", pathLength + 1, obj["_timestamp"]);
+        storeParams(v, `${path}.${k}`, pathLength + 1, obj["_timestamp"]);
       }
     }
 
-    if ((obj["_object"] || !pathLength) && obj["_timestamp"])
+    if (obj["_object"] && obj["_timestamp"])
-      res.push([Path.parse(path + "*"), obj["_timestamp"]]);
+      res.push([Path.parse(path + ".*"), obj["_timestamp"]]);
   }
 
+  const ts: number = device["_timestamp"] || 0;
+  if (ts) res.push([Path.parse("*"), ts]);
+
   for (const [k, v] of Object.entries(device)) {
     switch (k) {
       case "_lastInform":
@@ -234,10 +250,9 @@ export async function fetchDevice(
           ]);
         }
 
-        for (let t of v as string[]) {
+        for (const t of v as string[]) {
-          t = t.replace(/[^a-zA-Z0-9-]+/g, "_");
           res.push([
-            Path.parse("Tags." + t),
+            Path.parse("Tags." + encodeTag(t)),
             timestamp,
             {
               object: [timestamp, 0],
@@ -295,10 +310,11 @@ export async function fetchDevice(
             },
           ]);
         }
+        break;
+      default:
+        if (!k.startsWith("_")) storeParams(v, k, 1, ts);
     }
   }
-
-  storeParams(device, "", 0, 0);
   return res;
 }
 
@@ -691,6 +707,11 @@ export async function getOperations(
   for (const r of res) {
     const commandKey = r._id.slice(deviceId.length + 1);
     delete r._id;
+    // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
+    if (typeof r.provisions !== "string") {
+      operations[commandKey] = r;
+      continue;
+    }
     r.timestamp = +r.timestamp;
     if (r.args) r.args = JSON.parse(r.args);
     r.provisions = JSON.parse(r.provisions);
@@ -712,7 +733,7 @@ export async function saveOperation(
   o["provisions"] = JSON.stringify(operation.provisions);
   o["retries"] = JSON.stringify(operation.retries);
   o["args"] = JSON.stringify(operation.args);
-  await operationsCollection.replaceOne({ _id: id }, operation, {
+  await operationsCollection.replaceOne({ _id: id }, o, {
     upsert: true,
   });
 }

lib/debug.ts

@@ -25,7 +25,7 @@ import {
 } from "http";
 import { Socket } from "net";
 import { appendFileSync } from "fs";
-import * as yaml from "yaml";
+import { stringify } from "./common/yaml";
 import * as config from "./config";
 
 const DEBUG_FILE = "" + config.get("DEBUG_FILE");
@@ -49,7 +49,7 @@ export function incomingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
   const msg = {
     event: "incoming HTTP request",
     timestamp: now,
@@ -64,7 +64,7 @@ export function incomingHttpRequest(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -77,7 +77,7 @@ export function outgoingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
   const msg = {
     event: "outgoing HTTP response",
     timestamp: now,
@@ -90,7 +90,7 @@ export function outgoingHttpResponse(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -104,7 +104,7 @@ export function outgoingHttpRequest(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
@@ -119,7 +119,7 @@ export function outgoingHttpRequest(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -147,7 +147,7 @@ export function outgoingHttpRequestError(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -160,20 +160,20 @@ export function incomingHttpResponse(
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
   const msg = {
     event: "incoming HTTP response",
     timestamp: now,
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
-    connection: getConnectionTimestamp(httpResponse.connection),
+    connection: getConnectionTimestamp(httpResponse.socket),
     statusCode: httpResponse.statusCode,
     headers: httpResponse.headers,
     body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -197,7 +197,7 @@ export function outgoingUdpMessage(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);

lib/extensions.ts

@@ -22,6 +22,8 @@ import * as crypto from "crypto";
 import * as config from "./config";
 import { Fault } from "./types";
 import { ROOT_DIR } from "./config";
+import * as logger from "./logger";
+import readline from "readline";
 
 const TIMEOUT = +config.get("EXT_TIMEOUT");
 
@@ -45,7 +47,7 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
 
     if (!processes[scriptName]) {
       const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
-        stdio: ["inherit", "inherit", "inherit", "ipc"],
+        stdio: ["ignore", "pipe", "pipe", "ipc"],
       });
       processes[scriptName] = p;
 
@@ -58,6 +60,7 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
             });
           }
 
+          // eslint-disable-next-line @typescript-eslint/no-floating-promises
           kill(processes[scriptName]);
           delete processes[scriptName];
         }
@@ -68,6 +71,16 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
       });
 
       p.on("message", messageHandler);
+
+      const rlstdout = readline.createInterface(p.stdout);
+      rlstdout.on("line", (line) => {
+        logger.info({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
+      });
+
+      const rlstderr = readline.createInterface(p.stderr);
+      rlstderr.on("line", (line) => {
+        logger.warn({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
+      });
     }
 
     setTimeout(() => {

lib/fs.ts

@@ -20,49 +20,75 @@
 import * as url from "url";
 import * as querystring from "querystring";
 import { IncomingMessage, ServerResponse } from "http";
-import { GridFSBucket } from "mongodb";
+import { Collection, GridFSBucket } from "mongodb";
-import * as db from "./db";
+import { onConnect } from "./db";
 import * as logger from "./logger";
 import { getRequestOrigin } from "./forwarded";
+import memoize from "./common/memoize";
 
-export function listener(
+let filesCollection: Collection;
+let filesBucket: GridFSBucket;
+
+onConnect(async (db) => {
+  filesCollection = db.collection("fs.files");
+  filesBucket = new GridFSBucket(db);
+});
+
+const getFile = memoize(
+  (md5: string, size: number, filename: string): Promise<Buffer> => {
+    return new Promise((resolve, reject) => {
+      const buffer = Buffer.allocUnsafe(size);
+      let i = 0;
+      const downloadStream = filesBucket.openDownloadStreamByName(filename);
+      downloadStream.on("error", reject);
+      downloadStream.on("data", (data: Buffer) => {
+        data.copy(buffer, i);
+        i += data.length;
+      });
+      downloadStream.on("end", () => {
+        if (i !== size) reject(new Error("File size mismatch"));
+        else resolve(buffer);
+      });
+    });
+  }
+);
+
+export async function listener(
   request: IncomingMessage,
   response: ServerResponse
-): void {
+): Promise<void> {
-  const urlParts = url.parse(request.url, true);
+  if (request.method !== "GET") {
-  if (request.method === "GET") {
-    const filename = querystring.unescape(urlParts.pathname.substring(1));
-
-    const log = {
-      message: "Fetch file",
-      filename: filename,
-      remoteAddress: getRequestOrigin(request).remoteAddress,
-    };
-
-    db.filesCollection.findOne({ _id: filename }, (err, file) => {
-      if (err) throw err;
-
-      if (!file) {
-        response.writeHead(404);
-        response.end();
-        log.message += " not found";
-        logger.accessError(log);
-        return;
-      }
-
-      response.writeHead(200, {
-        "Content-Type": file.contentType || "application/octet-stream",
-        "Content-Length": file.length,
-      });
-
-      const bucket = new GridFSBucket(db.client.db());
-      const downloadStream = bucket.openDownloadStreamByName(filename);
-      downloadStream.pipe(response);
-
-      logger.accessInfo(log);
-    });
-  } else {
     response.writeHead(405, { Allow: "GET" });
     response.end("405 Method Not Allowed");
+    return;
   }
+
+  const urlParts = url.parse(request.url, true);
+  const filename = querystring.unescape(urlParts.pathname.substring(1));
+
+  const log = {
+    message: "Fetch file",
+    filename: filename,
+    remoteAddress: getRequestOrigin(request).remoteAddress,
+  };
+
+  const file = await filesCollection.findOne({ _id: filename });
+
+  if (!file) {
+    response.writeHead(404);
+    response.end();
+    log.message += " not found";
+    logger.accessError(log);
+    return;
+  }
+
+  const buffer = await getFile(file.md5, file.length, filename);
+
+  response.writeHead(200, {
+    "Content-Type": file.contentType || "application/octet-stream",
+    "Content-Length": file.length,
+  });
+
+  response.end(buffer);
+  logger.accessInfo(log);
 }

lib/local-cache.ts

@@ -22,7 +22,7 @@ import * as crypto from "crypto";
 import * as config from "./config";
 import * as db from "./db";
 import * as cache from "./cache";
-import { mongoQueryToFilter } from "./mongodb-functions";
+import { convertOldPrecondition } from "./mongodb-functions";
 import * as logger from "./logger";
 import * as scheduling from "./scheduling";
 import * as expression from "./common/expression";
@@ -50,8 +50,8 @@ interface Snapshot {
   ui: UiConfig;
 }
 
-const REFRESH = 3000;
+const REFRESH = 5000;
-const EVICT_TIMEOUT = 60000;
+const EVICT_TIMEOUT = 120000;
 
 const snapshots = new Map<string, Snapshot>();
 let currentSnapshot: string = null;
@@ -173,7 +173,9 @@ async function fetchPresets(): Promise<Preset[]> {
       try {
         precondition = parse(preset["precondition"]);
       } catch (error) {
-        precondition = mongoQueryToFilter(JSON.parse(preset["precondition"]));
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"])
+        );
       }
 
       // Simplify expression
@@ -186,7 +188,7 @@ async function fetchPresets(): Promise<Preset[]> {
     for (const c of preset["configurations"]) {
       switch (c.type) {
         case "age":
-          _provisions.push(["refresh", c.name, c.age]);
+          _provisions.push(["refresh", c.name, +c.age]);
           break;
 
         case "value":
@@ -396,7 +398,7 @@ async function refresh(): Promise<void> {
     return;
   }
 
-  const unlockOrExtend = await cache.lock("presets_hash_lock", 3);
+  const lockToken = await cache.acquireLock("presets_hash_lock", 5000, 5000);
 
   const res = await Promise.all([
     fetchPresets(),
@@ -430,7 +432,7 @@ async function refresh(): Promise<void> {
   currentSnapshot = computeHash(snapshot);
   snapshots.set(currentSnapshot, snapshot);
   await cache.set("presets_hash", currentSnapshot, 300);
-  await unlockOrExtend(0);
+  await cache.releaseLock("presets_hash_lock", lockToken);
 
   nextRefresh = now + (REFRESH - (now % REFRESH));
 }
@@ -473,7 +475,7 @@ export function getConfig(
   key: string,
   context: Record<string, unknown>,
   now: number,
-  cb?: (Expression) => Expression
+  cb?: (e: Expression) => Expression
 ): string | number | boolean | null {
   const snapshot = snapshots.get(snapshotKey);
   if (!snapshot) throw new Error("Cache snapshot does not exist");

lib/mongodb-functions.ts

@@ -19,8 +19,9 @@
 
 import { ObjectID } from "mongodb";
 import { map, parse, stringify, parseList } from "./common/expression-parser";
-import { likePatternToRegExp, evaluate } from "./common/expression";
+import { likePatternToRegExp } from "./common/expression";
 import { Expression, Fault, Task } from "./types";
+import { decodeTag, encodeTag } from "./common";
 
 const isArray = Array.isArray;
 
@@ -43,13 +44,11 @@ export function processDeviceFilter(filter: Expression): Expression {
       exp[1][0] === "PARAM" &&
       exp[1][1].startsWith("Tags.")
     ) {
-      const t = exp[1][1].slice(5);
+      const t = decodeTag(exp[1][1].slice(5));
-      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
+      if (exp[0] === "IS NULL") return ["_tags", t, false];
-      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
-      else if (exp[0] === "=" && exp[2] === true)
+      else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
-        return ["=", ["PARAM", "_tags"], t];
+      else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
-      else if (exp[0] === "<>" && exp[2] !== true)
-        return ["=", ["PARAM", "_tags"], t];
     } else if (
       [
         "=",
@@ -117,140 +116,131 @@ export function processFaultsFilter(filter: Expression): Expression {
 }
 
 export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
-  const ops = {
+  function recursive(filter, negate): Record<string, unknown> {
-    OR: 0,
-    AND: 0,
-    NOT: 0,
-    "=": 1,
-    "<>": 1,
-    ">": 1,
-    ">=": 1,
-    "<": 1,
-    "<=": 1,
-    LIKE: 1,
-    "NOT LIKE": 1,
-    "IS NULL": 1,
-    "IS NOT NULL": 1,
-  };
-
-  function recursive(filter, negate, res = {}): Record<string, unknown> {
     const op = filter[0];
 
-    if (ops[op] === 0) {
+    if (op === "AND") {
-      if (op === "AND") filter = filter.filter((f) => f !== true);
+      filter = filter.filter((f) => f !== true);
-      if (op === "OR") filter = filter.filter((f) => f !== false);
+      if (filter.length === 2) return recursive(filter[1], negate);
-      if (filter.length === 2) return recursive(filter[1], negate, res);
-
-      for (let i = 1; i < filter.length; ++i) {
-        if (!isArray(filter[i]) || ops[filter[i][0]] == null)
-          throw new Error(`Invalid expression in ${op} clause`);
-      }
-
-      if ((!negate && op === "AND") || (negate && op === "OR")) {
-        res["$and"] = res["$and"] || [];
-        for (let i = 1; i < filter.length; ++i)
-          res["$and"].push(recursive(filter[i], negate));
-      } else if ((!negate && op === "OR") || (negate && op === "AND")) {
-        res["$or"] = res["$or"] || [];
-
-        for (let i = 1; i < filter.length; ++i)
-          res["$or"].push(recursive(filter[i], negate));
-      } else if (op === "NOT") {
-        recursive(filter[1], !negate, res);
-      }
-    } else if (ops[op] === 1) {
-      if (isArray(filter[2])) {
-        throw new Error(`Right operand of ${op} clause is not a primitive`);
-      } else if (
-        ["LIKE", "NOT LIKE"].includes(op) &&
-        isArray(filter[1]) &&
-        filter[1][0] === "FUNC" &&
-        ["UPPER", "LOWER"].includes(filter[1][1])
-      ) {
-        if (
-          (filter[1][1] === "UPPER" && filter[2] !== filter[2].toUpperCase()) ||
-          (filter[1][1] === "LOWER" && filter[2] !== filter[2].toLowerCase())
-        ) {
-          throw new Error(
-            `Cannot compare ${
-              filter[1][1]
-            }() against non-${filter[1][1].toLowerCase()}case pattern`
-          );
-        }
-      } else if (!isArray(filter[1]) || filter[1][0] !== "PARAM") {
-        throw new Error(`Left operand of ${op} clause is not a parameter`);
-      }
-
-      if (op === "=") {
-        const param = filter[1][1];
-        const p = (res[param] = res[param] || {});
-        if (negate) p["$ne"] = filter[2];
-        else p["$eq"] = filter[2];
-      } else if (op === "<>") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$ne"] = filter[2];
-        if (param !== "_tags") p["$exists"] = true;
-      } else if (op === ">") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gt"] = filter[2];
-      } else if (op === ">=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gte"] = filter[2];
-      } else if (op === "<") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lt"] = filter[2];
-      } else if (op === "<=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lte"] = filter[2];
-      } else if (op === "IS NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: negate };
-      } else if (op === "IS NOT NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: !negate };
-      } else {
-        if (op === "NOT LIKE") negate = !negate;
-        let param;
-        let flags;
-        if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
-          param = filter[1][2][1];
-          flags = "i";
-        } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
-          param = filter[1][2][1];
-          flags = "i";
-        } else {
-          param = filter[1][1];
-          flags = "";
-        }
-        const r = likePatternToRegExp(filter[2], filter[3], flags);
-        if (negate) res[param] = { $not: r };
-        else res[param] = r;
-      }
-    } else {
-      throw new Error(`Unrecognized operator ${op}`);
     }
 
-    return res;
+    if (op === "OR") {
+      filter = filter.filter((f) => f !== false);
+      if (filter.length === 2) return recursive(filter[1], negate);
+    }
+
+    if ((!negate && op === "AND") || (negate && op === "OR"))
+      return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if ((!negate && op === "OR") || (negate && op === "AND"))
+      return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if (op === "NOT") return recursive(filter[1], !negate);
+
+    if (isArray(filter[2]))
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "LIKE" || op === "NOT LIKE") {
+      if (op === "NOT LIKE") negate = !negate;
+      let param;
+      let flags;
+      if (!Array.isArray(filter[1]))
+        throw new Error(`Invalid LHS operand of ${op} clause`);
+      if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
+        if (filter[2] !== filter[2].toUpperCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
+        param = filter[1][2][1];
+        flags = "i";
+      } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
+        if (filter[2] !== filter[2].toLowerCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
+        param = filter[1][2][1];
+        flags = "i";
+      } else if (filter[1][0] === "PARAM") {
+        param = filter[1][1];
+        flags = "";
+      } else {
+        throw new Error(`Invalid LHS operand of ${op} clause`);
+      }
+      const r = likePatternToRegExp(filter[2], filter[3], flags);
+      if (negate) return { [param]: { $nin: [r, null] } };
+      else return { [param]: r };
+    }
+
+    if (op === "_tags") {
+      let t = filter[2];
+      if (negate) t = !t;
+      if (t) return { _tags: filter[1] };
+      else return { _tags: { $ne: filter[1] } };
+    }
+
+    if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
+      throw new Error(`Invalid LHS operand of ${op} clause`);
+
+    if (op === "IS NULL") {
+      const p = filter[1][1];
+      return { [p]: null };
+    }
+
+    if (op === "IS NOT NULL") {
+      const p = filter[1][1];
+      return { [p]: { $ne: null } };
+    }
+
+    if (isArray(filter[2]) || filter[2] == null)
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (!negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === "<>") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === ">") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lte: v } };
+      else return { [p]: { $gt: v } };
+    }
+
+    if (op === ">=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lt: v } };
+      else return { [p]: { $gte: v } };
+    }
+
+    if (op === "<") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gte: v } };
+      else return { [p]: { $lt: v } };
+    }
+
+    if (op === "<=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gt: v } };
+      else return { [p]: { $lte: v } };
+    }
+
+    throw new Error(`Unrecognized operator ${op}`);
   }
 
-  const _exp = evaluate(exp, null, Date.now());
+  if (!isArray(exp)) {
-
+    if (exp === true) return {};
-  if (!isArray(_exp)) {
-    if (_exp === true) return {};
     throw new Error("Primitives are not valid queries");
   }
 
-  return recursive(_exp, false);
+  return recursive(exp, false);
 }
 
 export function processDeviceProjection(
@@ -417,7 +407,7 @@ export function flattenDevice(device: Record<string, unknown>): FlatDevice {
           };
 
           for (const t of tree as string[]) {
-            output[`Tags.${t}`] = {
+            output[`Tags.${encodeTag(t)}`] = {
               value: [true, "xsd:boolean"],
               valueTimestamp: timestamp,
               writable: true,
@@ -495,7 +485,9 @@ export function flattenTask(task: unknown): Task {
   return t;
 }
 
-export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
+export function convertOldPrecondition(
+  query: Record<string, unknown>
+): Expression {
   function recursive(_query): Expression {
     const expressions: Expression[] = [];
     for (const [k, v] of Object.entries(_query)) {
@@ -514,14 +506,27 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
       } else if (k === "_tags") {
         if (typeof v === "object") {
           if (isArray(v)) throw new Error(`Invalid type`);
-
+          const conjs: Expression[] = [];
-          if (v.hasOwnProperty("$ne"))
+          for (const [op, val] of Object.entries(v)) {
-            expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]);
+            if (op === "$ne") {
-          else if (v.hasOwnProperty("$eq"))
+              if (typeof v["$ne"] !== "string")
-            expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v["$eq"]}`]]);
+                throw new Error("Only string values are allowed for _tags");
-          else throw new Error(`Invalid tag query`);
+              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else {
+              throw new Error(`Invalid tag query`);
+            }
+          }
+          if (conjs.length === 1) expressions.push(conjs[0]);
+          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
         } else {
-          expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
+          expressions.push([
+            "IS NOT NULL",
+            ["PARAM", `Tags.${encodeTag(v as string)}`],
+          ]);
         }
       } else if (k.startsWith("Tags.")) {
         let exists: boolean;
@@ -537,34 +542,31 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
 
         const exps: Expression[] = [];
         for (const [kk, vv] of Object.entries(v)) {
-          let op;
+          if (kk === "$eq") {
-          switch (kk) {
+            exps.push(["=", ["PARAM", k], vv]);
-            case "$eq":
+          } else if (kk === "$ne") {
-              op = "=";
+            exps.push([
-              break;
+              "OR",
-            case "$ne":
+              ["<>", ["PARAM", k], vv],
-              op = "<>";
+              ["IS NULL", ["PARAM", k]],
-              break;
+            ]);
-            case "$lt":
+          } else if (kk === "$lt") {
-              op = "<";
+            exps.push(["<", ["PARAM", k], vv]);
-              break;
+          } else if (kk === "$lte") {
-            case "$lte":
+            exps.push(["<=", ["PARAM", k], vv]);
-              op = "<=";
+          } else if (kk === "$gt") {
-              break;
+            exps.push([">", ["PARAM", k], vv]);
-            case "$gt":
+          } else if (kk === "$gte") {
-              op = ">";
+            exps.push([">=", ["PARAM", k], vv]);
-              break;
+          } else {
-            case "$gte":
+            throw new Error(`Operator ${kk} not supported`);
-              op = ">=";
-              break;
-            default:
-              throw new Error(`Operator ${kk} not supported`);
           }
-          exps.push([op, ["PARAM", k], vv]);
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
         }
         if (exps.length === 1) {
           expressions.push(exps[0]);
-        } else {
+        } else if (exps.length > 1) {
           const and: Expression = ["AND"];
           expressions.push(and.concat(exps));
         }
@@ -573,8 +575,8 @@ export function mongoQueryToFilter(query: Record<string, unknown>): Expression {
       }
     }
     if (expressions.length === 1) return expressions[0];
-    const and: Expression = ["AND"];
+    if (expressions.length === 0) return true;
-    return and.concat(expressions);
+    return ["AND", ...expressions];
   }
 
   // empty filter
@@ -592,7 +594,9 @@ export function flattenPreset(
       // Try parse to check expression validity
       parse(p.precondition as string);
     } catch (error) {
-      p.precondition = mongoQueryToFilter(JSON.parse(p.precondition as string));
+      p.precondition = convertOldPrecondition(
+        JSON.parse(p.precondition as string)
+      );
       p.precondition = (p.precondition as string).length
         ? stringify(p.precondition as Expression)
         : "";

lib/nbi.ts

@@ -18,17 +18,18 @@
  */
 
 import * as url from "url";
-import * as mongodb from "mongodb";
+import { Collection, GridFSBucket, ObjectId } from "mongodb";
 import * as querystring from "querystring";
 import * as vm from "vm";
 import * as config from "./config";
-import * as db from "./db";
+import { onConnect } from "./db";
 import * as query from "./query";
 import * as apiFunctions from "./api-functions";
 import { IncomingMessage, ServerResponse } from "http";
 import * as cache from "./cache";
 import { version as VERSION } from "../package.json";
 import { ping } from "./ping";
+import * as logger from "./logger";
 
 const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
 const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
@@ -43,6 +44,26 @@ const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
 const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
 
+const collections = {
+  tasks: null as Collection,
+  devices: null as Collection,
+  presets: null as Collection,
+  objects: null as Collection,
+  files: null as Collection,
+  provisions: null as Collection,
+  virtualParameters: null as Collection,
+  faults: null as Collection,
+};
+let filesBucket: GridFSBucket;
+
+onConnect(async (db) => {
+  for (const k of Object.keys(collections)) {
+    if (k === "files") collections[k] = db.collection("fs.files");
+    else collections[k] = db.collection(k);
+  }
+  filesBucket = new GridFSBucket(db);
+});
+
 function throwError(err: Error, httpResponse?: ServerResponse): never {
   if (httpResponse) {
     httpResponse.writeHead(500, { Connection: "close" });
@@ -78,6 +99,14 @@ export function listener(
   request.addListener("end", () => {
     const body = getBody();
     const urlParts = url.parse(request.url, true);
+
+    logger.accessInfo(
+      Object.assign({}, urlParts.query, {
+        remoteAddress: request.connection.remoteAddress,
+        message: `${request.method} ${urlParts.pathname}`,
+      })
+    );
+
     if (PRESETS_REGEX.test(urlParts.pathname)) {
       const presetName = querystring.unescape(
         PRESETS_REGEX.exec(urlParts.pathname)[1]
@@ -85,7 +114,7 @@ export function listener(
       if (request.method === "PUT") {
         const preset = JSON.parse(body.toString());
         preset._id = presetName;
-        db.presetsCollection.replaceOne(
+        collections.presets.replaceOne(
           { _id: presetName },
           preset,
           { upsert: true },
@@ -106,7 +135,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        db.presetsCollection.deleteOne({ _id: presetName }, (err) => {
+        collections.presets.deleteOne({ _id: presetName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -132,7 +161,7 @@ export function listener(
       if (request.method === "PUT") {
         const object = JSON.parse(body.toString());
         object._id = objectName;
-        db.objectsCollection.replaceOne(
+        collections.objects.replaceOne(
           { _id: objectName },
           object,
           { upsert: true },
@@ -153,7 +182,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        db.objectsCollection.deleteOne({ _id: objectName }, (err) => {
+        collections.objects.deleteOne({ _id: objectName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -190,7 +219,7 @@ export function listener(
           return;
         }
 
-        db.provisionsCollection.replaceOne(
+        collections.provisions.replaceOne(
           { _id: provisionName },
           object,
           { upsert: true },
@@ -211,7 +240,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        db.provisionsCollection.deleteOne({ _id: provisionName }, (err) => {
+        collections.provisions.deleteOne({ _id: provisionName }, (err) => {
           if (err) return void throwError(err, response);
 
           cache
@@ -248,7 +277,7 @@ export function listener(
           return;
         }
 
-        db.virtualParametersCollection.replaceOne(
+        collections.virtualParameters.replaceOne(
           { _id: virtualParameterName },
           object,
           { upsert: true },
@@ -269,7 +298,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        db.virtualParametersCollection.deleteOne(
+        collections.virtualParameters.deleteOne(
           { _id: virtualParameterName },
           (err) => {
             if (err) return void throwError(err, response);
@@ -296,7 +325,7 @@ export function listener(
       const deviceId = querystring.unescape(r[1]);
       const tag = querystring.unescape(r[2]);
       if (request.method === "POST") {
-        db.devicesCollection.updateOne(
+        collections.devices.updateOne(
           { _id: deviceId },
           { $addToSet: { _tags: tag } },
           (err) => {
@@ -306,7 +335,7 @@ export function listener(
           }
         );
       } else if (request.method === "DELETE") {
-        db.devicesCollection.updateOne(
+        collections.devices.updateOne(
           { _id: deviceId },
           { $pull: { _tags: tag } },
           (err) => {
@@ -327,12 +356,12 @@ export function listener(
         );
         const deviceId = faultId.split(":", 1)[0];
         const channel = faultId.slice(deviceId.length + 1);
-        db.faultsCollection.deleteOne({ _id: faultId }, (err) => {
+        collections.faults.deleteOne({ _id: faultId }, (err) => {
           if (err) return void throwError(err, response);
 
           if (channel.startsWith("task_")) {
-            const objId = new mongodb.ObjectID(channel.slice(5));
+            const objId = new ObjectId(channel.slice(5));
-            return void db.tasksCollection.deleteOne({ _id: objId }, (err) => {
+            return void collections.tasks.deleteOne({ _id: objId }, (err) => {
               if (err) return void throwError(err, response);
 
               cache
@@ -375,78 +404,112 @@ export function listener(
           task.device = deviceId;
           apiFunctions
             .insertTasks(task)
-            .then(() => {
+            .then(async () => {
-              cache
+              const lastInform = Date.now();
-                .del(`${deviceId}_tasks_faults_operations`)
-                .then(() => {
-                  if (urlParts.query.connection_request != null) {
-                    apiFunctions
-                      .connectionRequest(deviceId)
-                      .then(() => {
-                        const taskTimeout =
-                          (urlParts.query.timeout &&
-                            parseInt(urlParts.query.timeout as string)) ||
-                          (config.get(
-                            "DEVICE_ONLINE_THRESHOLD",
-                            deviceId
-                          ) as number);
 
-                        apiFunctions
+              const socketTimeout: number = request.socket["timeout"];
-                          .watchTask(deviceId, task._id, taskTimeout)
-                          .then((status) => {
-                            if (status === "timeout") {
-                              response.writeHead(
-                                202,
-                                "Task queued but not processed",
-                                {
-                                  "Content-Type": "application/json",
-                                }
-                              );
-                              response.end(JSON.stringify(task));
-                            } else if (status === "fault") {
-                              db.tasksCollection.findOne(
-                                { _id: task._id },
-                                (err, task2) => {
-                                  if (err)
-                                    return void throwError(err, response);
 
-                                  response.writeHead(202, "Task faulted", {
+              // Disable socket timeout while waiting for session
-                                    "Content-Type": "application/json",
+              if (socketTimeout) request.socket.setTimeout(0);
-                                  });
+
-                                  response.end(JSON.stringify(task2));
+              const notInSession = await apiFunctions.awaitSessionEnd(
-                                }
+                deviceId,
-                              );
+                30000
-                            } else {
+              );
-                              response.writeHead(200, {
+              await cache.del(`${deviceId}_tasks_faults_operations`);
-                                "Content-Type": "application/json",
+              if (urlParts.query.connection_request == null) {
-                              });
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
-                              response.end(JSON.stringify(task));
+                response.writeHead(202, {
-                            }
+                  "Content-Type": "application/json",
-                          })
-                          .catch((err) => {
-                            setTimeout(() => {
-                              throwError(err, response);
-                            });
-                          });
-                      })
-                      .catch((err) => {
-                        response.writeHead(202, err.message, {
-                          "Content-Type": "application/json",
-                        });
-                        response.end(JSON.stringify(task));
-                      });
-                  } else {
-                    response.writeHead(202, {
-                      "Content-Type": "application/json",
-                    });
-                    response.end(JSON.stringify(task));
-                  }
-                })
-                .catch((err) => {
-                  setTimeout(() => {
-                    throwError(err, response);
-                  });
                 });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              if (!notInSession) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const status = await apiFunctions.connectionRequest(deviceId);
+
+              if (status) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, status, {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const onlineThreshold =
+                (urlParts.query.timeout &&
+                  parseInt(urlParts.query.timeout as string)) ||
+                (config.get("DEVICE_ONLINE_THRESHOLD", deviceId) as number);
+
+              const sessionStarted = await apiFunctions.awaitSessionStart(
+                deviceId,
+                lastInform,
+                onlineThreshold
+              );
+              if (!sessionStarted) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const sessionEnded = await apiFunctions.awaitSessionEnd(
+                deviceId,
+                120000
+              );
+              if (!sessionEnded) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const prom1 = collections.tasks.findOne(
+                { _id: task._id },
+                { projection: { _id: 1 } }
+              );
+              const prom2 = collections.faults.findOne(
+                { _id: `${deviceId}:task_${task._id}` },
+                {
+                  projection: { _id: 1 },
+                }
+              );
+
+              const [t, f] = await Promise.all([prom1, prom2]);
+
+              // Restore socket timeout
+              if (socketTimeout) request.socket.setTimeout(socketTimeout);
+
+              if (f) {
+                response.writeHead(202, "Task faulted", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(t || task));
+              } else if (t) {
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(t));
+              } else {
+                response.writeHead(200, {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+              }
             })
             .catch((err) => {
               setTimeout(() => {
@@ -479,8 +542,8 @@ export function listener(
       const action = r[2];
       if (!action || action === "/") {
         if (request.method === "DELETE") {
-          db.tasksCollection.findOne(
+          collections.tasks.findOne(
-            { _id: new mongodb.ObjectID(taskId) },
+            { _id: new ObjectId(taskId) },
             { projection: { device: 1 } },
             (err, task) => {
               if (err) return void throwError(err, response);
@@ -492,12 +555,12 @@ export function listener(
               }
 
               const deviceId = task.device;
-              db.tasksCollection.deleteOne(
+              collections.tasks.deleteOne(
-                { _id: new mongodb.ObjectID(taskId) },
+                { _id: new ObjectId(taskId) },
                 (err) => {
                   if (err) return void throwError(err, response);
 
-                  db.faultsCollection.deleteOne(
+                  collections.faults.deleteOne(
                     { _id: `${deviceId}:task_${taskId}` },
                     (err) => {
                       if (err) return void throwError(err, response);
@@ -525,14 +588,14 @@ export function listener(
         }
       } else if (action === "/retry") {
         if (request.method === "POST") {
-          db.tasksCollection.findOne(
+          collections.tasks.findOne(
-            { _id: new mongodb.ObjectID(taskId) },
+            { _id: new ObjectId(taskId) },
             { projection: { device: 1 } },
             (err, task) => {
               if (err) return void throwError(err, response);
 
               const deviceId = task.device;
-              db.faultsCollection.deleteOne(
+              collections.faults.deleteOne(
                 { _id: `${deviceId}:task_${taskId}` },
                 (err) => {
                   if (err) return void throwError(err, response);
@@ -571,9 +634,8 @@ export function listener(
           productClass: request.headers.productclass,
           version: request.headers.version,
         };
-        const bucket = new mongodb.GridFSBucket(db.client.db());
+        filesBucket.delete((filename as unknown) as ObjectId, () => {
-        bucket.delete((filename as unknown) as mongodb.ObjectId, () => {
+          const uploadStream = filesBucket.openUploadStreamWithId(
-          const uploadStream = bucket.openUploadStreamWithId(
             filename,
             filename,
             {
@@ -591,8 +653,7 @@ export function listener(
           });
         });
       } else if (request.method === "DELETE") {
-        const bucket = new mongodb.GridFSBucket(db.client.db());
+        filesBucket.delete((filename as unknown) as ObjectId, (err) => {
-        bucket.delete((filename as unknown) as mongodb.ObjectId, (err) => {
           if (err) {
             if (err.message.startsWith("FileNotFound")) {
               response.writeHead(404);
@@ -669,7 +730,7 @@ export function listener(
         return;
       }
 
-      const collection = db[`${collectionName}Collection`];
+      const collection = collections[collectionName];
       if (!collection) {
         response.writeHead(404);
         response.end("404 Not Found");
@@ -693,7 +754,7 @@ export function listener(
           break;
         case "tasks":
           q = query.sanitizeQueryTypes(q, {
-            _id: (v) => new mongodb.ObjectID(v as string),
+            _id: (v) => new ObjectId(v as string),
             timestamp: (v) => new Date(v as number),
             retries: Number,
           });

lib/ping.ts

@@ -35,6 +35,7 @@ export function ping(
   callback: (err: Error, res?: PingResult, stdout?: string) => void
 ): void {
   let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
+  host = host.replace("[", "").replace("]", "");
   switch (platform()) {
     case "linux":
       cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;

lib/server.ts

@@ -52,35 +52,43 @@ function closeServer(timeout, callback): void {
   });
 }
 
+interface ServerOptions {
+  port?: number;
+  host?: string;
+  ssl?: { key: string; cert: string };
+  timeout?: number;
+  keepAliveTimeout?: number;
+  onConnection?: (socket: Socket) => void;
+}
+
 export function start(
-  port: number,
+  options: ServerOptions,
-  networkInterface: string,
+  _listener: http.RequestListener
-  ssl: { key: string; cert: string },
-  _listener: http.RequestListener,
-  onConnection?: (socket: Socket) => void,
-  keepAliveTimeout = -1
 ): void {
   listener = _listener;
 
-  if (ssl && ssl.key && ssl.cert) {
+  if (options.ssl) {
-    const options = {
+    const opts = {
-      key: ssl.key
+      key: options.ssl.key
         .split(":")
         .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
-      cert: ssl.cert
+      cert: options.ssl.cert
         .split(":")
         .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
     };
 
-    server = https.createServer(options, listener);
+    server = https.createServer(opts, listener);
-    if (onConnection != null) server.on("secureConnection", onConnection);
+    if (options.onConnection)
+      server.on("secureConnection", options.onConnection);
   } else {
     server = http.createServer(listener);
-    if (onConnection != null) server.on("connection", onConnection);
+    if (options.onConnection) server.on("connection", options.onConnection);
   }
 
-  if (keepAliveTimeout >= 0) server.keepAliveTimeout = keepAliveTimeout;
+  server.timeout = options.timeout || 0;
-  server.listen(port, networkInterface);
+  if (options.keepAliveTimeout != null)
+    server.keepAliveTimeout = options.keepAliveTimeout;
+  server.listen({ port: options.port, host: options.host });
 }
 
 export function stop(): Promise<void> {

lib/session.ts

@@ -299,7 +299,7 @@ export async function transferComplete(
   sessionContext.operationsTouched[commandKey] = 1;
 
   if (rpcReq.faultStruct && rpcReq.faultStruct.faultCode !== "0") {
-    await revertDownloadParameters(sessionContext, operation.args.instance);
+    revertDownloadParameters(sessionContext, operation.args.instance);
 
     const fault: Fault = {
       code: `cwmp.${rpcReq.faultStruct.faultCode}`,
@@ -398,10 +398,10 @@ export async function transferComplete(
   };
 }
 
-async function revertDownloadParameters(
+function revertDownloadParameters(
   sessionContext: SessionContext,
   instance
-): Promise<void> {
+): void {
   const timestamp = sessionContext.timestamp + sessionContext.iteration + 1;
 
   let p;
@@ -910,10 +910,15 @@ function runDeclarations(
         if (!unpacked)
           unpacked = device.unpack(sessionContext.deviceData, path);
 
-        for (const u of unpacked)
+        for (const u of unpacked) {
           mergeAttributeValues(u, declaration.attrSet, declaration.defer);
+          // Ensure writable attr is available
+          mergeAttributeTimestamps(u, { writable: 1 });
+        }
       } else {
         mergeAttributeValues(path, declaration.attrSet, declaration.defer);
+        // Ensure writable attr is available
+        mergeAttributeTimestamps(path, { writable: 1 });
       }
     }
 
@@ -937,14 +942,15 @@ function runDeclarations(
         keys = {};
       }
 
-      if (
+      if (!parent.wildcard && !parent.alias) {
-        ((path.wildcard | path.alias) & ((1 << (path.length - 1)) - 1)) ===
-        0
-      ) {
         parent = sessionContext.deviceData.paths.add(parent);
         if (!unpacked)
           unpacked = device.unpack(sessionContext.deviceData, path);
 
+        // Ensure writable attr is available
+        mergeAttributeTimestamps(parent, { writable: 1 });
+        for (const u of unpacked) mergeAttributeTimestamps(u, { writable: 1 });
+
         processInstances(
           sessionContext,
           parent,
@@ -960,13 +966,19 @@ function runDeclarations(
           parent
         );
         for (const par of parentsUnpacked) {
+          const up = device.unpack(
+            sessionContext.deviceData,
+            par.concat(path.slice(-1))
+          );
+
+          // Ensure writable attr is available
+          mergeAttributeTimestamps(par, { writable: 1 });
+          for (const u of up) mergeAttributeTimestamps(u, { writable: 1 });
+
           processInstances(
             sessionContext,
             par,
-            device.unpack(
+            up,
-              sessionContext.deviceData,
-              par.concat(path.slice(-1))
-            ),
             keys,
             minInstances,
             maxInstances,
@@ -1734,20 +1746,35 @@ function generateSetRpcRequest(
 
   const deviceData = sessionContext.deviceData;
 
+  const SKIP_WRITABLE_CHECK = !!localCache.getConfig(
+    sessionContext.cacheSnapshot,
+    "cwmp.skipWritableCheck",
+    {},
+    sessionContext.timestamp,
+    (e) => configContextCallback(sessionContext, e)
+  );
+
+  const canWrite = (attrs: Attributes): boolean =>
+    SKIP_WRITABLE_CHECK || (attrs.writable && !!attrs.writable[1]);
+
   // Delete instance
   for (const instances of syncState.instancesToDelete.values()) {
-    const instance = instances.values().next().value;
+    for (const instance of instances) {
-    if (instance && sessionContext.deviceData.attributes.has(instance)) {
+      const attrs = sessionContext.deviceData.attributes.get(instance);
-      return {
+      if (attrs && canWrite(attrs)) {
-        name: "DeleteObject",
+        instances.delete(instance);
-        objectName: instance.toString() + ".",
+        return {
-      };
+          name: "DeleteObject",
+          objectName: instance.toString() + ".",
+        };
+      }
     }
   }
 
   // Create instance
   for (const [param, instances] of syncState.instancesToCreate) {
-    if (sessionContext.deviceData.attributes.has(param)) {
+    const attrs = sessionContext.deviceData.attributes.get(param);
+    if (attrs && canWrite(attrs)) {
       const instance = instances.values().next().value;
       if (instance) {
         instances.delete(instance);
@@ -1791,7 +1818,7 @@ function generateSetRpcRequest(
     syncState.spv.delete(k);
     const attrs = sessionContext.deviceData.attributes.get(k);
     const curVal = attrs.value ? attrs.value[1] : null;
-    if (curVal && attrs.writable && attrs.writable[1]) {
+    if (curVal && canWrite(attrs)) {
       const val = v.slice() as [string | number | boolean, string];
       if (!val[1]) val[1] = curVal[1];
       device.sanitizeParameterValue(val);
@@ -2392,7 +2419,7 @@ export async function rpcResponse(
   sessionContext: SessionContext,
   id: string,
   _rpcRes: CpeResponse
-): Promise<void> {
+): Promise<Fault> {
   if (id !== generateRpcId(sessionContext))
     throw new Error("Request ID not recognized");
 
@@ -2487,6 +2514,14 @@ export async function rpcResponse(
     if (rpcReq.name !== "GetParameterValues")
       throw new Error("Response name does not match request name");
 
+    if (rpcRes.parameterList.length !== rpcReq.parameterNames.length) {
+      return {
+        code: `invalid_response`,
+        message:
+          "The list of parameters returned in the GetParameterValuesResponse message does not match the requested parameters",
+      };
+    }
+
     for (const p of rpcRes.parameterList) {
       toClear = device.set(
         sessionContext.deviceData,
@@ -2609,7 +2644,6 @@ export async function rpcResponse(
         timestamp + 1,
         {
           object: [timestamp + 1, 0],
-          writable: [timestamp + 1, 1],
           value: [
             timestamp + 1,
             p.slice(1) as [string | number | boolean, string],
@@ -2787,6 +2821,8 @@ export async function rpcResponse(
     for (const c of toClear)
       device.clear(sessionContext.deviceData, c[0], c[1], c[2], c[3]);
   }
+
+  return null;
 }
 
 export async function rpcFault(

lib/soap.ts

@@ -127,7 +127,8 @@ function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
 
       if (parsed == null) {
         warnings.push({
-          message: "Invalid writable attribute",
+          message: "Missing or invalid XML node",
+          element: "Writable",
           parameter: param,
         });
         parsed = false;
@@ -167,7 +168,8 @@ function parameterValueList(
       let valueType = getValueType(valueElement.attrs);
       if (!valueType) {
         warnings.push({
-          message: "Invalid value type attribute",
+          message: "Missing or invalid XML node",
+          attribute: "type",
           parameter: param,
         });
         valueType = "xsd:string";
@@ -179,7 +181,8 @@ function parameterValueList(
         parsed = parseBool(value);
         if (parsed == null) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
+            element: "Value",
             parameter: param,
           });
           parsed = value;
@@ -188,7 +191,8 @@ function parameterValueList(
         parsed = parseInt(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
+            element: "Value",
             parameter: param,
           });
           parsed = value;
@@ -197,7 +201,8 @@ function parameterValueList(
         parsed = Date.parse(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
+            message: "Missing or invalid XML node",
+            element: "Value",
             parameter: param,
           });
           parsed = value;
@@ -232,7 +237,8 @@ function parameterAttributeList(xml: Element): [Path, number, string[]][] {
       let notification = parseInt(notificationElement.text);
       if (isNaN(notification)) {
         warnings.push({
-          message: "Invalid notification attribute",
+          message: "Missing or invalid XML node",
+          element: "Notification",
           parameter: param,
         });
         notification = 0;
@@ -331,7 +337,10 @@ function SetParameterValuesResponse(xml: Element): SetParameterValuesResponse {
   }
 
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid SetParameterValuesResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
@@ -392,8 +401,14 @@ function AddObjectResponse(xml: Element): AddObjectResponse {
     }
   }
 
+  if (!/^[0-9]+$/.test(instanceNumber))
+    throw new Error("Missing or invalid instance number");
+
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid AddObjectResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
@@ -424,7 +439,10 @@ function DeleteObjectResponse(xml: Element): DeleteObjectResponse {
   }
 
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid DeleteObjectResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
@@ -495,10 +513,29 @@ function DownloadResponse(xml: Element): DownloadResponse {
   }
 
   if (!(status >= 0)) {
-    warnings.push({ message: "Invalid DownloadResponse status" });
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
     status = 0;
   }
 
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "DownloadResponse",
     status: status,
@@ -537,6 +574,30 @@ function Inform(xml: Element): InformRequest {
     }
   }
 
+  if (!deviceId || !deviceId.SerialNumber || !deviceId.OUI)
+    throw new Error("Missing or invalid DeviceId element");
+
+  if (!parameterList) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "ParameterList",
+    });
+    parameterList = [];
+  }
+
+  if (!evnt) {
+    warnings.push({ message: "Missing or invalid XML node", element: "Event" });
+    evnt = [];
+  }
+
+  if (retryCount == null || isNaN(retryCount)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "RetryCount",
+    });
+    retryCount = 0;
+  }
+
   return {
     name: "Inform",
     parameterList: parameterList,
@@ -563,7 +624,10 @@ function GetRPCMethodsResponse(methodResponse): string {
 }
 
 function TransferComplete(xml: Element): TransferCompleteRequest {
-  let commandKey, _faultStruct, startTime, completeTime;
+  let commandKey: string,
+    _faultStruct: FaultStruct,
+    startTime: number,
+    completeTime: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "CommandKey":
@@ -581,6 +645,38 @@ function TransferComplete(xml: Element): TransferCompleteRequest {
     }
   }
 
+  if (commandKey == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CommandKey",
+    });
+    commandKey = "";
+  }
+
+  if (!_faultStruct) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultStruct",
+    });
+    _faultStruct = { faultCode: "0", faultString: "" };
+  }
+
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "TransferComplete",
     commandKey: commandKey,
@@ -606,7 +702,12 @@ function RequestDownloadResponse(): string {
 }
 
 function faultStruct(xml: Element): FaultStruct {
-  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
+  let faultCode: string,
+    faultString: string,
+    setParameterValuesFault: SpvFault[],
+    pn: string,
+    fc: string,
+    fs: string;
   for (const c of xml.children) {
     switch (c.localName) {
       case "FaultCode":
@@ -617,6 +718,7 @@ function faultStruct(xml: Element): FaultStruct {
         break;
       case "SetParameterValuesFault":
         setParameterValuesFault = setParameterValuesFault || [];
+        pn = fc = fs = null;
         for (const cc of c.children) {
           switch (cc.localName) {
             case "ParameterName":
@@ -638,11 +740,27 @@ function faultStruct(xml: Element): FaultStruct {
     }
   }
 
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultCode",
+    });
+    faultCode = "";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultString",
+    });
+    faultString = "";
+  }
+
   return { faultCode, faultString, setParameterValuesFault };
 }
 
 function fault(xml: Element): CpeFault {
-  let faultCode, faultString, detail;
+  let faultCode: string, faultString: string, detail: FaultStruct;
   for (const c of xml.children) {
     switch (c.localName) {
       case "faultcode":
@@ -657,7 +775,25 @@ function fault(xml: Element): CpeFault {
     }
   }
 
-  return { faultCode, faultString, detail };
+  if (!detail) throw new Error("Missing detail element");
+
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultcode",
+    });
+    faultCode = "Client";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultstring",
+    });
+    faultString = "CWMP fault";
+  }
+
+  return { faultCode, faultString, detail } as CpeFault;
 }
 
 export function request(

lib/types.ts

@@ -163,6 +163,7 @@ export interface SessionContext {
   cacheUntil?: number;
   syncState?: SyncState;
   lastActivity?: number;
+  extendLock?: number;
   rpcRequest?: AcsRequest;
   operationsTouched?: { [commandKey: string]: 1 | 0 };
   provisionsRet?: any[];
@@ -289,8 +290,8 @@ export interface FaultStruct {
 }
 
 export interface CpeFault {
-  faultCode: string;
+  faultCode: "Client" | "Server";
-  faultString: string;
+  faultString: "CWMP fault";
   detail?: FaultStruct;
 }
 

lib/ui.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Z_SYNC_FLUSH } from "zlib";
+import { constants } from "zlib";
 import Koa from "koa";
 import Router from "koa-router";
 import * as jwt from "jsonwebtoken";
@@ -60,7 +60,9 @@ const getAuthorizer = memoize(
 );
 
 koa.on("error", async (err) => {
-  throw err;
+  setTimeout(() => {
+    throw err;
+  });
 });
 
 koa.use(async (ctx, next) => {
@@ -243,7 +245,7 @@ router.get("/", async (ctx) => {
         )};
         window.permissionSets = ${JSON.stringify(permissionSets)};
       </script>
-      <script src="app.js"></script>${wizard} 
+      <script type="module" src="app.js"></script>${wizard} 
     </body>
   </html>
   `;
@@ -252,10 +254,15 @@ router.get("/", async (ctx) => {
 koa.use(
   koaCompress({
     gzip: {
-      flush: Z_SYNC_FLUSH,
+      flush: constants.Z_SYNC_FLUSH,
     },
     deflate: {
-      flush: Z_SYNC_FLUSH,
+      flush: constants.Z_SYNC_FLUSH,
+    },
+    br: {
+      params: {
+        [constants.BROTLI_PARAM_QUALITY]: 5,
+      },
     },
   })
 );

lib/ui/api-functions.ts

@@ -24,7 +24,8 @@ import { getUsers } from "../local-cache";
 import { hashPassword } from "../auth";
 import {
   insertTasks,
-  watchTask,
+  awaitSessionEnd,
+  awaitSessionStart,
   connectionRequest,
   deleteDevice,
 } from "../api-functions";
@@ -104,23 +105,45 @@ export async function postTasks(
   const statuses = tasks.map((t) => {
     return { _id: t._id, status: "pending" };
   });
-
+  const lastInform = Date.now();
-  await del(`${deviceId}_tasks_faults_operations`);
+  const notInSession = await awaitSessionEnd(deviceId, 30000);
-
+  if (!notInSession) {
-  try {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    await connectionRequest(deviceId, device);
-  } catch (err) {
-    for (const t of statuses) db.deleteTask(new ObjectID(t._id));
     return {
-      connectionRequest: err.message,
+      connectionRequest: "Session took too long to complete",
       tasks: statuses,
     };
   }
 
-  const sample = tasks[tasks.length - 1];
+  await del(`${deviceId}_tasks_faults_operations`);
 
-  // Waiting for session to finish or timeout
+  const status = await connectionRequest(deviceId, device);
-  await watchTask(deviceId, sample._id, timeout);
+
+  if (status) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: status,
+      tasks: statuses,
+    };
+  }
+
+  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
+  if (!sessionStarted) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: "No contact from CPE",
+      tasks: statuses,
+    };
+  }
+
+  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
+  if (!sessionEnded) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: "Session took too long to complete",
+      tasks: statuses,
+    };
+  }
 
   const promises = [];
   for (const s of statuses) {
@@ -138,20 +161,11 @@ export async function postTasks(
       r.status = "fault";
       r["fault"] = res[i * 2 + 1][0];
     }
-    db.deleteTask(new ObjectID(r._id));
   }
 
-  return { connectionRequest: "OK", tasks: statuses };
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-}
 
-interface PingResponse {
+  return { connectionRequest: "OK", tasks: statuses };
-  packetsTransmitted: number;
-  packetsReceived: number;
-  packetLoss: string;
-  min: number;
-  avg: number;
-  max: number;
-  mdev: number;
 }
 
 export async function putResource(

lib/ui/api.ts

@@ -31,6 +31,8 @@ import { del } from "../cache";
 import Authorizer from "../common/authorizer";
 import { ping } from "../ping";
 import * as url from "url";
+import { decodeTag } from "../common";
+import { stringify as yamlStringify } from "../common/yaml";
 
 const router = new Router();
 export default router;
@@ -69,6 +71,10 @@ const resources = {
   tasks: 0,
 };
 
+function singleParam(p: string | string[]): string {
+  return Array.isArray(p) ? p[p.length - 1] : p;
+}
+
 router.get(`/devices/:id.csv`, async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
@@ -85,7 +91,7 @@ router.get(`/devices/:id.csv`, async (ctx) => {
 
   if (!authorizer.hasAccess("devices", 2)) {
     logUnauthorizedWarning(log);
-    return void (ctx.status = 404);
+    return void (ctx.status = 403);
   }
 
   const res = await db.query("devices", filter);
@@ -105,19 +111,28 @@ router.get(`/devices/:id.csv`, async (ctx) => {
 
   for (const k of Object.keys(res[0]).sort()) {
     const p = res[0][k];
+    let value = "";
+    let type = "";
+    if (p.value) {
+      value = p.value[0];
+      type = p.value[1];
+      if (type === "xsd:dateTime" && typeof value === "number")
+        value = new Date(value).toJSON();
+    }
+
     const row = [
       k,
       p.object,
-      p.objectTimestamp,
+      new Date(p.objectTimestamp).toJSON(),
       p.writable,
-      p.writableTimestamp,
+      new Date(p.writableTimestamp).toJSON(),
-      p.value != null ? `"${p.value[0].toString().replace(/"/g, '""')}"` : "",
+      `"${value.toString().replace(/"/g, '""')}"`,
-      p.value != null ? p.value[1] : "",
+      type,
-      p.valueTimestamp,
+      new Date(p.valueTimestamp).toJSON(),
       p.notification,
-      p.notificationTimestamp,
+      new Date(p.notificationTimestamp).toJSON(),
       p.accessList ? p.accessList.join(", ") : "",
-      p.accessListTimestamp,
+      new Date(p.accessListTimestamp).toJSON(),
     ];
     ctx.body.write(row.map((r) => (r != null ? r : "")).join(",") + "\n");
   }
@@ -126,11 +141,11 @@ router.get(`/devices/:id.csv`, async (ctx) => {
 });
 
 for (const [resource, flags] of Object.entries(resources)) {
-  router.head(`/${resource}`, async (ctx, next) => {
+  router.head(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     let filter: Expression = authorizer.getFilter(resource, 1);
     if (ctx.request.query.filter)
-      filter = and(filter, parse(ctx.request.query.filter));
+      filter = and(filter, parse(singleParam(ctx.request.query.filter)));
 
     const log = {
       message: `Count ${resource}`,
@@ -141,14 +156,15 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     if (!authorizer.hasAccess(resource, 1)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
+        "OR",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -161,18 +177,18 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.get(`/${resource}`, async (ctx, next) => {
+  router.get(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = {};
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
-      filter = and(filter, parse(ctx.request.query.filter));
+      filter = and(filter, parse(singleParam(ctx.request.query.filter)));
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
-      options.sort = JSON.parse(ctx.request.query.sort);
+      options.sort = JSON.parse(singleParam(ctx.request.query.sort));
     if (ctx.request.query.projection) {
-      options.projection = ctx.request.query.projection
+      options.projection = singleParam(ctx.request.query.projection)
         .split(",")
         .reduce((obj, k) => Object.assign(obj, { [k]: 1 }), {});
     }
@@ -189,14 +205,15 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
+        "OR",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -214,16 +231,16 @@ for (const [resource, flags] of Object.entries(resources)) {
   });
 
   // CSV download
-  router.get(`/${resource}.csv`, async (ctx, next) => {
+  router.get(`/${resource}.csv`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const options: QueryOptions = { projection: {} };
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
-      filter = and(filter, parse(ctx.request.query.filter));
+      filter = and(filter, parse(singleParam(ctx.request.query.filter)));
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
-      options.sort = JSON.parse(ctx.request.query.sort);
+      options.sort = JSON.parse(singleParam(ctx.request.query.sort));
 
     const log = {
       message: `Query ${resource} (CSV)`,
@@ -236,11 +253,11 @@ for (const [resource, flags] of Object.entries(resources)) {
 
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const columns: Record<string, Expression> = JSON.parse(
-      ctx.request.query.columns
+      singleParam(ctx.request.query.columns)
     );
     const now = Date.now();
 
@@ -254,8 +271,9 @@ for (const [resource, flags] of Object.entries(resources)) {
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
       filter = and(filter, [
-        "NOT",
+        "OR",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000],
+        [">=", ["PARAM", "expiry"], Date.now() + 60000],
+        ["IS NULL", ["PARAM", "expiry"]],
       ]);
     }
 
@@ -277,15 +295,27 @@ for (const [resource, flags] of Object.entries(resources)) {
                 if (e[1] === "Tags") {
                   const tags = [];
                   for (const p in obj)
-                    if (p.startsWith("Tags.")) tags.push(p.slice(5));
+                    if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
 
                   return tags.join(", ");
                 }
+                if (e === exp) {
+                  const p = obj[e[1]];
+                  if (
+                    p &&
+                    p.value &&
+                    p.value[1] === "xsd:dateTime" &&
+                    typeof p.value[0] === "number"
+                  )
+                    return new Date(p.value[0]).toJSON();
+                }
+              } else if (resource === "faults") {
+                if (e[1] === "detail") return yamlStringify(obj["detail"]);
               }
             } else if (e[0] === "FUNC") {
               if (e[1] === "DATE_STRING") {
                 if (e[2] && !Array.isArray(e[2]))
-                  return new Date(e[2]).toISOString();
+                  return new Date(e[2]).toJSON();
               }
             }
           }
@@ -304,7 +334,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.head(`/${resource}/:id`, async (ctx, next) => {
+  router.head(`/${resource}/:id`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Count ${resource}`,
@@ -319,18 +349,18 @@ for (const [resource, flags] of Object.entries(resources)) {
     ]);
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void next();
+    if (!res.length) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = "";
   });
 
-  router.get(`/${resource}/:id`, async (ctx, next) => {
+  router.get(`/${resource}/:id`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Query ${resource}`,
@@ -345,19 +375,19 @@ for (const [resource, flags] of Object.entries(resources)) {
     ]);
     if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     const res = await db.query(resource, filter);
 
-    if (!res.length) return void next();
+    if (!res.length) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = res[0];
   });
 
   if (flags & RESOURCE_DELETE) {
-    router.delete(`/${resource}/:id`, async (ctx, next) => {
+    router.delete(`/${resource}/:id`, async (ctx) => {
       const authorizer: Authorizer = ctx.state.authorizer;
       const log = {
         message: `Delete ${resource}`,
@@ -372,10 +402,10 @@ for (const [resource, flags] of Object.entries(resources)) {
       ]);
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
       const res = await db.query(resource, filter);
-      if (!res.length) return void next();
+      if (!res.length) return void (ctx.status = 404);
 
       const validate = authorizer.getValidator(resource, res[0]);
       if (!validate("delete")) {
@@ -392,7 +422,7 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 
   if (flags & RESOURCE_PUT) {
-    router.put(`/${resource}/:id`, async (ctx, next) => {
+    router.put(`/${resource}/:id`, async (ctx) => {
       const authorizer: Authorizer = ctx.state.authorizer;
       const id = ctx.params.id;
 
@@ -404,7 +434,7 @@ for (const [resource, flags] of Object.entries(resources)) {
 
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
 
       const obj = ctx.request.body;
@@ -431,7 +461,37 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 }
 
-router.put("/files/:id", async (ctx, next) => {
+router.get("/blob/files/:id", async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
+  const resource = "files";
+  const id = ctx.params.id;
+
+  const log = {
+    message: `Download ${resource}`,
+    context: ctx,
+    id: id,
+  };
+
+  const filter = and(authorizer.getFilter(resource, 2), [
+    "=",
+    ["PARAM", RESOURCE_IDS[resource]],
+    ctx.params.id,
+  ]);
+
+  if (!authorizer.hasAccess(resource, 2)) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 403);
+  }
+
+  const count = await db.count(resource, filter);
+  if (!count) return void (ctx.status = 404);
+
+  logger.accessInfo(log);
+  ctx.body = db.downloadFile(id);
+  ctx.attachment(id);
+});
+
+router.put("/files/:id", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const resource = "files";
   const id = ctx.params.id;
@@ -445,14 +505,15 @@ router.put("/files/:id", async (ctx, next) => {
 
   if (!authorizer.hasAccess(resource, 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
   const metadata = {
-    fileType: ctx.request.headers["metadata.filetype"] || "",
+    fileType: singleParam(ctx.request.headers["metadata.filetype"]) || "",
-    oui: ctx.request.headers["metadata.oui"] || "",
+    oui: singleParam(ctx.request.headers["metadata.oui"]) || "",
-    productClass: ctx.request.headers["metadata.productclass"] || "",
+    productClass:
-    version: ctx.request.headers["metadata.version"] || "",
+      singleParam(ctx.request.headers["metadata.productclass"]) || "",
+    version: singleParam(ctx.request.headers["metadata.version"]) || "",
   };
 
   const validate = authorizer.getValidator(resource, metadata);
@@ -474,7 +535,7 @@ router.put("/files/:id", async (ctx, next) => {
   ctx.body = "";
 });
 
-router.post("/devices/:id/tasks", async (ctx, next) => {
+router.post("/devices/:id/tasks", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Commit tasks",
@@ -490,10 +551,10 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
   ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
   const devices = await db.query("devices", filter);
-  if (!devices.length) return void next();
+  if (!devices.length) return void (ctx.status = 404);
   const device = devices[0];
 
   const validate = authorizer.getValidator("devices", device);
@@ -527,6 +588,11 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
     }
   ) as number;
 
+  const socketTimeout: number = ctx.socket["timeout"];
+
+  // Disable socket timeout while waiting for postTasks()
+  if (socketTimeout) ctx.socket.setTimeout(0);
+
   const res = await apiFunctions.postTasks(
     ctx.params.id,
     ctx.request.body,
@@ -534,6 +600,9 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
     device
   );
 
+  // Restore socket timeout
+  if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
+
   log.tasks = res.tasks.map((t) => t._id).join(",");
 
   logger.accessInfo(log);
@@ -542,7 +611,7 @@ router.post("/devices/:id/tasks", async (ctx, next) => {
   ctx.body = res.tasks;
 });
 
-router.post("/devices/:id/tags", async (ctx, next) => {
+router.post("/devices/:id/tags", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Update tags",
@@ -558,10 +627,10 @@ router.post("/devices/:id/tags", async (ctx, next) => {
   ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
   const res = await db.query("devices", filter);
-  if (!res.length) return void next();
+  if (!res.length) return void (ctx.status = 404);
 
   const validate = authorizer.getValidator("devices", res[0]);
   if (!validate("tags", ctx.request.body)) {
@@ -584,7 +653,7 @@ router.post("/devices/:id/tags", async (ctx, next) => {
 });
 
 router.get("/ping/:host", async (ctx) => {
-  return new Promise((resolve) => {
+  return new Promise<void>((resolve) => {
     ping(ctx.params.host, (err, parsed) => {
       if (parsed) {
         ctx.body = parsed;
@@ -597,7 +666,7 @@ router.get("/ping/:host", async (ctx) => {
   });
 });
 
-router.put("/users/:id/password", async (ctx, next) => {
+router.put("/users/:id/password", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const username = ctx.params.id;
   const log = {
@@ -622,7 +691,7 @@ router.put("/users/:id/password", async (ctx, next) => {
     }
   } else if (!authorizer.hasAccess("users", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
   const filter = and(authorizer.getFilter("users", 3), [
@@ -631,7 +700,7 @@ router.put("/users/:id/password", async (ctx, next) => {
     username,
   ]);
   const res = await db.query("users", filter);
-  if (!res.length) return void next();
+  if (!res.length) return void (ctx.status = 404);
 
   const newPassword = ctx.request.body.newPassword;
   if (ctx.state.user) {

lib/ui/db.ts

@@ -17,92 +17,42 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, GridFSBucket, ObjectID } from "mongodb";
+import { Db, GridFSBucket, ObjectID } from "mongodb";
 import { Script } from "vm";
-import * as config from "../config";
+import { onConnect } from "../db";
 import * as mongodbFunctions from "../mongodb-functions";
 import * as expression from "../common/expression";
 import { QueryOptions, Expression } from "../types";
 import { Readable } from "stream";
 import { minimize } from "../common/boolean-expression";
 
-const CACHE_TTL = 300000;
-
-let clientPromise: Promise<MongoClient>;
-
 const RESOURCE_COLLECTION = {
   files: "fs.files",
 };
 
-function ensureIndexes(client): void {
+let db: Db;
-  client
-    .db()
-    .collection("cache")
-    .createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
-}
 
-function getClient(): Promise<MongoClient> {
+onConnect(async (_db) => {
-  if (!clientPromise) {
+  db = _db;
-    clientPromise = new Promise((resolve, reject) => {
+});
-      const CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
-      MongoClient.connect(
-        CONNECTION_URL,
-        { useNewUrlParser: true, useUnifiedTopology: true },
-        (err, client) => {
-          if (err) return void reject(err);
-          ensureIndexes(client);
-          resolve(client);
-        }
-      );
-    });
-  }
-
-  return clientPromise;
-}
-
-export function cache<T>(
-  key: string,
-  valueGetter: () => Promise<T>,
-  ttl: number
-): Promise<T> {
-  return new Promise((resolve, reject) => {
-    getClient()
-      .then((client) => {
-        const collection = client.db().collection("cache");
-        collection.findOne({ _id: key }, (err, doc) => {
-          if (err) return void reject(err);
-          if (doc != null) return void resolve(JSON.parse(doc.value) as T);
-          valueGetter()
-            .then((res) => {
-              const expire = Date.now() + (ttl || CACHE_TTL);
-              const cacheDoc = {
-                _id: key,
-                value: JSON.stringify(res),
-                expire: new Date(expire),
-              };
-              collection.updateOne(
-                { _id: key },
-                { $set: cacheDoc },
-                { upsert: true },
-                (err) => {
-                  if (err) reject(err);
-                  else resolve(res);
-                }
-              );
-            })
-            .catch(reject);
-        });
-      })
-      .catch(reject);
-  });
-}
 
+export function query(
+  resource: string,
+  filter: Expression,
+  options?: QueryOptions
+): Promise<any[]>;
+export function query(
+  resource: string,
+  filter: Expression,
+  options: QueryOptions,
+  callback: (doc: any) => void
+): Promise<void>;
 export function query(
   resource: string,
   filter: Expression,
   options?: QueryOptions,
   callback?: (doc: any) => void
-): Promise<any[]> {
+): Promise<void | any[]> {
   options = options || {};
   let q;
   filter = expression.evaluate(filter, null, Date.now());
@@ -123,83 +73,78 @@ export function query(
     }
 
     q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (filter != null && !filter) {
+  } else if (!filter) {
     return Promise.resolve([]);
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then((client) => {
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-      const collection = client
+    const cursor = collection.find(q);
-        .db()
+    if (options.projection) {
-        .collection(RESOURCE_COLLECTION[resource] || resource);
+      let projection = options.projection;
-      const cursor = collection.find(q);
+      if (resource === "devices") {
-      if (options.projection) {
+        projection = mongodbFunctions.processDeviceProjection(
-        let projection = options.projection;
+          options.projection
-        if (resource === "devices") {
-          projection = mongodbFunctions.processDeviceProjection(
-            options.projection
-          );
-        }
-
-        if (resource === "presets") projection.configurations = 1;
-        cursor.project(projection);
-      }
-
-      if (resource === "users") cursor.project({ password: 0, salt: 0 });
-
-      if (options.skip) cursor.skip(options.skip);
-      if (options.limit) cursor.limit(options.limit);
-
-      if (options.sort) {
-        let s = Object.entries(options.sort)
-          .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
-          .reduce(
-            (obj, [k, v]) =>
-              Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
-            {}
-          );
-
-        if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
-        cursor.sort(s);
-      }
-
-      if (!callback) {
-        cursor.toArray((err, docs) => {
-          if (err) return reject(err);
-          if (resource === "devices")
-            docs = docs.map((d) => mongodbFunctions.flattenDevice(d));
-          else if (resource === "faults")
-            docs = docs.map((d) => mongodbFunctions.flattenFault(d));
-          else if (resource === "tasks")
-            docs = docs.map((d) => mongodbFunctions.flattenTask(d));
-          else if (resource === "presets")
-            docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
-          else if (resource === "files")
-            docs = docs.map((d) => mongodbFunctions.flattenFile(d));
-          return resolve(docs);
-        });
-      } else {
-        cursor.forEach(
-          (doc) => {
-            if (resource === "devices")
-              doc = mongodbFunctions.flattenDevice(doc);
-            else if (resource === "faults")
-              doc = mongodbFunctions.flattenFault(doc);
-            else if (resource === "tasks")
-              doc = mongodbFunctions.flattenTask(doc);
-            else if (resource === "presets")
-              doc = mongodbFunctions.flattenPreset(doc);
-            else if (resource === "files")
-              doc = mongodbFunctions.flattenFile(doc);
-            callback(doc);
-          },
-          (err) => {
-            if (err) reject(err);
-            else resolve();
-          }
         );
       }
-    });
+
+      if (resource === "presets") projection.configurations = 1;
+      cursor.project(projection);
+    }
+
+    if (resource === "users") cursor.project({ password: 0, salt: 0 });
+
+    if (options.skip) cursor.skip(options.skip);
+    if (options.limit) cursor.limit(options.limit);
+
+    if (options.sort) {
+      let s = Object.entries(options.sort)
+        .sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]))
+        .reduce(
+          (obj, [k, v]) =>
+            Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
+          {}
+        );
+
+      if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
+      cursor.sort(s);
+    }
+
+    if (!callback) {
+      cursor.toArray((err, docs) => {
+        if (err) return reject(err);
+        if (resource === "devices")
+          docs = docs.map((d) => mongodbFunctions.flattenDevice(d));
+        else if (resource === "faults")
+          docs = docs.map((d) => mongodbFunctions.flattenFault(d));
+        else if (resource === "tasks")
+          docs = docs.map((d) => mongodbFunctions.flattenTask(d));
+        else if (resource === "presets")
+          docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
+        else if (resource === "files")
+          docs = docs.map((d) => mongodbFunctions.flattenFile(d));
+        return resolve(docs);
+      });
+    } else {
+      cursor.forEach(
+        (doc) => {
+          if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc);
+          else if (resource === "faults")
+            doc = mongodbFunctions.flattenFault(doc);
+          else if (resource === "tasks")
+            doc = mongodbFunctions.flattenTask(doc);
+          else if (resource === "presets")
+            doc = mongodbFunctions.flattenPreset(doc);
+          else if (resource === "files")
+            doc = mongodbFunctions.flattenFile(doc);
+          callback(doc);
+        },
+        (err) => {
+          if (err) reject(err);
+          else resolve();
+        }
+      );
+    }
   });
 }
 
@@ -216,71 +161,48 @@ export function count(resource: string, filter: Expression): Promise<number> {
     else if (resource === "faults")
       filter = mongodbFunctions.processFaultsFilter(filter);
     q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (filter != null && !filter) {
+  } else if (!filter) {
     return Promise.resolve(0);
   }
 
   return new Promise((resolve, reject) => {
-    getClient().then((client) => {
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-      const collection = client
+    collection.find(q).count((err, c) => {
-        .db()
+      if (err) reject(err);
-        .collection(RESOURCE_COLLECTION[resource] || resource);
+      else resolve(c);
-      collection.find(q).count((err, c) => {
-        if (err) reject(err);
-        else resolve(c);
-      });
     });
   });
 }
 
-export function updateDeviceTags(
+export async function updateDeviceTags(
   deviceId: string,
   tags: Record<string, boolean>
 ): Promise<void> {
-  return new Promise((resolve, reject) => {
+  const add = [];
-    const add = [];
+  const pull = [];
-    const pull = [];
 
-    const regex = /^[0-9a-zA-Z_]+$/;
+  for (let [tag, onOff] of Object.entries(tags)) {
-    for (let [tag, onOff] of Object.entries(tags)) {
+    tag = tag.trim();
-      tag = tag.trim();
+    if (onOff) add.push(tag);
-      if (onOff) {
+    else pull.push(tag);
-        if (!tag.match(regex))
+  }
-          return void reject(new Error(`Invalid tag '${tag}'`));
-        add.push(tag);
-      } else {
-        pull.push(tag);
-      }
-    }
-    getClient()
-      .then((client) => {
-        const collection = client.db().collection("devices");
 
-        const object = {};
+  const collection = db.collection("devices");
-        if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
+  const object = {};
-        if (pull && pull.length) object["$pullAll"] = { _tags: pull };
+
-        collection.updateOne({ _id: deviceId }, object, (err) => {
+  if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
-          if (err) return void reject(err);
+  if (pull && pull.length) object["$pullAll"] = { _tags: pull };
-          resolve();
+
-        });
+  await collection.updateOne({ _id: deviceId }, object);
-      })
-      .catch(reject);
-  });
 }
 
 function putResource(resource, id, object): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-      .then((client) => {
+    collection.replaceOne({ _id: id }, object, { upsert: true }, (err) => {
-        const collection = client
+      if (err) return void reject(err);
-          .db()
+      resolve();
-          .collection(RESOURCE_COLLECTION[resource] || resource);
+    });
-        collection.replaceOne({ _id: id }, object, { upsert: true }, (err) => {
-          if (err) return void reject(err);
-          resolve();
-        });
-      })
-      .catch(reject);
   });
 }
 
@@ -289,17 +211,11 @@ function deleteResource(
   id: string | ObjectID
 ): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
+    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-      .then((client) => {
+    collection.deleteOne({ _id: id }, (err) => {
-        const collection = client
+      if (err) return void reject(err);
-          .db()
+      resolve();
-          .collection(RESOURCE_COLLECTION[resource] || resource);
+    });
-        collection.deleteOne({ _id: id }, (err) => {
-          if (err) return void reject(err);
-          resolve();
-        });
-      })
-      .catch(reject);
   });
 }
 
@@ -321,9 +237,17 @@ export function putProvision(
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
-    new Script(`"use strict";(function(){\n${object.script}\n})();`);
+    new Script(`"use strict";(function(){\n${object.script}\n})();`, {
-  } catch (error) {
+      filename: id,
-    return Promise.reject(error);
+      lineOffset: -1,
+    });
+  } catch (err) {
+    if (err.stack && err.stack.startsWith(`${id}:`)) {
+      return Promise.reject(
+        new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`)
+      );
+    }
+    return Promise.reject(err);
   }
   return putResource("provisions", id, object);
 }
@@ -338,9 +262,17 @@ export function putVirtualParameter(
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
-    new Script(`"use strict";(function(){\n${object.script}\n})();`);
+    new Script(`"use strict";(function(){\n${object.script}\n})();`, {
-  } catch (error) {
+      filename: id,
-    return Promise.reject(error);
+      lineOffset: -1,
+    });
+  } catch (err) {
+    if (err.stack && err.stack.startsWith(`${id}:`)) {
+      return Promise.reject(
+        new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`)
+      );
+    }
+    return Promise.reject(err);
   }
   return putResource("virtualParameters", id, object);
 }
@@ -376,21 +308,17 @@ export function putUser(
   object: Record<string, unknown>
 ): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
+    const collection = db.collection("users");
-      .then((client) => {
+    // update instead of replace to keep the password if not set by user
-        const collection = client.db().collection("users");
+    collection.updateOne(
-        // update instead of replace to keep the password if not set by user
+      { _id: id },
-        collection.updateOne(
+      { $set: object },
-          { _id: id },
+      { upsert: true },
-          { $set: object },
+      (err) => {
-          { upsert: true },
+        if (err) return void reject(err);
-          (err) => {
+        resolve();
-            if (err) return void reject(err);
+      }
-            resolve();
+    );
-          }
-        );
-      })
-      .catch(reject);
   });
 }
 
@@ -398,38 +326,35 @@ export function deleteUser(id: string): Promise<void> {
   return deleteResource("users", id);
 }
 
+export function downloadFile(filename: string): Readable {
+  const bucket = new GridFSBucket(db);
+  return bucket.openDownloadStreamByName(filename);
+}
+
 export function putFile(
   filename: string,
   metadata: Record<string, string>,
   contentStream: Readable
 ): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
+    const bucket = new GridFSBucket(db);
-      .then((client) => {
+    const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
-        const bucket = new GridFSBucket(client.db());
+      metadata: metadata,
-        const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
+    });
-          metadata: metadata,
+    uploadStream.on("error", reject);
-        });
+    contentStream.on("error", reject);
-        uploadStream.on("error", reject);
+    uploadStream.on("finish", resolve);
-        contentStream.on("error", reject);
+    contentStream.pipe(uploadStream);
-        uploadStream.on("finish", resolve);
-        contentStream.pipe(uploadStream);
-      })
-      .catch(reject);
   });
 }
 
 export function deleteFile(filename: string): Promise<void> {
   return new Promise((resolve, reject) => {
-    getClient()
+    const bucket = new GridFSBucket(db);
-      .then((client) => {
+    bucket.delete(filename as any, (err) => {
-        const bucket = new GridFSBucket(client.db());
+      if (err) return void reject(err);
-        bucket.delete(filename as any, (err) => {
+      resolve();
-          if (err) return void reject(err);
+    });
-          resolve();
-        });
-      })
-      .catch(reject);
   });
 }
 
@@ -440,7 +365,3 @@ export function deleteFault(id: string): Promise<void> {
 export function deleteTask(id: ObjectID): Promise<void> {
   return deleteResource("tasks", id);
 }
-
-export async function disconnect(): Promise<void> {
-  if (clientPromise) await (await clientPromise).close();
-}

lib/xml-parser.ts

@@ -53,12 +53,13 @@ export interface Element {
 }
 
 export function parseXmlDeclaration(buffer: Buffer): Attribute[] {
-  for (const enc of ["utf16le", "utf8", "latin1", "ascii"]) {
+  const encodings: BufferEncoding[] = ["utf16le", "utf8", "latin1", "ascii"];
+  for (const enc of encodings) {
     let str = buffer.toString(enc, 0, 150);
     if (str.startsWith("<?xml")) {
-      str = str.split("\n")[0].trim();
+      str = str.slice(0, str.indexOf("?>"));
       try {
-        return parseAttrs(str.slice(5, -2));
+        return parseAttrs(str.slice(5));
       } catch (err) {
         // Ignore
       }
@@ -103,10 +104,12 @@ export function parseAttrs(string: string): Attribute[] {
         continue;
 
       case CHAR_COLON:
+        if (state) continue;
         if (idx >= colonIdx) colonIdx = i;
         continue;
 
       case CHAR_EQUAL:
+        if (state) continue;
         if (name) throw new Error(`Unexpected character at ${i}`);
         name = string.slice(idx, i).trim();
         // TODO validate name

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.0",
+  "version": "1.2.5",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -14,72 +14,69 @@
   ],
   "author": "Zaid Abdulla <zaid@genieacs.com>",
   "license": "AGPL-3.0",
-  "preferGlobal": true,
+  "private": true,
   "directories": {
-    "bin": "bin",
+    "bin": "bin"
-    "tools": "tools"
   },
   "dependencies": {
+    "espresso-iisojs": "^1.0.3",
     "iconv-lite": "^0.6.2",
     "ipaddr.js": "^2.0.0",
     "jsonwebtoken": "^8.5.1",
-    "koa": "^2.13.0",
+    "koa": "^2.13.1",
     "koa-bodyparser": "^4.3.0",
     "koa-compress": "^5.0.1",
     "koa-jwt": "^4.0.0",
-    "koa-router": "^9.4.0",
+    "koa-router": "^10.0.0",
     "koa-static": "^5.0.0",
     "later": "^1.2.0",
-    "mongodb": "^3.6.0",
+    "mongodb": "^3.6.3",
     "parsimmon": "^1.16.0",
-    "seedrandom": "^3.0.5",
+    "seedrandom": "^3.0.5"
-    "yaml": "^1.10.0",
-    "yebool": "^1.0.2"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^15.0.0",
+    "@rollup/plugin-commonjs": "^17.1.0",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^9.0.0",
+    "@rollup/plugin-node-resolve": "^11.2.0",
-    "@rollup/plugin-typescript": "^5.0.2",
+    "@rollup/plugin-typescript": "^8.2.0",
+    "@types/codemirror": "0.0.108",
     "@types/jsonwebtoken": "^8.5.0",
-    "@types/koa": "^2.11.4",
+    "@types/koa": "^2.13.1",
-    "@types/koa-compress": "^4.0.0",
+    "@types/koa-compress": "^4.0.1",
     "@types/koa-router": "^7.4.1",
     "@types/koa-static": "^4.0.1",
-    "@types/mithril": "^2.0.3",
+    "@types/mithril": "^2.0.6",
-    "@types/mongodb": "^3.5.26",
+    "@types/mongodb": "^3.6.9",
-    "@types/parsimmon": "^1.10.2",
+    "@types/node": "^14.14.34",
-    "@types/seedrandom": "^2.4.28",
+    "@types/parsimmon": "^1.10.6",
-    "@types/svgo": "^1.3.3",
+    "@types/seedrandom": "^3.0.0",
-    "@typescript-eslint/eslint-plugin": "^3.10.1",
+    "@typescript-eslint/eslint-plugin": "^4.17.0",
-    "@typescript-eslint/parser": "^3.10.1",
+    "@typescript-eslint/parser": "^4.17.0",
-    "ava": "^3.12.1",
+    "ava": "^3.15.0",
-    "codemirror": "^5.57.0",
+    "codemirror": "^5.59.4",
     "cssnano": "^4.1.10",
-    "eslint": "^7.8.0",
+    "eslint": "^7.22.0",
-    "eslint-config-prettier": "^6.11.0",
+    "eslint-config-prettier": "^8.1.0",
     "esm": "^3.2.25",
-    "jsbi": "^3.1.3",
     "mithril": "^2.0.4",
-    "postcss": "^7.0.32",
+    "postcss": "^8.2.8",
-    "postcss-import": "^12.0.1",
+    "postcss-import": "^14.0.0",
     "postcss-preset-env": "^6.7.0",
-    "prettier": "^2.1.1",
+    "prettier": "^2.2.1",
-    "rollup": "^2.26.9",
+    "rollup": "^2.41.2",
-    "rollup-plugin-terser": "^7.0.1",
+    "rollup-plugin-terser": "^7.0.2",
-    "source-map-loader": "^1.1.0",
+    "svgo": "^2.2.2",
-    "svgo": "^1.3.2",
+    "ts-node": "^9.1.1",
-    "ts-node": "^9.0.0",
+    "typescript": "^4.2.3",
-    "typescript": "^4.0.2",
+    "yaml": "^1.10.0"
-    "webpack": "^4.44.1"
   },
   "engines": {
     "node": ">=12.13.0"
   },
   "scripts": {
     "test": "ava --verbose --color",
-    "lint-eslint": "eslint --fix \"bin/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/*.ts\"",
+    "lint-eslint": "eslint --fix \"bin/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\"",
-    "lint-prettier": "prettier --parser typescript --write \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
+    "lint-prettier": "prettier --prose-wrap always --write .",
     "lint-tsc": "tsc --noEmit",
     "lint": "npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
     "build": "node -r esm -r ts-node/register/transpile-only build/build"

test/mongodb-functions.ts

@@ -1,13 +1,13 @@
 import ava from "ava";
 import * as mongodbFunctions from "../lib/mongodb-functions";
-import { stringify } from "../lib/common/expression-parser";
+import { stringify, parse } from "../lib/common/expression-parser";
 
-ava("mongoQueryToFilter", (t) => {
+ava("convertOldPrecondition", (t) => {
   const tests = [
     [{}, "true"],
     [{ test: "test" }, 'test = "test"'],
     [{ test: { $eq: "test" } }, 'test = "test"'],
-    [{ test: { $ne: "test" } }, 'test <> "test"'],
+    [{ test: { $ne: "test" } }, 'test <> "test" OR test IS NULL'],
     [{ test: { $gte: "test" } }, 'test >= "test"'],
     [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
     [{ "Tags.test": false }, "Tags.test IS NULL"],
@@ -22,20 +22,20 @@ ava("mongoQueryToFilter", (t) => {
     [{ _tags: { $eq: "test" } }, "Tags.test IS NOT NULL"],
     [
       { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" AND test <> "test"',
+      'test = "test" AND (test <> "test" OR test IS NULL)',
     ],
     [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
     [
       { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" OR test <> "test"',
+      'test = "test" OR (test <> "test" OR test IS NULL)',
     ],
     [
       { test: { $gte: "test1", $ne: "test2" } },
-      'test >= "test1" AND test <> "test2"',
+      'test >= "test1" AND (test <> "test2" OR test IS NULL)',
     ],
     [
       { test: "test", test2: { $ne: "test2" } },
-      'test = "test" AND test2 <> "test2"',
+      'test = "test" AND (test2 <> "test2" OR test2 IS NULL)',
     ],
   ];
 
@@ -50,13 +50,83 @@ ava("mongoQueryToFilter", (t) => {
 
   t.plan(tests.length + 2 * shouldFailTests.length);
   for (const test of tests)
-    t.is(stringify(mongodbFunctions.mongoQueryToFilter(test[0])), test[1]);
+    t.is(stringify(mongodbFunctions.convertOldPrecondition(test[0])), test[1]);
 
   for (const test of shouldFailTests) {
     const func = (): void => {
-      mongodbFunctions.mongoQueryToFilter(test[0] as Record<string, unknown>);
+      mongodbFunctions.convertOldPrecondition(
+        test[0] as Record<string, unknown>
+      );
     };
     const error = t.throws(func, { instanceOf: Error });
     t.is(error.message, test[1]);
   }
 });
+
+ava("filterToMongoQuery", async (t) => {
+  const queries: [string, Record<string, unknown>][] = [
+    ["true", {}],
+    ["Tags.tag1 = true", { _tags: "tag1" }],
+    ["Tags.tag1 <> false", { _tags: "tag1" }],
+    ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }],
+    ["Tags.tag1 = 123", { "Tags.tag1": 123 }],
+    ["Param1 = 'value1'", { "Param1._value": "value1" }],
+    ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }],
+    ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
+    ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
+    ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }],
+    ["LOWER(Param1) LIKE 'value'", { "Param1._value": /^value$/i }],
+    [
+      "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          {
+            $and: [
+              { "Param2._value": { $nin: ["value1", null] } },
+              { "Param1._value": { $gte: "value2" } },
+            ],
+          },
+        ],
+      },
+    ],
+    [
+      "Param1 <> 'value2' OR Param1 IS NULL",
+      {
+        $or: [
+          { "Param1._value": { $nin: ["value2", null] } },
+          { "Param1._value": null },
+        ],
+      },
+    ],
+  ];
+
+  for (const [expStr, expect] of queries) {
+    const exp = parse(expStr);
+    const query = mongodbFunctions.filterToMongoQuery(
+      mongodbFunctions.processDeviceFilter(exp)
+    );
+    t.deepEqual(query, expect);
+  }
+
+  const failQueries: [any, string][] = [
+    ["Param1 = Param2", "Invalid RHS operand of = clause"],
+    ["Param1 LIKE Param2", "Invalid RHS operand of LIKE clause"],
+    ["1 LIKE '1'", "Invalid LHS operand of LIKE clause"],
+    ["1 = 2", "Invalid LHS operand of = clause"],
+    ["1 IS NULL", "Invalid LHS operand of IS NULL clause"],
+    ["false", "Primitives are not valid queries"],
+    ["UPPER(Param1) LIKE 'value'", "Invalid RHS operand of LIKE clause"],
+  ];
+
+  for (const [expStr, err] of failQueries) {
+    const exp = parse(expStr);
+    t.throws(
+      () =>
+        mongodbFunctions.filterToMongoQuery(
+          mongodbFunctions.processDeviceFilter(exp)
+        ),
+      { message: err }
+    );
+  }
+});

test/yaml-tests.json

@@ -0,0 +1,861 @@
+[
+  [
+    {
+      "name": "Mark McGwire",
+      "hr": 65,
+      "avg": 0.278
+    },
+    {
+      "name": "Sammy Sosa",
+      "hr": 63,
+      "avg": 0.288
+    }
+  ],
+  {
+    "top1": {
+      "key1": "scalar1"
+    },
+    "top2": {
+      "key2": "scalar2"
+    },
+    "top3": {
+      "scalar1": "scalar3"
+    },
+    "top4": {
+      "scalar2": "scalar4"
+    },
+    "top5": "scalar5",
+    "top6": {
+      "key6": "scalar6"
+    }
+  },
+  "text",
+  ["a", "b", 42, "d"],
+  {
+    "a!\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~": "safe",
+    "?foo": "safe question mark",
+    ":foo": "safe colon",
+    "-foo": "safe dash",
+    "this is#not": "a comment"
+  },
+  {
+    "": "a"
+  },
+  "foo",
+  {
+    "key": "value",
+    "foo": "key"
+  },
+  [1, -2, 33],
+  {
+    "plain": "a b\nc"
+  },
+  [["s1_i1", "s1_i2"], "s2"],
+  {
+    "First occurrence": "Foo",
+    "Second occurrence": "Foo",
+    "Override anchor": "Bar",
+    "Reuse anchor": "Bar"
+  },
+  "k:#foo &a !t s",
+  ["a"],
+  {
+    "escaped slash": "a/b"
+  },
+  {
+    "plain": "This unquoted scalar spans many lines.",
+    "quoted": "So does this quoted scalar.\n"
+  },
+  "here's to \"quotes\"",
+  {
+    "foo": "bar"
+  },
+  "ab cd\nef\n\ngh\n",
+  ["detected\n", "\n\n# detected\n", " explicit\n", "detected\n"],
+  "foo: bar\": baz",
+  "plain\\value\\with\\backslashes",
+  {
+    "plain": "text lines",
+    "quoted": "text lines",
+    "block": "text\n \tlines\n"
+  },
+  "a",
+  {
+    "foo": "you",
+    "bar": "far"
+  },
+  {
+    "sequence": ["entry", ["nested"]],
+    "mapping": {
+      "foo": "bar"
+    }
+  },
+  {
+    "literal": "some\ntext\n",
+    "folded": "some text\n"
+  },
+  [
+    {
+      "one": "two",
+      "three": "four"
+    },
+    {
+      "five": "six",
+      "seven": "eight"
+    }
+  ],
+  {
+    "Folding": "Empty line\nas a line feed",
+    "Chomping": "Clipped empty lines\n"
+  },
+  [
+    ["one", "two"],
+    ["three", "four"]
+  ],
+  {
+    "foo": "bar"
+  },
+  {
+    "key": "value"
+  },
+  {
+    "explicit key": null,
+    "block key\n": ["one", "two"]
+  },
+  ["foo"],
+  [
+    {
+      "foo": "bar"
+    },
+    ["baz", "baz"]
+  ],
+  "ab\n\n \n",
+  {
+    "foo: bar\\": "baz'"
+  },
+  {
+    "Not indented": {
+      "By one space": "By four\n  spaces\n",
+      "Flow style": ["By two", "Also by two", "Still by two"]
+    }
+  },
+  "\\//||\\/||\n// ||  ||__\n",
+  {
+    "foo": [
+      "a",
+      {
+        "key": "value"
+      }
+    ]
+  },
+  {
+    "a": null,
+    "b": null
+  },
+  "foo",
+  {
+    "a": "b",
+    "": "a"
+  },
+  {
+    "foo\nbar:baz\tx \\$%^&*()x": 23,
+    "x\\ny:z\\tx $%^&*()x": 24
+  },
+  "Sammy Sosa completed another fine season with great stats.\n\n  63 Home Runs\n  0.288 Batting Average\n\nWhat a year!\n",
+  " foo\nbar\nbaz ",
+  { "matches %": 20 },
+  [
+    "flow in block",
+    "Block scalar\n",
+    {
+      "foo": "bar"
+    }
+  ],
+  {
+    "a": "b",
+    "c": 42,
+    "e": "f",
+    "g": "h",
+    "23": false
+  },
+  "ab",
+  " 1st non-empty\n2nd non-empty 3rd non-empty ",
+  {
+    "top1": {
+      "key1": "one"
+    },
+    "top2": {
+      "key2": "two"
+    },
+    "top3": {
+      "key3": "three"
+    },
+    "top4": {
+      "key4": "four"
+    },
+    "top5": {
+      "key5": "five"
+    },
+    "top6": "six",
+    "top7": "seven"
+  },
+  {
+    "hr": ["Mark McGwire", "Sammy Sosa"],
+    "rbi": ["Sammy Sosa", "Ken Griffey"]
+  },
+  "\nfolded line\nnext line\n  * bullet\n\n  * list\n  * lines\n\nlast line\n",
+  ["word1", "word2"],
+  {
+    "a": null,
+    "b": null,
+    "c": null
+  },
+  {
+    "nested sequences": [[[[]]], [[{}]]],
+    "key1": [],
+    "key2": {}
+  },
+  "---word1 word2",
+  {
+    "implicit block key": [
+      {
+        "implicit flow key": "value"
+      }
+    ]
+  },
+  {
+    "key ends with two colons::": "value"
+  },
+  [
+    {
+      "single line": null,
+      "a": "b"
+    },
+    {
+      "multi line": null,
+      "a": "b"
+    }
+  ],
+  "a",
+  {
+    "key": ["item1", "item2"]
+  },
+  [
+    "double quoted",
+    "single quoted",
+    "plain text",
+    ["nested"],
+    {
+      "single": "pair"
+    }
+  ],
+  ["unicode anchor"],
+  [
+    {
+      "key": "value",
+      "key2": "value2"
+    },
+    {
+      "key3": "value3"
+    }
+  ],
+  "trimmed\n\n\nas space",
+  "Mark McGwire's year was crippled by a knee injury.\n",
+  [
+    {
+      "single line": null,
+      "a": "b"
+    },
+    {
+      "multi line": null,
+      "a": "b"
+    }
+  ],
+  {
+    "a": {
+      "b": {
+        "c": "d"
+      },
+      "e": {
+        "f": "g"
+      }
+    },
+    "h": "i"
+  },
+  {
+    "foo": {
+      "bar": "baz"
+    }
+  },
+  [
+    {
+      "single line": "value"
+    },
+    {
+      "multi line": "value"
+    }
+  ],
+  {
+    "single": "text",
+    "double": "text"
+  },
+  " 1st non-empty\n2nd non-empty 3rd non-empty ",
+  [
+    {
+      "item": "Super Hoop",
+      "quantity": 1
+    },
+    {
+      "item": "Basketball",
+      "quantity": 4
+    },
+    {
+      "item": "Big Shoes",
+      "quantity": 1
+    }
+  ],
+  "a b c d\ne",
+  {
+    "a": ["b", ["c", "d"]]
+  },
+  {
+    "strip": "text",
+    "clip": "text\n",
+    "keep": "text\n"
+  },
+  {
+    "a": "b c",
+    "d": "e f"
+  },
+  ["single multiline - sequence entry"],
+  {
+    "1": [2, 3],
+    "4": 5
+  },
+  [
+    {
+      "bla\"keks": "foo"
+    },
+    {
+      "bla]keks": "foo"
+    }
+  ],
+  "folded text\n",
+  "foo",
+  {
+    "key": {
+      "a": "b"
+    }
+  },
+  {
+    "adjacent": "value",
+    "readable": "value",
+    "empty": null
+  },
+  [
+    {
+      "a": "b"
+    },
+    {
+      "c": "d"
+    },
+    {
+      "e": "f"
+    },
+    {
+      "g": "h"
+    }
+  ],
+  {
+    "tab": "\tstring"
+  },
+  [
+    {
+      "foo bar": "baz"
+    }
+  ],
+  {
+    "anchored": "value",
+    "alias": "value"
+  },
+  ["explicit indent and chomp", "chomp and explicit indent"],
+  {
+    "a": ["b", "c"]
+  },
+  {
+    "foo": "bar"
+  },
+  [
+    "::vector",
+    ": - ()",
+    "Up, up, and away!",
+    -123,
+    "http://example.com/foo#bar",
+    [
+      "::vector",
+      ": - ()",
+      "Up, up and away!",
+      -123,
+      "http://example.com/foo#bar"
+    ]
+  ],
+  {
+    "a": "b",
+    "seq": ["a"],
+    "c": "d"
+  },
+  ["foo", "bar", 42],
+  "line1 # no comment line3\n",
+  "\n\nliteral\n \n\ntext\n",
+  {
+    "a": "b"
+  },
+  {
+    "k": ["a", "b"]
+  },
+  "a b c d\ne",
+  "---word1 word2",
+  ["a", 2, 4, "d"],
+  {
+    "a": [
+      "b",
+      "c",
+      {
+        "d": ["e", "f"]
+      }
+    ]
+  },
+  {
+    "a": " more indented\nregular\n",
+    "b": "\n\n more indented\nregular\n"
+  },
+  {
+    "strip": "# text",
+    "clip": "# text\n",
+    "keep": "# text\n\n"
+  },
+  {
+    "safe": "a!\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~ !\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~",
+    "safe question mark": "?foo",
+    "safe colon": ":foo",
+    "safe dash": "-foo"
+  },
+  "line1 line2 line3\n",
+  ["Mark McGwire", "Sammy Sosa", "Ken Griffey"],
+  ["a"],
+  ["a", ["b", "c"]],
+  {
+    "unicode": "Sosa did fine.☺",
+    "control": "\b1998\t1999\t2000\n",
+    "hex esc": "\r\n is \r\n",
+    "single": "\"Howdy!\" he cried.",
+    "quoted": " # Not a 'comment'.",
+    "tie-fighter": "|\\-*-/|"
+  },
+  [["-", "-"]],
+  "folded text\n",
+  {
+    "a": 13,
+    "1.5": "d"
+  },
+  {
+    "foo": 1,
+    "bar": 2,
+    "text": "a\n  \nb\n\nc\n\nd\n"
+  },
+  {
+    "wanted": "love ♥ and peace ☮"
+  },
+  {
+    "name": "Mark McGwire",
+    "accomplishment": "Mark set a major league home run record in 1998.\n",
+    "stats": "65 Home Runs\n0.278 Batting Average\n"
+  },
+  {
+    "foo": "bar",
+    "baz": "foo"
+  },
+  "1st non-empty\n2nd non-empty 3rd non-empty",
+  {
+    "quoted": "Quoted \t",
+    "block": "void main() {\n\tprintf(\"Hello, world!\\n\");\n}\n"
+  },
+  {
+    "foo": "blue",
+    "bar": "arrr",
+    "baz": "jazz"
+  },
+  [
+    {
+      "Mark McGwire": 65
+    },
+    {
+      "Sammy Sosa": 63
+    },
+    {
+      "Ken Griffy": 58
+    }
+  ],
+  {
+    "1": 2,
+    "3": 4
+  },
+  {
+    "hr": ["Mark McGwire", "Sammy Sosa"],
+    "rbi": ["Sammy Sosa", "Ken Griffey"]
+  },
+  "k:#foo &a !t s",
+  {
+    "block sequence": [
+      "one",
+      {
+        "two": "three"
+      }
+    ]
+  },
+  {
+    "First occurrence": "Value",
+    "Second occurrence": "Value"
+  },
+  {
+    "a true": "null d",
+    "e 42": null
+  },
+  {
+    "foo": "bar"
+  },
+  ["foo", "bar", 42],
+  "trimmed\n\n\nas space",
+  "scalar",
+  {
+    "strip": "",
+    "clip": "",
+    "keep": "\n"
+  },
+  {
+    "foo": {
+      "bar": 1
+    },
+    "baz": 2
+  },
+  {
+    "a": 47,
+    "c": "d"
+  },
+  {
+    "implicit block key": [
+      {
+        "implicit flow key": "value"
+      }
+    ]
+  },
+  ["a", "b", "c", "c", ""],
+  [
+    ["a", "b", "c"],
+    {
+      "a": "b",
+      "c": "d",
+      "e": "f"
+    },
+    []
+  ],
+  {
+    "implicit block key": [
+      {
+        "implicit flow key": "value"
+      }
+    ]
+  },
+  {
+    "a": "ab\n\ncd\nef\n"
+  },
+  {
+    "literal": "value\n",
+    "folded": "value\n"
+  },
+  {
+    "a": [
+      "b",
+      "c",
+      {
+        "d": ["e", "f"]
+      }
+    ]
+  },
+  "literal\n\ttext\n",
+  "foo \n\n\t bar\n\nbaz\n",
+  [
+    {
+      "a": "b"
+    }
+  ],
+  "ab",
+  ["plain", "double quoted", "single quoted", "block\n", "plain again"],
+  {
+    "a": " ",
+    "b": " ",
+    "c": " ",
+    "d": " ",
+    "e": "\n",
+    "f": "\n",
+    "g": "\n\n",
+    "h": "\n\n"
+  },
+  {
+    "key": "value with\ntabs"
+  },
+  {
+    "": null
+  },
+  [
+    {
+      "single line": "value"
+    },
+    {
+      "multi line": "value"
+    }
+  ],
+  "folded to a space,\nto a line feed, or \t \tnon-content",
+  ["literal\n", " folded\n", "keep\n\n", " strip"],
+  {
+    "key": "value"
+  },
+  {
+    "american": ["Boston Red Sox", "Detroit Tigers", "New York Yankees"],
+    "national": ["New York Mets", "Chicago Cubs", "Atlanta Braves"]
+  },
+  " 1st non-empty\n2nd non-empty 3rd non-empty ",
+  {},
+  [
+    ["a", "b"],
+    {
+      "a": "b"
+    },
+    "a",
+    "b",
+    "c"
+  ],
+  "folded to a space,\nto a line feed, or \t \tnon-content",
+  [
+    {
+      "foo": "bar"
+    }
+  ],
+  ["detected\n", "\n\n# detected\n", " explicit\n", "\t\ndetected\n"],
+  {
+    "top1": [
+      "item1",
+      {
+        "key2": "value2"
+      },
+      "item3"
+    ],
+    "top2": "value2"
+  },
+  {
+    "foo": [42],
+    "bar": [44]
+  },
+  {
+    "23": "d",
+    "a": 4.2
+  },
+  "Document",
+  {
+    "Date": "2001-11-23 15:03:17 -5",
+    "User": "ed",
+    "Fatal": "Unknown variable \"bar\"",
+    "Stack": [
+      {
+        "file": "TopClass.py",
+        "line": 23,
+        "code": "x = MoreObject(\"345\\n\")\n"
+      },
+      { "file": "MoreClass.py", "line": 58, "code": "foo = bar" }
+    ]
+  },
+  {
+    "plain key": "in-line value",
+    "": null,
+    "quoted key": ["entry"]
+  },
+  ["12", 12, "12"],
+  {
+    "aaa": "bbb"
+  },
+  [":,"],
+  {
+    "sequence": ["one", "two"],
+    "mapping": {
+      "sky": "blue",
+      "sea": "green"
+    }
+  },
+  {
+    "seq": ["a", "b"]
+  },
+  "here's to \"quotes\"",
+  {
+    "hr": 65,
+    "avg": 0.278,
+    "rbi": 147
+  },
+  "\n\nliteral\n \n\ntext\n",
+  " 1st non-empty\n2nd non-empty 3rd non-empty ",
+  "literal\n\ttext\n",
+  {
+    "block mapping": {
+      "key": "value"
+    }
+  },
+  " foo\nbar\nbaz ",
+  "ab cd\nef\n\ngh\n",
+  "foo",
+  {
+    "top1": {
+      "key1": "one"
+    },
+    "top2": {
+      "key2": "two"
+    },
+    "top3": {
+      "key3": "three"
+    },
+    "top4": {
+      "key4": "four"
+    },
+    "top5": {
+      "key5": "five"
+    },
+    "top6": "six",
+    "top7": "seven"
+  },
+  [
+    {
+      "url": "http://example.org"
+    }
+  ],
+  {
+    "sequence": ["one", "two"],
+    "mapping": {
+      "sky": "blue",
+      "sea": "green"
+    }
+  },
+  {
+    "invoice": 34843,
+    "date": "2001-01-23",
+    "bill-to": {
+      "given": "Chris",
+      "family": "Dumars",
+      "address": {
+        "lines": "458 Walkman Dr.\nSuite #292\n",
+        "city": "Royal Oak",
+        "state": "MI",
+        "postal": 48046
+      }
+    },
+    "ship-to": {
+      "given": "Chris",
+      "family": "Dumars",
+      "address": {
+        "lines": "458 Walkman Dr.\nSuite #292\n",
+        "city": "Royal Oak",
+        "state": "MI",
+        "postal": 48046
+      }
+    },
+    "product": [
+      {
+        "sku": "BL394D",
+        "quantity": 4,
+        "description": "Basketball",
+        "price": 450
+      },
+      {
+        "sku": "BL4438H",
+        "quantity": 1,
+        "description": "Super Hoop",
+        "price": 2392
+      }
+    ],
+    "tax": 251.42,
+    "total": 4443.52,
+    "comments": "Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338."
+  },
+  ["a", "b", "a", "b"],
+  [
+    null,
+    "block node\n",
+    ["one", "two"],
+    {
+      "one": "two"
+    }
+  ],
+  {
+    "a": "scalar a",
+    "b": "scalar a"
+  },
+  {
+    "foo": "",
+    "": "bar"
+  },
+  {
+    "key": "value"
+  },
+  "scalar %YAML 1.2",
+  {
+    "Folding": "Empty line\nas a line feed",
+    "Chomping": "Clipped empty lines\n"
+  },
+  {
+    "key": "value"
+  },
+  [
+    ["name", "hr", "avg"],
+    ["Mark McGwire", 65, 0.278],
+    ["Sammy Sosa", 63, 0.288]
+  ],
+  {
+    "literal": "value\n",
+    "folded": "value\n"
+  },
+  {
+    "Mark McGwire": {
+      "hr": 65,
+      "avg": 0.278
+    },
+    "Sammy Sosa": {
+      "hr": 63,
+      "avg": 0.288
+    }
+  },
+  {
+    "a": "b",
+    "c": "d"
+  },
+  {
+    "key": [[["value"]]]
+  },
+  {
+    "a": 1,
+    "b": null,
+    "c": 3
+  },
+  {
+    "event": "outgoing HTTP response",
+    "timestamp": "2021-01-21T18:44:57.116Z",
+    "remoteAddress": "127.0.0.1",
+    "deviceId": "202BC1-BM632w-000000",
+    "connection": "2021-01-21T18:44:57.105Z",
+    "statusCode": 200,
+    "headers": {
+      "content-length": 528,
+      "server": "GenieACS/1.2.3+20210121000910",
+      "soapserver": "GenieACS/1.2.3+20210121000910",
+      "content-type": "text/xml; charset=\"utf-8\"",
+      "set-cookie": "session=ea31d97ccf6832fb"
+    },
+    "body": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<soap-env:Envelope xmlns:soap-enc=\"http://schemas.xmlsoap.org/soap/encoding/\" xmlns:soap-env=\"http://schemas.xmlsoap.org/soap/envelope/\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:cwmp=\"urn:dslforum-org:cwmp-1-0\"><soap-env:Header><cwmp:ID soap-env:mustUnderstand=\"1\">wdsx50vq</cwmp:ID></soap-env:Header><soap-env:Body><cwmp:InformResponse><MaxEnvelopes>1</MaxEnvelopes></cwmp:InformResponse></soap-env:Body></soap-env:Envelope>"
+  }
+]

test/yaml.ts

@@ -0,0 +1,14 @@
+import ava from "ava";
+import * as yaml from "yaml";
+import { stringify } from "../lib/common/yaml";
+
+import testCases from "./yaml-tests.json";
+
+ava("stringify", (t) => {
+  for (const testCase of testCases) {
+    let str = stringify(testCase);
+    if (str.startsWith(">2")) str = ">3" + str.slice(2);
+    if (str.startsWith("|2")) str = "|3" + str.slice(2);
+    t.deepEqual(yaml.parse(yaml.stringify(testCase)), yaml.parse(str));
+  }
+});

tools/dump-data-model.ts

@@ -1,82 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as fs from "fs";
-import * as path from "path";
-import { connect, devicesCollection } from "../lib/db";
-
-const outputFilename = path.resolve(`data_model_${process.argv[2]}.json`);
-process.chdir(path.resolve(path.dirname(fs.realpathSync(__filename)), ".."));
-
-const dataModel = {};
-
-function recursive(object, p = ""): void {
-  for (const key in object) {
-    if (key[0] === "_" || !object[key]) continue;
-
-    const isObject = object[key]["_object"];
-    const isValue = "_value" in object[key];
-
-    if ((isObject && isValue) || (!isObject && !isValue))
-      throw new Error("Unexpected object");
-
-    if (isValue) {
-      let value = object[key]["_value"];
-      if (typeof value === "boolean") value = +value;
-
-      const values = [object[key]["_writable"] || false, value.toString()];
-      if (object[key]["_type"]) values.push(object[key]["_type"]);
-
-      dataModel[`${p}${key}`] = values;
-    }
-
-    if (isObject) {
-      const values = [object[key]["_writable"] || false];
-      dataModel[`${p}${key}.`] = values;
-      recursive(object[key], `${p}${key}.`);
-    }
-  }
-}
-
-connect()
-  .then(() => {
-    devicesCollection.findOne(
-      {
-        _id: process.argv[2],
-      },
-      {},
-      (err, device) => {
-        if (err) throw err;
-
-        if (!device) throw new Error("Device not found");
-
-        recursive(device);
-
-        fs.writeFile(outputFilename, JSON.stringify(dataModel, null, 2), () => {
-          if (err) throw err;
-
-          process.stdout.write(`${outputFilename} saved\n`);
-          process.exit(0);
-        });
-      }
-    );
-  })
-  .catch((err) => {
-    throw err;
-  });

tsconfig.json

@@ -1,8 +1,6 @@
 {
   "compilerOptions": {
-    "lib": [
+    "lib": ["esnext", "dom"],
-      "esnext", "dom"
-    ],
     "module": "esnext",
     "moduleResolution": "node",
     "target": "esnext",
@@ -14,11 +12,10 @@
     "allowJs": true
   },
   "include": [
-      "./bin/*.ts",
+    "./bin/*.ts",
-      "./lib/**/*",
+    "./lib/**/*",
-      "./ui/**/*",
+    "./ui/**/*",
-      "./test/**/*",
+    "./test/**/*",
-      "./build/**/*",
+    "./build/**/*"
-      "./tools/*.ts"
   ]
 }

ui/app.ts

@@ -64,50 +64,37 @@ const adminPages = [
 
 let state;
 
-let lastRenderTimestamp = 0;
-let pageVisitTimestamp = 0;
-let fulfillTimeout = null;
-
-function fulfill(): void {
-  clearTimeout(fulfillTimeout);
-  fulfillTimeout = setTimeout(() => {
-    store.fulfill(lastRenderTimestamp, pageVisitTimestamp).then((updated) => {
-      if (updated) m.redraw();
-    });
-  }, 100);
-}
-
 function pagify(pageName, page): RouteResolver {
   const component: RouteResolver = {
     render: () => {
-      lastRenderTimestamp = Date.now();
+      const lastRenderTimestamp = Date.now();
       let p;
       if (state && state.error) p = m(errorPage.component, state);
       else p = m(contextifyComponent(page.component), state);
-      fulfill();
       const attrs = {};
       attrs["page"] = pageName;
+      attrs["oncreate"] = attrs["onupdate"] = () => {
+        store.fulfill(lastRenderTimestamp);
+      };
       return m(layout, attrs, p);
     },
     onmatch: null,
   };
 
   component.onmatch = (args, requestedPath) => {
-    pageVisitTimestamp = Date.now();
+    store.setTimestamp(Date.now());
     if (!page.init) {
       state = null;
-      fulfill();
       return null;
     }
 
-    return new Promise((resolve) => {
+    return new Promise<void>((resolve) => {
       page
         .init(args)
         .then((st) => {
           if (!st) return void m.route.set("/");
           state = st;
           resolve();
-          fulfill();
         })
         .catch((err) => {
           if (!window.username && err.message.indexOf("authorized") >= 0) {

ui/code-editor-component.ts

@@ -28,21 +28,24 @@ const component: ClosureComponent = (): Component => {
         name: vnode.attrs["id"],
         value: vnode.attrs["value"],
         oncreate: (_vnode) => {
-          const editor = codeMirror.fromTextArea(_vnode.dom, {
+          const editor = codeMirror.fromTextArea(
-            mode: vnode.attrs["mode"],
+            _vnode.dom as HTMLTextAreaElement,
-            lineNumbers: true,
+            {
-            readOnly: vnode.attrs["readOnly"],
+              mode: vnode.attrs["mode"],
-            extraKeys: {
+              lineNumbers: true,
-              "Ctrl-Enter": () => {
+              readOnly: vnode.attrs["readOnly"],
-                if (vnode.attrs["onSubmit"])
+              extraKeys: {
-                  vnode.attrs["onSubmit"](_vnode.dom);
+                "Ctrl-Enter": () => {
+                  if (vnode.attrs["onSubmit"])
+                    vnode.attrs["onSubmit"](_vnode.dom);
+                },
+                "Cmd-Enter": () => {
+                  if (vnode.attrs["onSubmit"])
+                    vnode.attrs["onSubmit"](_vnode.dom);
+                },
               },
-              "Cmd-Enter": () => {
+            }
-                if (vnode.attrs["onSubmit"])
+          );
-                  vnode.attrs["onSubmit"](_vnode.dom);
-              },
-            },
-          });
 
           if (vnode.attrs["onChange"]) {
             editor.on("change", (e) => {

ui/components.ts

@@ -39,6 +39,7 @@ import tags from "./components/tags";
 import ping from "./components/ping";
 import deviceLink from "./components/device-link";
 import longTextComponent from "./long-text-component";
+import loading from "./components/loading";
 
 const comps = {
   parameter,
@@ -54,6 +55,7 @@ const comps = {
   ping,
   "device-link": deviceLink,
   "long-text": longTextComponent,
+  loading: loading,
 };
 
 const contextifiedComponents = new WeakMap<ComponentTypes, ComponentTypes>();
@@ -137,9 +139,13 @@ export function contextifyComponent(component: ComponentTypes): ComponentTypes {
         const view = state.view;
         state.view = function (vnode) {
           const context = vnodeContext.get(vnode) || {};
-          const res = Reflect.apply(view, this, [vnode]);
+          try {
-          applyContext(res, context);
+            const res = Reflect.apply(view, this, [vnode]);
-          return res;
+            applyContext(res, context);
+            return res;
+          } catch (err) {
+            return m("p.error", { title: err.message }, "Error!");
+          }
         };
         return state;
       };

ui/components/all-parameters.ts

@@ -55,6 +55,8 @@ const component: ClosureComponent = (): Component => {
     view: (vnode) => {
       const device = vnode.attrs["device"];
 
+      const limit = vnode.attrs["limit"] > 0 ? vnode.attrs["limit"] : 100;
+
       const search = m("input", {
         type: "text",
         placeholder: "Search parameters",
@@ -86,7 +88,7 @@ const component: ClosureComponent = (): Component => {
           const str = p.value && p.value[0] ? `${k} ${p.value[0]}` : k;
           if (re && !re.test(str)) continue;
           ++c;
-          if (count < 100) filteredKeys.push(k);
+          if (count < limit) filteredKeys.push(k);
         }
         count += c;
       }
@@ -169,25 +171,29 @@ const component: ClosureComponent = (): Component => {
       });
 
       return m(
-        ".all-parameters",
+        "loading",
+        { queries: [vnode.attrs["deviceQuery"]] },
         m(
-          "a.download-csv",
+          ".all-parameters",
-          {
-            href: `api/devices/${encodeURIComponent(
-              device["DeviceID.ID"].value[0]
-            )}.csv`,
-            download: "",
-            style: "float: right;",
-          },
-          "Download"
-        ),
-        search,
-        m(
-          ".parameter-list",
-          m("table", m("tbody", rows)),
           m(
-            "m",
+            "a.download-csv",
-            `Displaying ${filteredKeys.length} out of ${count} parameters.`
+            {
+              href: `api/devices/${encodeURIComponent(
+                device["DeviceID.ID"].value[0]
+              )}.csv`,
+              download: "",
+              style: "float: right;",
+            },
+            "Download"
+          ),
+          search,
+          m(
+            ".parameter-list",
+            m("table", m("tbody", rows)),
+            m(
+              "m",
+              `Displaying ${filteredKeys.length} out of ${count} parameters.`
+            )
           )
         )
       );

ui/components/device-actions.ts

@@ -70,7 +70,7 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageDownload({
                 name: "download",
-                device: device["DeviceID.ID"].value[0],
+                devices: [device["DeviceID.ID"].value[0]],
               });
             },
           },

ui/components/device-faults.ts

@@ -22,6 +22,7 @@ import { m } from "../components";
 import * as store from "../store";
 import * as notifications from "../notifications";
 import { getIcon } from "../icons";
+import { stringify } from "../../lib/common/yaml";
 
 const component: ClosureComponent = (): Component => {
   return {
@@ -38,6 +39,7 @@ const component: ClosureComponent = (): Component => {
         "Channel",
         "Code",
         "Message",
+        "Detail",
         "Retries",
         "Timestamp",
       ].map((l) => m("th", l));
@@ -48,7 +50,9 @@ const component: ClosureComponent = (): Component => {
         rows.push([
           m("td", f["channel"]),
           m("td", f["code"]),
-          m("td", f["message"]),
+          m("td", m("long-text", { text: f["message"] })),
+          m("td", m("long-text", { text: stringify(f["detail"]) })),
+
           m("td", f["retries"]),
           m("td", new Date(f["timestamp"]).toLocaleString()),
           m(
@@ -63,12 +67,12 @@ const component: ClosureComponent = (): Component => {
                     .deleteResource("faults", f["_id"])
                     .then(() => {
                       notifications.push("success", "Fault deleted");
-                      store.fulfill(Date.now(), Date.now());
+                      store.setTimestamp(Date.now());
                       m.redraw();
                     })
                     .catch((err) => {
                       notifications.push("error", err.message);
-                      store.fulfill(Date.now(), Date.now());
+                      store.setTimestamp(Date.now());
                     });
                 },
               },
@@ -91,7 +95,11 @@ const component: ClosureComponent = (): Component => {
         );
       }
 
-      return m("table.table", thead, tbody);
+      return m(
+        "loading",
+        { queries: [faults] },
+        m("table.table", thead, tbody)
+      );
     },
   };
 };

ui/components/loading.ts

@@ -0,0 +1,80 @@
+/**
+ * Copyright 2013-2021  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { VnodeDOM, ClosureComponent, Component } from "mithril";
+
+const component: ClosureComponent = (): Component => {
+  let overlay: HTMLElement;
+  let dom: Element;
+  let loading = false;
+
+  function apply(vnode: VnodeDOM): void {
+    if (!loading) {
+      if (overlay) overlay.parentElement.remove();
+      if (dom) dom.classList.remove("loading");
+      overlay = null;
+      dom = null;
+      return;
+    }
+
+    if (dom && dom !== vnode.dom) dom.classList.remove("loading");
+
+    dom = vnode.dom;
+    dom.classList.add("loading");
+
+    if (!overlay) {
+      const wrapper = document.createElement("div");
+      wrapper.style.position = "relative";
+      wrapper.style.pointerEvents = "none";
+      overlay = document.createElement("div");
+      overlay.classList.add("loading-overlay");
+      overlay.style.position = "absolute";
+      wrapper.appendChild(overlay);
+    }
+
+    const wrapper = overlay.parentElement;
+    if (wrapper.parentElement !== dom.parentElement)
+      dom.parentNode.appendChild(wrapper);
+
+    const wrapperRect = wrapper.getBoundingClientRect();
+    const domRect = dom.getBoundingClientRect();
+    overlay.style.width = `${dom.scrollWidth}px`;
+    overlay.style.height = `${dom.scrollHeight}px`;
+    overlay.style.left = `${domRect.left - wrapperRect.left}px`;
+    overlay.style.top = `${domRect.top - wrapperRect.top}px`;
+  }
+
+  return {
+    view: (vnode) => {
+      const queries = vnode.attrs["queries"];
+      loading = queries.some((q) => q.fulfilling);
+      return vnode.children;
+    },
+    oncreate: apply,
+    onupdate: apply,
+    onremove: () => {
+      if (overlay) overlay.parentElement.remove();
+      if (dom) dom.classList.remove("loading");
+      overlay = null;
+      dom = null;
+    },
+  };
+};
+
+export default component;

ui/components/parameter-list.ts

@@ -38,6 +38,11 @@ const component: ClosureComponent = (): Component => {
         return m(
           "tr",
           {
+            oncreate: (vn) => {
+              (vn.dom as HTMLElement).style.display = (p as VnodeDOM).dom
+                ? ""
+                : "none";
+            },
             onupdate: (vn) => {
               (vn.dom as HTMLElement).style.display = (p as VnodeDOM).dom
                 ? ""
@@ -49,7 +54,11 @@ const component: ClosureComponent = (): Component => {
         );
       });
 
-      return m("table.parameter-list", rows);
+      return m(
+        "loading",
+        { queries: [vnode.attrs["deviceQuery"]] },
+        m("table.parameter-list", rows)
+      );
     },
   };
 };

ui/components/parameter-table.ts

@@ -149,7 +149,14 @@ const component: ClosureComponent = (): Component => {
       let label;
       if (vnode.attrs["label"]) label = m("h2", vnode.attrs["label"]);
 
-      return [label, m("table.table", thead, m("tbody", rows))];
+      return [
+        label,
+        m(
+          "loading",
+          { queries: [vnode.attrs["deviceQuery"]] },
+          m("table.table", thead, m("tbody", rows))
+        ),
+      ];
     },
   };
 };

ui/components/parameter.ts

@@ -62,6 +62,8 @@ const evaluateParam = memoize((exp, obj, now: number) => {
       timestamp = p.valueTimestamp;
       value = p.value[0];
       parameter = exp[1];
+      if (p.value[1] === "xsd:dateTime" && typeof value === "number")
+        value = new Date(value).toLocaleString();
     }
   }
 
@@ -90,7 +92,7 @@ const component: ClosureComponent = (): Component => {
             onclick: () => {
               taskQueue.stageSpv({
                 name: "setParameterValues",
-                device: device["DeviceID.ID"].value[0],
+                devices: [device["DeviceID.ID"].value[0]],
                 parameterValues: [
                   [
                     parameter,

ui/components/ping.ts

@@ -27,23 +27,28 @@ const component: ClosureComponent = (vn): Component => {
   let interval: ReturnType<typeof setInterval>;
   let host: string;
 
-  const refresh = async (): Promise<void> => {
+  const refresh = (): void => {
-    let status = "";
+    if (!host) {
-    if (host) {
+      const dom = (vn as VnodeDOM).dom;
-      try {
+      if (dom) dom.innerHTML = "";
-        const res = await store.ping(host);
+      return;
-        if (res["avg"] != null) status = `${Math.trunc(res["avg"])} ms`;
-        else status = "Unreachable";
-      } catch (err) {
-        setTimeout(() => {
-          throw err;
-        }, 0);
-      }
     }
 
-    const dom = (vn as VnodeDOM).dom;
+    let status = "";
-
+    store
-    if (dom) dom.innerHTML = `Pinging ${host}: ${status}`;
+      .ping(host)
+      .then((res) => {
+        if (res["avg"] != null) status = `${Math.trunc(res["avg"])} ms`;
+        else status = "Unreachable";
+      })
+      .catch(() => {
+        status = "Error!";
+        clearInterval(interval);
+      })
+      .finally(() => {
+        const dom = (vn as VnodeDOM).dom;
+        if (dom) dom.innerHTML = `Pinging ${host}: ${status}`;
+      });
   };
 
   return {
@@ -58,9 +63,11 @@ const component: ClosureComponent = (vn): Component => {
         param = device["Device.ManagementServer.ConnectionRequestURL"];
 
       let h;
-      if (param && param.value) {
+      try {
         const url = new URL(param.value[0]);
         h = url.hostname;
+      } catch (err) {
+        // Ignore
       }
 
       if (host !== h) {
@@ -72,7 +79,7 @@ const component: ClosureComponent = (vn): Component => {
         }
       }
 
-      return m("div", `Pinging ${host}:`);
+      return m("div", host ? `Pinging ${host}:` : "");
     },
   };
 };

ui/components/summon-button.ts

@@ -78,7 +78,11 @@ const component: ClosureComponent = (): Component => {
               )
               .then(() => {
                 e.target.disabled = false;
-                store.fulfill(0, Date.now());
+                store.setTimestamp(Date.now());
+              })
+              .catch((err) => {
+                e.target.disabled = false;
+                notifications.push("error", err.message);
               });
           },
         },

ui/components/tags.ts

@@ -22,6 +22,7 @@ import { m } from "../components";
 import * as notifications from "../notifications";
 import * as store from "../store";
 import { getIcon } from "../icons";
+import { decodeTag } from "../../lib/common";
 
 const component: ClosureComponent = (): Component => {
   return {
@@ -32,7 +33,7 @@ const component: ClosureComponent = (): Component => {
 
       const tags = [];
       for (const p of Object.keys(device))
-        if (p.startsWith("Tags.")) tags.push(p.slice(5));
+        if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
 
       tags.sort();
 
@@ -63,7 +64,7 @@ const component: ClosureComponent = (): Component => {
                         "success",
                         `${deviceId}: Tags updated`
                       );
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     })
                     .catch((err) => {
                       e.target.disabled = false;
@@ -98,7 +99,7 @@ const component: ClosureComponent = (): Component => {
                   .then(() => {
                     e.target.disabled = false;
                     notifications.push("success", `${deviceId}: Tags updated`);
-                    store.fulfill(0, Date.now());
+                    store.setTimestamp(Date.now());
                   })
                   .catch((err) => {
                     e.target.disabled = false;

ui/config-functions.ts

@@ -42,6 +42,35 @@ export function flattenConfig(config: Record<string, unknown>): any {
   return flatten;
 }
 
+// Order keys such that nested objects come last
+function orderKeys(config: any): number {
+  let res = 1;
+  if (typeof config !== "object") return res;
+  if (Array.isArray(config)) {
+    for (const c of config) res += orderKeys(c);
+    return res;
+  }
+
+  const weights: [string, number][] = Object.entries(config).map(([k, v]) => [
+    k,
+    orderKeys(v),
+  ]);
+
+  weights.sort((a, b) => {
+    if (a[1] !== b[1]) return a[1] - b[1];
+    if (b[0] > a[0]) return -1;
+    else return 1;
+  });
+
+  for (const [k, w] of weights) {
+    res += w;
+    const v = config[k];
+    delete config[k];
+    config[k] = v;
+  }
+  return res;
+}
+
 export function structureConfig(config: Config[]): any {
   config.sort((a, b) => (a._id > b._id ? 1 : a._id < b._id ? -1 : 0));
   const _config = {};
@@ -93,7 +122,9 @@ export function structureConfig(config: Config[]): any {
     return object;
   };
 
-  return toArray(_config);
+  const res = toArray(_config);
+  orderKeys(res);
+  return res;
 }
 
 export function diffConfig(

ui/config-page.ts

@@ -60,12 +60,12 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
         .resourceExists("config", id)
         .then((exists) => {
           if (exists && isNew) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ _id: "Config already exists" });
           }
 
           if (!exists && !isNew) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ _id: "Config does not exist" });
           }
 
@@ -76,8 +76,8 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
                 "success",
                 `Config ${exists ? "updated" : "created"}`
               );
-              store.fulfill(0, Date.now());
+              store.setTimestamp(Date.now());
-              resolve();
+              resolve(null);
             })
             .catch(reject);
         })
@@ -87,11 +87,11 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
         .deleteResource("config", object["_id"])
         .then(() => {
           notifications.push("success", "Config deleted");
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
-          resolve();
+          resolve(null);
         })
         .catch((err) => {
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
           reject(err);
         });
     } else {
@@ -147,37 +147,42 @@ function renderTable(confsResponse, searchString): Children {
       {
         title: "Edit config value",
         onclick: () => {
-          const cb = (): Children => {
+          let cb: () => Children = null;
-            return m(
+          const comp = m(
-              putFormComponent,
+            putFormComponent,
-              Object.assign(
+            Object.assign(
-                {
+              {
-                  base: conf,
+                base: conf,
-                  actionHandler: (action, object) => {
+                actionHandler: (action, object) => {
-                    return new Promise((resolve) => {
+                  return new Promise((resolve) => {
-                      putActionHandler(action, object, false)
+                    putActionHandler(action, object, false)
-                        .then((errors) => {
+                      .then((errors) => {
-                          const ErrorList = errors ? Object.values(errors) : [];
+                        const ErrorList = errors ? Object.values(errors) : [];
-                          if (ErrorList.length) {
+                        if (ErrorList.length) {
-                            for (const err of ErrorList)
+                          for (const err of ErrorList)
-                              notifications.push("error", err);
+                            notifications.push("error", err);
-                          } else {
+                        } else {
-                            overlay.close(cb);
+                          overlay.close(cb);
-                          }
+                        }
-                          resolve();
+                        resolve(null);
-                        })
+                      })
-                        .catch((err) => {
+                      .catch((err) => {
-                          notifications.push("error", err.message);
+                        notifications.push("error", err.message);
-                          resolve();
+                        resolve(null);
-                        });
+                      });
-                    });
+                  });
-                  },
                 },
-                formData
+              },
-              )
+              formData
-            );
+            )
-          };
+          );
-          overlay.open(cb);
+          cb = () => comp;
+          overlay.open(
+            cb,
+            () =>
+              !comp.state["current"]["modified"] ||
+              confirm("You have unsaved changes. Close anyway?")
+          );
         },
       },
       getIcon("edit")
@@ -244,38 +249,43 @@ export const component: ClosureComponent = (): Component => {
           {
             title: "Create new config",
             onclick: () => {
-              const cb = (): Children => {
+              let cb: () => Children = null;
-                return m(
+              const comp = m(
-                  putFormComponent,
+                putFormComponent,
-                  Object.assign(
+                Object.assign(
-                    {
+                  {
-                      actionHandler: (action, object) => {
+                    actionHandler: (action, object) => {
-                        return new Promise((resolve) => {
+                      return new Promise<void>((resolve) => {
-                          putActionHandler(action, object, true)
+                        putActionHandler(action, object, true)
-                            .then((errors) => {
+                          .then((errors) => {
-                              const errorList = errors
+                            const errorList = errors
-                                ? Object.values(errors)
+                              ? Object.values(errors)
-                                : [];
+                              : [];
-                              if (errorList.length) {
+                            if (errorList.length) {
-                                for (const err of errorList)
+                              for (const err of errorList)
-                                  notifications.push("error", err);
+                                notifications.push("error", err);
-                              } else {
+                            } else {
-                                overlay.close(cb);
+                              overlay.close(cb);
-                              }
+                            }
-                              resolve();
+                            resolve(null);
-                            })
+                          })
-                            .catch((err) => {
+                          .catch((err) => {
-                              notifications.push("error", err.message);
+                            notifications.push("error", err.message);
-                              resolve();
+                            resolve();
-                            });
+                          });
-                        });
+                      });
-                      },
                     },
-                    formData
+                  },
-                  )
+                  formData
-                );
+                )
-              };
+              );
-              overlay.open(cb);
+              cb = () => comp;
+              overlay.open(
+                cb,
+                () =>
+                  !comp.state["current"]["modified"] ||
+                  confirm("You have unsaved changes. Close anyway?")
+              );
             },
           },
           "New config"
@@ -307,39 +317,44 @@ export const component: ClosureComponent = (): Component => {
               "button",
               {
                 onclick: () => {
-                  const cb = (): Children => {
+                  let cb: () => Children = null;
-                    return m(
+                  const comp = m(
-                      uiConfigComponent,
+                    uiConfigComponent,
-                      Object.assign(
+                    Object.assign(
-                        {
+                      {
-                          onUpdate: (errs: string[]) => {
+                        onUpdate: (errs: string[]) => {
-                            const errors = errs ? Object.values(errs) : [];
+                          const errors = errs ? Object.values(errs) : [];
-                            if (errors.length) {
+                          if (errors.length) {
-                              for (const err of errors)
+                            for (const err of errors)
-                                notifications.push("error", err);
+                              notifications.push("error", err);
-                            } else {
+                          } else {
-                              notifications.push(
+                            notifications.push(
-                                "success",
+                              "success",
-                                `${sub.name.replace(
+                              `${sub.name.replace(
-                                  /^[a-z]/,
+                                /^[a-z]/,
-                                  sub.name[0].toUpperCase()
+                                sub.name[0].toUpperCase()
-                                )} config updated`
+                              )} config updated`
-                              );
+                            );
-                              overlay.close(cb);
-                            }
-                            store.fulfill(0, Date.now());
-                          },
-                          onError: (err) => {
-                            notifications.push("error", err.message);
-                            store.fulfill(0, Date.now());
                             overlay.close(cb);
-                          },
+                          }
+                          store.setTimestamp(Date.now());
                         },
-                        attrs
+                        onError: (err) => {
-                      )
+                          notifications.push("error", err.message);
-                    );
+                          store.setTimestamp(Date.now());
-                  };
+                          overlay.close(cb);
-                  overlay.open(cb);
+                        },
+                      },
+                      attrs
+                    )
+                  );
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               `Edit ${sub.name}`
@@ -351,14 +366,18 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing config"),
         m(
-          ".all-parameters",
+          "loading",
-          search,
+          { queries: [confs] },
           m(
-            ".parameter-list",
+            ".all-parameters",
-            { style: "height: 400px" },
+            search,
-            renderTable(confs, vnode.state["searchString"])
+            m(
-          ),
+              ".parameter-list",
-          m(".actions-bar", [newConfig].concat(subs))
+              { style: "height: 400px" },
+              renderTable(confs, vnode.state["searchString"])
+            ),
+            m(".actions-bar", [newConfig].concat(subs))
+          )
         ),
       ];
     },

ui/css/app.css

@@ -46,6 +46,7 @@ button,
 select,
 option,
 textarea {
+  outline-color: var(--color4);
   font-family: sans-serif;
   color: var(--color5);
 }
@@ -178,6 +179,8 @@ body {
       }
 
       & > .queue {
+        max-height: 400px;
+        overflow-y: auto;
         & > .pending {
           animation: 0.5s linear 0s infinite alternate pending-blink;
           border-radius: 0.4em;
@@ -283,9 +286,21 @@ body {
   }
 
   & > .logo {
-    margin: 10px;
+    display: inline-block;
-    display: inline;
+    position: relative;
-    height: 56px;
+    & > img {
+      margin: 10px;
+      height: 56px;
+    }
+
+    & > .version {
+      position: absolute;
+      bottom: 5px;
+      right: 10px;
+      font-family: monospace;
+      font-size: 10px;
+      color: #666;
+    }
   }
 
   & > .user-menu {
@@ -327,12 +342,18 @@ body {
   flex-grow: 1;
   margin: 10px;
   min-height: 250px;
+
+  /* Workaround for Chrome ignoring bottom margin when flex container overflows */
+  &::after {
+    content: "";
+    display: block;
+    padding-bottom: 1px;
+  }
 }
 
 table.table {
   border-spacing: 0;
   text-align: left;
-  display: block;
   overflow-x: auto;
 
   & table.table {
@@ -738,3 +759,31 @@ span.parameter-value {
     opacity: 0.3;
   }
 }
+
+.loading {
+  opacity: 0.6;
+}
+
+@keyframes spin {
+  0% {
+    transform: rotate(0deg);
+  }
+  100% {
+    transform: rotate(360deg);
+  }
+}
+
+.loading-overlay {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  &::after {
+    background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxMDAgMTAwIiBwcmVzZXJ2ZUFzcGVjdFJhdGlvPSJ4TWlkWU1pZCI+PHBhdGggZD0iTTEwIDUwYTQwIDQwIDAgMDA4MCAwIDQwIDQyIDAgMDEtODAgMCIgZmlsbD0iI2IxMmQ1YyIgLz48L3N2Zz4=);
+    content: "";
+    width: 48px;
+    height: 48px;
+    animation: spin 1s linear infinite;
+    background-repeat: no-repeat;
+    background-size: 100% 100%;
+  }
+}

ui/datalist.ts

@@ -0,0 +1,62 @@
+/**
+ * Copyright 2013-2021  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import m, { ClosureComponent, Component, Vnode } from "mithril";
+
+const elements: Map<string, Vnode> = new Map();
+
+// Source: https://werxltd.com/wp/2010/05/13/javascript-implementation-of-javas-string-hashcode-method/
+function hash(str: string): number {
+  let res = 0;
+  for (let i = 0; i < str.length; ++i) {
+    const c = str.charCodeAt(i);
+    res = (res << 5) - res + c;
+    res |= 0;
+  }
+  return res;
+}
+
+export function getDatalistId(options: string[]): string {
+  const id = "datalist" + options.reduce((acc, cur) => acc ^ hash(cur), 0);
+  if (!elements.has(id)) {
+    const n = m(
+      "datalist",
+      { id },
+      options.map((o) => m("option", { value: o }))
+    );
+    elements.set(id, n);
+  }
+  return id;
+}
+
+const component: ClosureComponent = (): Component => {
+  return {
+    view: () => {
+      return [...elements.values()];
+    },
+    onupdate: () => {
+      for (const id of elements.keys()) {
+        const used = document.querySelector(`[list='${id}']`);
+        if (!used) elements.delete(id);
+      }
+    },
+  };
+};
+
+export default component;

ui/device-page.ts

@@ -42,13 +42,25 @@ export const component: ClosureComponent = (): Component => {
     view: (vnode) => {
       document.title = `${vnode.attrs["deviceId"]} - Devices - GenieACS`;
 
-      const dev = store.fetch("devices", vnode.attrs["deviceFilter"]).value;
+      const dev = store.fetch("devices", vnode.attrs["deviceFilter"]);
-      if (!dev.length) return "Loading";
+      if (!dev.value.length) {
+        if (!dev.fulfilling)
+          return m("p.error", `No such device ${vnode.attrs["deviceId"]}`);
+        return m(
+          "loading",
+          { queries: [dev] },
+          m("div", { style: "height: 100px;" })
+        );
+      }
+
       const conf = config.ui.device;
       const cmps = [];
 
-      for (const c of Object.values(conf))
+      for (const c of Object.values(conf)) {
-        cmps.push(m.context({ device: dev[0] }, c["type"], c));
+        cmps.push(
+          m.context({ device: dev.value[0], deviceQuery: dev }, c["type"], c)
+        );
+      }
 
       return [m("h1", vnode.attrs["deviceId"]), cmps];
     },

ui/devices-page.ts

@@ -23,7 +23,7 @@ import config from "./config";
 import indexTableComponent from "./index-table-component";
 import filterComponent from "./filter-component";
 import * as store from "./store";
-import * as taskQueue from "./task-queue";
+import { queueTask, stageDownload } from "./task-queue";
 import * as notifications from "./notifications";
 import { parse, stringify } from "../lib/common/expression-parser";
 import { evaluate, extractParams } from "../lib/common/expression";
@@ -91,12 +91,11 @@ function renderActions(selected: Set<string>): Children {
         title: "Reboot selected devices",
         disabled: !selected.size,
         onclick: () => {
-          for (const d of selected) {
+          const tasks = [...selected].map((s) => ({
-            taskQueue.queueTask({
+            name: "reboot",
-              name: "reboot",
+            device: s,
-              device: d,
+          }));
-            });
+          queueTask(...tasks);
-          }
         },
       },
       "Reboot"
@@ -110,18 +109,34 @@ function renderActions(selected: Set<string>): Children {
         title: "Factory reset selected devices",
         disabled: !selected.size,
         onclick: () => {
-          for (const d of selected) {
+          const tasks = [...selected].map((s) => ({
-            taskQueue.queueTask({
+            name: "factoryReset",
-              name: "factoryReset",
+            device: s,
-              device: d,
+          }));
-            });
+          queueTask(...tasks);
-          }
         },
       },
       "Reset"
     )
   );
 
+  buttons.push(
+    m(
+      "button.critical",
+      {
+        title: "Push a firmware or a config file",
+        disabled: !selected.size,
+        onclick: () => {
+          stageDownload({
+            name: "download",
+            devices: [...selected],
+          });
+        },
+      },
+      "Push file"
+    )
+  );
+
   buttons.push(
     m(
       "button.primary",
@@ -139,14 +154,14 @@ function renderActions(selected: Set<string>): Children {
               .deleteResource("devices", id)
               .then(() => {
                 notifications.push("success", `${id}: Deleted`);
-                if (--counter === 0) store.fulfill(0, Date.now());
+                if (--counter === 0) store.setTimestamp(Date.now());
               })
               .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
-                if (--counter === 0) store.fulfill(0, Date.now());
+                if (--counter === 0) store.setTimestamp(Date.now());
               });
           }
-          if (--counter === 0) store.fulfill(0, Date.now());
+          if (--counter === 0) store.setTimestamp(Date.now());
         },
       },
       "Delete"
@@ -171,14 +186,14 @@ function renderActions(selected: Set<string>): Children {
               .updateTags(id, { [tag]: true })
               .then(() => {
                 notifications.push("success", `${id}: Tags updated`);
-                if (--counter === 0) store.fulfill(0, Date.now());
+                if (--counter === 0) store.setTimestamp(Date.now());
               })
               .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
-                if (--counter === 0) store.fulfill(0, Date.now());
+                if (--counter === 0) store.setTimestamp(Date.now());
               });
           }
-          if (--counter === 0) store.fulfill(0, Date.now());
+          if (--counter === 0) store.setTimestamp(Date.now());
         },
       },
       "Tag"
@@ -205,14 +220,14 @@ function renderActions(selected: Set<string>): Children {
               .updateTags(id, { [tag]: false })
               .then(() => {
                 notifications.push("success", `${id}: Tags updated`);
-                if (--counter === 0) store.fulfill(0, Date.now());
+                if (--counter === 0) store.setTimestamp(Date.now());
               })
               .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
-                if (--counter === 0) store.fulfill(0, Date.now());
+                if (--counter === 0) store.setTimestamp(Date.now());
               });
           }
-          if (--counter === 0) store.fulfill(0, Date.now());
+          if (--counter === 0) store.setTimestamp(Date.now());
         },
       },
       "Untag"
@@ -321,7 +336,7 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing devices"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs),
+        m("loading", { queries: [devs, count] }, m(indexTableComponent, attrs)),
       ];
     },
   };

ui/drawer-component.ts

@@ -24,10 +24,21 @@ import m, {
   Component,
   VnodeDOM,
 } from "mithril";
-import * as taskQueue from "./task-queue";
 import * as store from "./store";
 import * as notifications from "./notifications";
 import { getIcon } from "./icons";
+import {
+  clear,
+  commit,
+  deleteTask,
+  getQueue,
+  getStaging,
+  QueueTask,
+  queueTask,
+  StageTask,
+} from "./task-queue";
+
+const invalid: WeakSet<StageTask> = new WeakSet();
 
 function mparam(param): Children {
   return m("span.parameter", { title: param }, `${param}`);
@@ -37,11 +48,11 @@ function mval(val): Children {
   return m("span.value", { title: val }, `${val}`);
 }
 
-function renderStagingSpv(task, queueFunc, cancelFunc): Children {
+function renderStagingSpv(task: StageTask, queueFunc, cancelFunc): Children {
-  function keydown(e): void {
+  function keydown(e: KeyboardEvent): void {
-    if (e.keyCode === 13) queueFunc();
+    if (e.key === "Enter") queueFunc();
-    else if (e.keyCode === 27) cancelFunc();
+    else if (e.key === "Escape") cancelFunc();
-    else e.redraw = false;
+    else e["redraw"] = false;
   }
 
   let input;
@@ -65,13 +76,13 @@ function renderStagingSpv(task, queueFunc, cancelFunc): Children {
       ]
     );
   } else {
+    const type = task.parameterValues[0][2];
+    let value = task.parameterValues[0][1];
+    if (type === "xsd:dateTime" && typeof value === "number")
+      value = new Date(value).toJSON() || value;
     input = m("input", {
-      type: ["xsd:dateTime", "xsd:int", "xsd:unsignedInt"].includes(
+      type: ["xsd:int", "xsd:unsignedInt"].includes(type) ? "number" : "text",
-        task.parameterValues[0][2]
+      value: value,
-      )
-        ? "number"
-        : "text",
-      value: task.parameterValues[0][1],
       oninput: (e) => {
         e.redraw = false;
         task.parameterValues[0][1] = input.dom.value;
@@ -90,21 +101,35 @@ function renderStagingSpv(task, queueFunc, cancelFunc): Children {
   return [m("span", "Editing ", mparam(task.parameterValues[0][0])), input];
 }
 
-function renderStagingDownload(task): Children {
+function renderStagingDownload(task: StageTask): Children {
-  task.invalid = !task.fileName || !task.fileType;
+  if (!task.fileName || !task.fileType) invalid.add(task);
+  else invalid.delete(task);
   const files = store.fetch("files", true);
-  const idParts = task.device.split("-");
+  let oui = "";
-  const oui = decodeURIComponent(idParts[0]);
+  let productClass = "";
-  const productClass =
+  for (const d of task.devices) {
-    idParts.length === 3 ? decodeURIComponent(idParts[1]) : "";
+    const parts = d.split("-");
+    if (oui === "") oui = parts[0];
+    else if (oui !== parts[0]) oui = null;
+    if (parts.length === 3) {
+      if (productClass === "") productClass = parts[1];
+      else if (productClass !== parts[1]) productClass = null;
+    }
+  }
+
+  if (oui) oui = decodeURIComponent(oui);
+  if (productClass) productClass = decodeURIComponent(productClass);
 
   const typesList = [
-    "",
+    ...new Set([
-    "1 Firmware Upgrade Image",
+      "",
-    "2 Web Content",
+      "1 Firmware Upgrade Image",
-    "3 Vendor Configuration File",
+      "2 Web Content",
-    "4 Tone File",
+      "3 Vendor Configuration File",
-    "5 Ringer File",
+      "4 Tone File",
+      "5 Ringer File",
+      ...files.value.map((f) => f["metadata.fileType"]).filter((f) => f),
+    ]),
   ].map((t) =>
     m(
       "option",
@@ -162,13 +187,17 @@ function renderStagingDownload(task): Children {
   ];
 }
 
-function renderStaging(staging): Child[] {
+function renderStaging(staging: Set<StageTask>): Child[] {
   const elements: Child[] = [];
 
   for (const s of staging) {
     const queueFunc = (): void => {
       staging.delete(s);
-      taskQueue.queueTask(s);
+      for (const d of s.devices) {
+        const t = Object.assign({ device: d }, s);
+        delete t.devices;
+        queueTask(t);
+      }
     };
     const cancelFunc = (): void => {
       staging.delete(s);
@@ -181,7 +210,7 @@ function renderStaging(staging): Child[] {
 
     const queue = m(
       "button.primary",
-      { title: "Queue task", onclick: queueFunc, disabled: s.invalid },
+      { title: "Queue task", onclick: queueFunc, disabled: invalid.has(s) },
       "Queue"
     );
     const cancel = m(
@@ -195,7 +224,7 @@ function renderStaging(staging): Child[] {
   return elements;
 }
 
-function renderQueue(queue): Child[] {
+function renderQueue(queue: Set<QueueTask>): Child[] {
   const details: Child[] = [];
   const devices: { [deviceId: string]: any[] } = {};
   for (const t of queue) {
@@ -215,7 +244,7 @@ function renderQueue(queue): Child[] {
             {
               title: "Retry this task",
               onclick: () => {
-                taskQueue.queueTask(t);
+                queueTask(t);
               },
             },
             getIcon("retry")
@@ -229,7 +258,7 @@ function renderQueue(queue): Child[] {
           {
             title: "Remove this task",
             onclick: () => {
-              taskQueue.deleteTask(t);
+              deleteTask(t);
             },
           },
           getIcon("remove")
@@ -329,7 +358,7 @@ function renderNotifications(notifs): Child[] {
           },
           onbeforeremove: (vnode) => {
             (vnode.dom as HTMLDivElement).style.opacity = "0";
-            return new Promise((resolve) => {
+            return new Promise<void>((resolve) => {
               setTimeout(() => {
                 resolve();
               }, 500);
@@ -347,8 +376,8 @@ function renderNotifications(notifs): Child[] {
 const component: ClosureComponent = (): Component => {
   return {
     view: (vnode) => {
-      const queue = taskQueue.getQueue();
+      const queue = getQueue();
-      const staging = taskQueue.getStaging();
+      const staging = getStaging();
       const notifs = notifications.getNotifications();
 
       let drawerElement, statusElement;
@@ -394,53 +423,55 @@ const component: ClosureComponent = (): Component => {
               title: "Commit queued tasks",
               disabled: !statusCount.queued,
               onclick: () => {
-                const tasks = Array.from(taskQueue.getQueue()).filter(
+                const tasks = Array.from(getQueue()).filter(
                   (t) => t["status"] === "queued"
                 );
-                taskQueue
+                commit(
-                  .commit(
+                  tasks,
-                    tasks,
+                  (deviceId, err, connectionRequestStatus, tasks2) => {
-                    (deviceId, err, connectionRequestStatus, tasks2) => {
+                    if (err) {
-                      if (err) {
-                        notifications.push(
-                          "error",
-                          `${deviceId}: ${err.message}`
-                        );
-                        return;
-                      }
-
-                      if (connectionRequestStatus !== "OK") {
-                        notifications.push(
-                          "error",
-                          `${deviceId}: ${connectionRequestStatus}`
-                        );
-                        return;
-                      }
-
-                      for (const t of tasks2) {
-                        if (t.status === "stale") {
-                          notifications.push(
-                            "error",
-                            `${deviceId}: No contact from device`
-                          );
-                          return;
-                        } else if (t.status === "fault") {
-                          notifications.push(
-                            "error",
-                            `${deviceId}: Task(s) faulted`
-                          );
-                          return;
-                        }
-                      }
-
                       notifications.push(
-                        "success",
+                        "error",
-                        `${deviceId}: Task(s) committed`
+                        `${deviceId}: ${err.message}`
                       );
+                      return;
                     }
-                  )
+
+                    if (connectionRequestStatus !== "OK") {
+                      notifications.push(
+                        "error",
+                        `${deviceId}: ${connectionRequestStatus}`
+                      );
+                      return;
+                    }
+
+                    for (const t of tasks2) {
+                      if (t.status === "stale") {
+                        notifications.push(
+                          "error",
+                          `${deviceId}: No contact from device`
+                        );
+                        return;
+                      } else if (t.status === "fault") {
+                        notifications.push(
+                          "error",
+                          `${deviceId}: Task(s) faulted`
+                        );
+                        return;
+                      }
+                    }
+
+                    notifications.push(
+                      "success",
+                      `${deviceId}: Task(s) committed`
+                    );
+                  }
+                )
                   .then(() => {
-                    store.fulfill(0, Date.now());
+                    store.setTimestamp(Date.now());
+                  })
+                  .catch((err) => {
+                    notifications.push("error", err.message);
                   });
               },
             },
@@ -450,7 +481,7 @@ const component: ClosureComponent = (): Component => {
             "button",
             {
               title: "Clear tasks",
-              onclick: taskQueue.clear,
+              onclick: clear,
               disabled: !queueElements.length,
             },
             "Clear"

ui/dynamic-loader.ts

@@ -19,8 +19,8 @@
 
 import * as notifications from "./notifications";
 
-export let codeMirror;
+export let codeMirror: typeof import("codemirror");
-export let yaml;
+export let yaml: typeof import("yaml");
 
 let note;
 
@@ -43,15 +43,13 @@ export function loadCodeMirror(): Promise<void> {
 
   return new Promise((resolve, reject) => {
     const promises = [
-      import(/* webpackChunkName: "codemirror" */ "codemirror"),
+      import("codemirror"),
-      import(
+      import("codemirror/mode/javascript/javascript"),
-        /* webpackChunkName: "codemirror" */ "codemirror/mode/javascript/javascript"
+      import("codemirror/mode/yaml/yaml"),
-      ),
-      import(/* webpackChunkName: "codemirror" */ "codemirror/mode/yaml/yaml"),
     ];
     Promise.all(promises)
       .then((modules) => {
-        codeMirror = modules[0];
+        codeMirror = modules[0].default;
         resolve();
       })
       .catch((err) => {
@@ -65,7 +63,7 @@ export function loadYaml(): Promise<void> {
   if (yaml) return Promise.resolve();
 
   return new Promise((resolve, reject) => {
-    import(/* webpackChunkName: "yaml" */ "yaml")
+    import("yaml")
       .then((module) => {
         yaml = module;
         resolve();

ui/faults-page.ts

@@ -27,7 +27,7 @@ import * as notifications from "./notifications";
 import memoize from "../lib/common/memoize";
 import * as smartQuery from "./smart-query";
 import { map, parse, stringify } from "../lib/common/expression-parser";
-import { loadYaml, yaml } from "./dynamic-loader";
+import { stringify as yamlStringify } from "../lib/common/yaml";
 
 const PAGE_SIZE = config.ui.pageSize || 10;
 
@@ -76,13 +76,7 @@ export function init(
 
   const sort = args.hasOwnProperty("sort") ? "" + args["sort"] : "";
   const filter = args.hasOwnProperty("filter") ? "" + args["filter"] : "";
-  return new Promise((resolve, reject) => {
+  return Promise.resolve({ filter, sort });
-    loadYaml()
-      .then(() => {
-        resolve({ filter, sort });
-      })
-      .catch(reject);
-  });
 }
 
 export const component: ClosureComponent = (): Component => {
@@ -147,8 +141,11 @@ export const component: ClosureComponent = (): Component => {
           return m("a", { href: deviceHref }, fault["device"]);
         }
 
+        if (attr.id === "message")
+          return m("long-text", { text: fault["message"] });
+
         if (attr.id === "detail")
-          return m("long-text", { text: yaml.stringify(fault["detail"]) });
+          return m("long-text", { text: yamlStringify(fault["detail"]) });
 
         if (attr.id === "timestamp")
           return new Date(fault["timestamp"]).toLocaleString();
@@ -190,11 +187,11 @@ export const component: ClosureComponent = (): Component => {
                       "success",
                       `${res.length} faults deleted`
                     );
-                    store.fulfill(0, Date.now());
+                    store.setTimestamp(Date.now());
                   })
                   .catch((err) => {
                     notifications.push("error", err.message);
-                    store.fulfill(0, Date.now());
+                    store.setTimestamp(Date.now());
                   });
               },
             },
@@ -211,7 +208,11 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing faults"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs),
+        m(
+          "loading",
+          { queries: [faults, count] },
+          m(indexTableComponent, attrs)
+        ),
       ];
     },
   };

ui/files-page.ts

@@ -45,7 +45,6 @@ const attributes: {
   {
     id: "metadata.fileType",
     label: "Type",
-    type: "combo",
     options: [
       "1 Firmware Upgrade Image",
       "2 Web Content",
@@ -93,7 +92,7 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
         .resourceExists("files", id)
         .then((exists) => {
           if (exists) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ file: "File already exists" });
           }
           const headers = Object.assign(
@@ -117,8 +116,8 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
                 "success",
                 `File ${exists ? "updated" : "created"}`
               );
-              store.fulfill(0, Date.now());
+              store.setTimestamp(Date.now());
-              resolve();
+              resolve(null);
             })
             .catch(reject);
         })
@@ -212,6 +211,9 @@ export const component: ClosureComponent = (): Component => {
       attrs["sortAttributes"] = sortAttributes;
       attrs["onSortChange"] = onSortChange;
       attrs["downloadUrl"] = downloadUrl;
+      attrs["recordActionsCallback"] = (file) => {
+        return [m("a", { href: "api/blob/files/" + file["_id"] }, "Download")];
+      };
 
       if (window.authorizer.hasAccess("files", 3)) {
         attrs["actionsCallback"] = (selected: Set<string>): Children => {
@@ -221,38 +223,43 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new file",
                 onclick: () => {
-                  const cb = (): Children => {
+                  let cb: () => Children = null;
-                    return m(
+                  const comp = m(
-                      putFormComponent,
+                    putFormComponent,
-                      Object.assign(
+                    Object.assign(
-                        {
+                      {
-                          actionHandler: (action, object) => {
+                        actionHandler: (action, object) => {
-                            return new Promise((resolve) => {
+                          return new Promise<void>((resolve) => {
-                              putActionHandler(action, object)
+                            putActionHandler(action, object)
-                                .then((errors) => {
+                              .then((errors) => {
-                                  const errorList = errors
+                                const errorList = errors
-                                    ? Object.values(errors)
+                                  ? Object.values(errors)
-                                    : [];
+                                  : [];
-                                  if (errorList.length) {
+                                if (errorList.length) {
-                                    for (const err of errorList)
+                                  for (const err of errorList)
-                                      notifications.push("error", err);
+                                    notifications.push("error", err);
-                                  } else {
+                                } else {
-                                    overlay.close(cb);
+                                  overlay.close(cb);
-                                  }
+                                }
-                                  resolve();
+                                resolve();
-                                })
+                              })
-                                .catch((err) => {
+                              .catch((err) => {
-                                  notifications.push("error", err.message);
+                                notifications.push("error", err.message);
-                                  resolve();
+                                resolve();
-                                });
+                              });
-                            });
+                          });
-                          },
                         },
-                        formData
+                      },
-                      )
+                      formData
-                    );
+                    )
-                  };
+                  );
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"
@@ -280,11 +287,11 @@ export const component: ClosureComponent = (): Component => {
                         "success",
                         `${res.length} files deleted`
                       );
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     })
                     .catch((err) => {
                       notifications.push("error", err.message);
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     });
                 },
               },
@@ -302,7 +309,11 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing files"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs),
+        m(
+          "loading",
+          { queries: [files, count] },
+          m(indexTableComponent, attrs)
+        ),
       ];
     },
   };

ui/filter-component.ts

@@ -44,8 +44,12 @@ const getAutocomplete = memoize((resource) => {
 
 function validateQuery(q: Expression): void {
   q = normalize(q);
-  if (Array.isArray(q) && q[0] === "CASE") q = ["AND", ...q.slice(1)];
+  if (Array.isArray(q) && q[0] === "CASE") {
-  filterToMongoQuery(q);
+    q = q.slice(1).filter((a) => Array.isArray(a));
+    if (q.length > 1) q = ["AND", ...q];
+    else q = q[0];
+  }
+  if (Array.isArray(q)) filterToMongoQuery(q);
 }
 
 function parseFilter(resource, f): Expression {

ui/index-table-component.ts

@@ -27,6 +27,8 @@ interface Attribute {
   type?: string;
 }
 
+const MAX_PAGE_SIZE = 200;
+
 function renderTable(
   attributes: Attribute[],
   data: Record<string, any>[],
@@ -37,7 +39,7 @@ function renderTable(
   onSortChange: (obj: Record<string, number>) => void,
   downloadUrl?: string,
   valueCallback?: (attr: Attribute, record: Record<string, any>) => Children,
-  actionsCallback?: Children | ((selected: Set<string>) => Children),
+  actionsCallback?: Children | ((sel: Set<string>) => Children),
   recordActionsCallback?: Children | ((record: Record<string, any>) => Children)
 ): Children {
   const records = data || [];
@@ -180,7 +182,8 @@ function renderTable(
       {
         title: "Show more records",
         onclick: showMoreCallback,
-        disabled: !data.length || records.length >= total,
+        disabled:
+          !data.length || records.length >= Math.min(MAX_PAGE_SIZE, total),
       },
       "More"
     )

ui/layout.ts

@@ -23,6 +23,8 @@ import drawerComponent from "./drawer-component";
 import userMenu from "./user-menu";
 import adminMenu from "./admin-menu";
 import * as overlay from "./overlay";
+import { version as VERSION } from "../package.json";
+import datalist from "./datalist";
 
 const adminPages = [
   "presets",
@@ -51,7 +53,11 @@ const component: ClosureComponent = (): Component => {
 
       return [
         m("#header", [
-          m("img.logo", { src: "logo.svg" }),
+          m(
+            "div.logo",
+            m("img", { src: "logo.svg" }),
+            m("span.version", "v" + VERSION)
+          ),
           m(userMenu),
           m(menu, attrs),
           m(drawerComponent),
@@ -64,6 +70,7 @@ const component: ClosureComponent = (): Component => {
           ])
         ),
         overlay.render(),
+        m(datalist),
       ];
     },
   };

ui/login-page.ts

@@ -81,7 +81,7 @@ export const component: ClosureComponent = (): Component => {
                       location.reload();
                     })
                     .catch((err) => {
-                      notifications.push("error", err.message);
+                      notifications.push("error", err.response || err.message);
                       e.target.disabled = false;
                     });
                   return false;

ui/overlay.ts

@@ -20,16 +20,24 @@
 import m, { Children } from "mithril";
 
 type OverlayCallback = () => Children;
+type CloseCallback = () => boolean;
 
 let overlayCallback: OverlayCallback = null;
+let closeCallback: CloseCallback = null;
 
-export function open(callback: OverlayCallback): void {
+export function open(
+  callback: OverlayCallback,
+  closeCb: CloseCallback = null
+): void {
   overlayCallback = callback;
+  closeCallback = closeCb;
 }
 
-export function close(callback: OverlayCallback): boolean {
+export function close(callback: OverlayCallback, force = true): boolean {
   if (callback === overlayCallback) {
+    if (!force && closeCallback && !closeCallback()) return false;
     overlayCallback = null;
+    closeCallback = null;
     return true;
   }
 
@@ -43,7 +51,7 @@ export function render(): Children {
       {
         tabindex: 0,
         onclick: () => {
-          close(overlayCallback);
+          close(overlayCallback, false);
         },
         style: "opacity: 0",
         oncreate: (vnode) => {
@@ -52,7 +60,7 @@ export function render(): Children {
         },
         onbeforeremove: (vnode) => {
           (vnode.dom as HTMLDivElement).style.opacity = "0";
-          return new Promise((resolve) => {
+          return new Promise<void>((resolve) => {
             setTimeout(() => {
               resolve();
             }, 500);
@@ -75,9 +83,10 @@ export function render(): Children {
 }
 
 document.addEventListener("keydown", (e) => {
-  if (overlayCallback && e.keyCode === 27 && close(overlayCallback)) m.redraw();
+  if (overlayCallback && e.key === "Escape" && close(overlayCallback, false))
+    m.redraw();
 });
 
 window.addEventListener("popstate", () => {
-  if (close(overlayCallback)) m.redraw();
+  if (close(overlayCallback, false)) m.redraw();
 });

ui/permissions-page.ts

@@ -117,7 +117,7 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
         .resourceExists("permissions", id)
         .then((exists) => {
           if (exists) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ _id: "Permission already exists" });
           }
 
@@ -125,8 +125,8 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
             .putResource("permissions", id, object)
             .then(() => {
               notifications.push("success", "Permission created");
-              store.fulfill(0, Date.now());
+              store.setTimestamp(Date.now());
-              resolve();
+              resolve(null);
             })
             .catch(reject);
         })
@@ -136,11 +136,11 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
         .deleteResource("permissions", object["_id"])
         .then(() => {
           notifications.push("success", "Permission deleted");
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
-          resolve();
+          resolve(null);
         })
         .catch((err) => {
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
           reject(err);
         });
     } else {
@@ -285,38 +285,43 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new permission",
                 onclick: () => {
-                  const cb = (): Children => {
+                  let cb: () => Children = null;
-                    return m(
+                  const comp = m(
-                      putFormComponent,
+                    putFormComponent,
-                      Object.assign(
+                    Object.assign(
-                        {
+                      {
-                          actionHandler: (action, object) => {
+                        actionHandler: (action, object) => {
-                            return new Promise((resolve) => {
+                          return new Promise<void>((resolve) => {
-                              putActionHandler(action, object)
+                            putActionHandler(action, object)
-                                .then((errors) => {
+                              .then((errors) => {
-                                  const errorList = errors
+                                const errorList = errors
-                                    ? Object.values(errors)
+                                  ? Object.values(errors)
-                                    : [];
+                                  : [];
-                                  if (errorList.length) {
+                                if (errorList.length) {
-                                    for (const err of errorList)
+                                  for (const err of errorList)
-                                      notifications.push("error", err);
+                                    notifications.push("error", err);
-                                  } else {
+                                } else {
-                                    overlay.close(cb);
+                                  overlay.close(cb);
-                                  }
+                                }
-                                  resolve();
+                                resolve();
-                                })
+                              })
-                                .catch((err) => {
+                              .catch((err) => {
-                                  notifications.push("error", err.message);
+                                notifications.push("error", err.message);
-                                  resolve();
+                                resolve();
-                                });
+                              });
-                            });
+                          });
-                          },
                         },
-                        formData
+                      },
-                      )
+                      formData
-                    );
+                    )
-                  };
+                  );
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"
@@ -346,11 +351,11 @@ export const component: ClosureComponent = (): Component => {
                         "success",
                         `${res.length} permissions deleted`
                       );
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     })
                     .catch((err) => {
                       notifications.push("error", err.message);
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     });
                 },
               },
@@ -368,7 +373,11 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing permissions"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs),
+        m(
+          "loading",
+          { queries: [permissions, count] },
+          m(indexTableComponent, attrs)
+        ),
       ];
     },
   };

ui/pie-chart-component.ts

@@ -116,21 +116,27 @@ function drawChart(chartData): Children {
 
   legend.push(m("span.legend-total", `Total: ${total}`));
 
-  return m("div", { class: "pie-chart" }, [
+  return m(
-    m(
+    "loading",
-      "svg",
+    {
-      {
+      queries: Object.values(chartData.slices).map((s) => s["count"]),
-        // Adding 2 as padding; strokes must not be more than 2
+    },
-        viewBox: "-102 -102 204 204",
+    m("div", { class: "pie-chart" }, [
-        width: "204px",
+      m(
-        height: "204px",
+        "svg",
-        xmlns: "http://www.w3.org/2000/svg",
+        {
-        "xmlns:xlink": "http://www.w3.org/1999/xlink",
+          // Adding 2 as padding; strokes must not be more than 2
-      },
+          viewBox: "-102 -102 204 204",
-      paths.concat(links)
+          width: "204px",
-    ),
+          height: "204px",
-    m(".legend", legend),
+          xmlns: "http://www.w3.org/2000/svg",
-  ]);
+          "xmlns:xlink": "http://www.w3.org/1999/xlink",
+        },
+        paths.concat(links)
+      ),
+      m(".legend", legend),
+    ])
+  );
 }
 
 const component: ClosureComponent = (): Component => {

ui/presets-page.ts

@@ -86,12 +86,12 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
         .resourceExists("presets", id)
         .then((exists) => {
           if (exists && isNew) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ _id: "Preset already exists" });
           }
 
           if (!exists && !isNew) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ _id: "Preset does not exist" });
           }
 
@@ -102,8 +102,8 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
                 "success",
                 `Preset ${exists ? "updated" : "created"}`
               );
-              store.fulfill(0, Date.now());
+              store.setTimestamp(Date.now());
-              resolve();
+              resolve(null);
             })
             .catch(reject);
         })
@@ -113,12 +113,12 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
         .deleteResource("presets", object["_id"])
         .then(() => {
           notifications.push("success", "Preset deleted");
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
-          resolve();
+          resolve(null);
         })
         .catch((err) => {
           reject(err);
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
         });
     } else {
       reject(new Error("Undefined action"));
@@ -287,7 +287,38 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
+                let cb: () => Children = null;
+                const comp = m(
+                  putFormComponent,
+                  Object.assign(
+                    {
+                      base: preset,
+                      actionHandler: (action, object) => {
+                        return new Promise<void>((resolve) => {
+                          putActionHandler(action, object, false)
+                            .then((errors) => {
+                              const errorList = errors
+                                ? Object.values(errors)
+                                : [];
+                              if (errorList.length) {
+                                for (const err of errorList)
+                                  notifications.push("error", err);
+                              } else {
+                                overlay.close(cb);
+                              }
+                              resolve();
+                            })
+                            .catch((err) => {
+                              notifications.push("error", err.message);
+                              resolve();
+                            });
+                        });
+                      },
+                    },
+                    formData
+                  )
+                );
+                cb = (): Children => {
                   if (!preset.provision) {
                     return m(
                       "div",
@@ -295,15 +326,37 @@ export const component: ClosureComponent = (): Component => {
                       "This UI only supports presets with a single 'provision' configuraiton. If this preset was originally created from the old UI (genieacs-gui), you must edit it there."
                     );
                   }
+                  return comp;
+                };
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
+              },
+            },
+            "Show"
+          ),
+        ];
+      };
 
-                  return m(
+      if (window.authorizer.hasAccess("presets", 3)) {
+        attrs["actionsCallback"] = (selected: Set<string>): Children => {
+          return [
+            m(
+              "button.primary",
+              {
+                title: "Create new preset",
+                onclick: () => {
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
-                        base: preset,
                         actionHandler: (action, object) => {
-                          return new Promise((resolve) => {
+                          return new Promise<void>((resolve) => {
-                            putActionHandler(action, object, false)
+                            putActionHandler(action, object, true)
                               .then((errors) => {
                                 const errorList = errors
                                   ? Object.values(errors)
@@ -326,55 +379,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                };
+                  cb = () => comp;
-                overlay.open(cb);
+                  overlay.open(
-              },
+                    cb,
-            },
+                    () =>
-            "Show"
+                      !comp.state["current"]["modified"] ||
-          ),
+                      confirm("You have unsaved changes. Close anyway?")
-        ];
+                  );
-      };
-
-      if (window.authorizer.hasAccess("presets", 3)) {
-        attrs["actionsCallback"] = (selected: Set<string>): Children => {
-          return [
-            m(
-              "button.primary",
-              {
-                title: "Create new preset",
-                onclick: () => {
-                  const cb = (): Children => {
-                    return m(
-                      putFormComponent,
-                      Object.assign(
-                        {
-                          actionHandler: (action, object) => {
-                            return new Promise((resolve) => {
-                              putActionHandler(action, object, true)
-                                .then((errors) => {
-                                  const errorList = errors
-                                    ? Object.values(errors)
-                                    : [];
-                                  if (errorList.length) {
-                                    for (const err of errorList)
-                                      notifications.push("error", err);
-                                  } else {
-                                    overlay.close(cb);
-                                  }
-                                  resolve();
-                                })
-                                .catch((err) => {
-                                  notifications.push("error", err.message);
-                                  resolve();
-                                });
-                            });
-                          },
-                        },
-                        formData
-                      )
-                    );
-                  };
-                  overlay.open(cb);
                 },
               },
               "New"
@@ -402,11 +413,11 @@ export const component: ClosureComponent = (): Component => {
                         "success",
                         `${res.length} presets deleted`
                       );
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     })
                     .catch((err) => {
                       notifications.push("error", err.message);
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     });
                 },
               },
@@ -424,7 +435,11 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing presets"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs),
+        m(
+          "loading",
+          { queries: [presets, count] },
+          m(indexTableComponent, attrs)
+        ),
       ];
     },
   };

ui/provisions-page.ts

@@ -66,12 +66,12 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
         .resourceExists("provisions", id)
         .then((exists) => {
           if (exists && isNew) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ _id: "Provision already exists" });
           }
 
           if (!exists && !isNew) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ _id: "Provision does not exist" });
           }
 
@@ -82,10 +82,16 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
                 "success",
                 `Provision ${exists ? "updated" : "created"}`
               );
-              store.fulfill(0, Date.now());
+              store.setTimestamp(Date.now());
-              resolve();
+              resolve(null);
             })
-            .catch(reject);
+            .catch((err) => {
+              if (err["code"] === 400 && err["response"]) {
+                reject(new Error(err["response"]));
+                return;
+              }
+              reject(err);
+            });
         })
         .catch(reject);
     } else if (action === "delete") {
@@ -93,11 +99,11 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
         .deleteResource("provisions", object["_id"])
         .then(() => {
           notifications.push("success", "Provision deleted");
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
-          resolve();
+          resolve(null);
         })
         .catch((err) => {
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
           reject(err);
         });
     } else {
@@ -207,15 +213,67 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
+                let cb: () => Children = null;
-                  return m(
+                const comp = m(
+                  putFormComponent,
+                  Object.assign(
+                    {
+                      base: provision,
+                      actionHandler: (action, object) => {
+                        return new Promise<void>((resolve) => {
+                          putActionHandler(action, object, false)
+                            .then((errors) => {
+                              const errorList = errors
+                                ? Object.values(errors)
+                                : [];
+                              if (errorList.length) {
+                                for (const err of errorList)
+                                  notifications.push("error", err);
+                              } else {
+                                overlay.close(cb);
+                              }
+                              resolve();
+                            })
+                            .catch((err) => {
+                              notifications.push("error", err.message);
+                              resolve();
+                            });
+                        });
+                      },
+                    },
+                    formData
+                  )
+                );
+                cb = () => comp;
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
+              },
+            },
+            "Show"
+          ),
+        ];
+      };
+
+      if (window.authorizer.hasAccess("provisions", 3)) {
+        attrs["actionsCallback"] = (selected: Set<string>): Children => {
+          return [
+            m(
+              "button.primary",
+              {
+                title: "Create new provision",
+                onclick: () => {
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
-                        base: provision,
                         actionHandler: (action, object) => {
-                          return new Promise((resolve) => {
+                          return new Promise<void>((resolve) => {
-                            putActionHandler(action, object, false)
+                            putActionHandler(action, object, true)
                               .then((errors) => {
                                 const errorList = errors
                                   ? Object.values(errors)
@@ -238,55 +296,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                };
+                  cb = () => comp;
-                overlay.open(cb);
+                  overlay.open(
-              },
+                    cb,
-            },
+                    () =>
-            "Show"
+                      !comp.state["current"]["modified"] ||
-          ),
+                      confirm("You have unsaved changes. Close anyway?")
-        ];
+                  );
-      };
-
-      if (window.authorizer.hasAccess("provisions", 3)) {
-        attrs["actionsCallback"] = (selected: Set<string>): Children => {
-          return [
-            m(
-              "button.primary",
-              {
-                title: "Create new provision",
-                onclick: () => {
-                  const cb = (): Children => {
-                    return m(
-                      putFormComponent,
-                      Object.assign(
-                        {
-                          actionHandler: (action, object) => {
-                            return new Promise((resolve) => {
-                              putActionHandler(action, object, true)
-                                .then((errors) => {
-                                  const errorList = errors
-                                    ? Object.values(errors)
-                                    : [];
-                                  if (errorList.length) {
-                                    for (const err of errorList)
-                                      notifications.push("error", err);
-                                  } else {
-                                    overlay.close(cb);
-                                  }
-                                  resolve();
-                                })
-                                .catch((err) => {
-                                  notifications.push("error", err.message);
-                                  resolve();
-                                });
-                            });
-                          },
-                        },
-                        formData
-                      )
-                    );
-                  };
-                  overlay.open(cb);
                 },
               },
               "New"
@@ -316,11 +332,11 @@ export const component: ClosureComponent = (): Component => {
                         "success",
                         `${res.length} provisions deleted`
                       );
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     })
                     .catch((err) => {
                       notifications.push("error", err.message);
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     });
                 },
               },
@@ -338,7 +354,11 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing provisions"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs),
+        m(
+          "loading",
+          { queries: [provisions, count] },
+          m(indexTableComponent, attrs)
+        ),
       ];
     },
   };

ui/put-form-component.ts

@@ -20,6 +20,7 @@
 import { VnodeDOM, ClosureComponent, Component, Children } from "mithril";
 import { m } from "./components";
 import codeEditorComponent from "./code-editor-component";
+import { getDatalistId } from "./datalist";
 
 const singular = {
   presets: "preset",
@@ -56,6 +57,7 @@ function createField(current, attr, focus): Children {
           : null,
         onchange: (e) => {
           current.object[attr.id] = e.target.value;
+          current.modified = true;
           e.redraw = false;
         },
       },
@@ -80,6 +82,7 @@ function createField(current, attr, focus): Children {
           if (e.target.checked) currentSelected.add(op);
           else currentSelected.delete(op);
           current.object[attr.id] = Array.from(currentSelected);
+          current.modified = true;
           e.redraw = false;
         },
       };
@@ -98,6 +101,7 @@ function createField(current, attr, focus): Children {
       },
       onChange: (value) => {
         current.object[attr.id] = value;
+        current.modified = true;
       },
     };
     return m(codeEditorComponent, attrs);
@@ -112,6 +116,7 @@ function createField(current, attr, focus): Children {
         : null,
       onchange: (e) => {
         current.object[attr.id] = e.target.files;
+        current.modified = true;
         e.redraw = false;
       },
     });
@@ -132,6 +137,7 @@ function createField(current, attr, focus): Children {
         : null,
       oninput: (e) => {
         current.object[attr.id] = e.target.value;
+        current.modified = true;
         e.redraw = false;
       },
       onkeypress: (e) => {
@@ -146,9 +152,14 @@ function createField(current, attr, focus): Children {
     });
   }
 
+  let datalist: string = null;
+  if (attr.options) datalist = getDatalistId(attr.options);
+
   return m("input", {
     type: attr.type === "password" ? "password" : "text",
     name: attr.id,
+    list: datalist,
+    autocomplete: datalist ? "off" : null,
     disabled: attr.id === "_id" && !current.isNew,
     value: current.object[attr.id],
     oncreate: focus
@@ -158,6 +169,7 @@ function createField(current, attr, focus): Children {
       : null,
     oninput: (e) => {
       current.object[attr.id] = e.target.value;
+      current.modified = true;
       e.redraw = false;
     },
   });
@@ -174,6 +186,7 @@ const component: ClosureComponent = (): Component => {
         vnode.state["current"] = {
           isNew: !base["_id"],
           object: Object.assign({}, base),
+          modified: false,
         };
       }
 

ui/smart-query.ts

@@ -19,6 +19,7 @@
 
 import config from "./config";
 import { Expression } from "../lib/types";
+import { encodeTag } from "../lib/common";
 
 const resources = {
   devices: {},
@@ -138,7 +139,7 @@ function queryMac(param, value): Expression {
 }
 
 function queryTag(tag: string): Expression {
-  const t = tag.replace(/[^a-zA-Z0-9-]+/g, "_");
+  const t = encodeTag(tag);
   return ["IS NOT NULL", ["PARAM", `Tags.${t}`]];
 }
 

ui/store.ts

@@ -76,14 +76,14 @@ for (const r of [
 }
 
 class QueryResponse {
-  public get fulfilled(): boolean {
+  public get fulfilled(): number {
     queries.accessed.set(this, Date.now());
-    return !!queries.fulfilled.get(this);
+    return queries.fulfilled.get(this) || 0;
   }
 
   public get fulfilling(): boolean {
     queries.accessed.set(this, Date.now());
-    return queries.fulfilling.has(this);
+    return !(queries.fulfilled.get(this) >= fulfillTimestamp);
   }
 
   public get value(): any {
@@ -152,6 +152,8 @@ function checkConnection(): void {
         }
       }
     },
+  }).catch((err) => {
+    notifications.push("error", err.message);
   });
 }
 
@@ -169,11 +171,26 @@ export async function xhrRequest(
   ): any => {
     if (typeof extract === "function") return extract(xhr, _options);
 
+    // https://mithril.js.org/request.html#error-handling
+    if (xhr.status !== 304 && Math.floor(xhr.status / 100) !== 2) {
+      if (xhr.status === 403) throw new Error("Not authorized");
+      const err = new Error();
+      err["message"] =
+        xhr.status === 0
+          ? "Server is unreachable"
+          : `Unexpected response status code ${xhr.status}`;
+      err["code"] = xhr.status;
+      err["response"] = xhr.responseText;
+      throw err;
+    }
+
     let response: any;
     if (typeof deserialize === "function") {
       response = deserialize(xhr.responseText);
     } else if (
-      xhr.getResponseHeader("content-type").startsWith("application/json")
+      (xhr.getResponseHeader("content-type") || "").startsWith(
+        "application/json"
+      )
     ) {
       try {
         response = xhr.responseText ? JSON.parse(xhr.responseText) : null;
@@ -184,15 +201,6 @@ export async function xhrRequest(
       response = xhr.responseText;
     }
 
-    // https://mithril.js.org/request.html#error-handling
-    if (xhr.status !== 304 && Math.floor(xhr.status / 100) !== 2) {
-      const err = new Error();
-      err["message"] = xhr.responseText;
-      err["code"] = xhr.status;
-      err["response"] = response;
-      throw err;
-    }
-
     return response;
   };
 
@@ -340,18 +348,7 @@ export function fetch(
   return queryResponse;
 }
 
-export function fulfill(
+export function fulfill(accessTimestamp: number): void {
-  accessTimestamp: number,
-  _fulfillTimestamp: number
-): Promise<boolean> {
-  let updated = false;
-
-  if (_fulfillTimestamp > fulfillTimestamp) {
-    for (const resource of Object.values(resources))
-      resource.combinedFilter = null;
-    fulfillTimestamp = _fulfillTimestamp;
-  }
-
   const allPromises = [];
 
   for (const [resourceType, resource] of Object.entries(resources)) {
@@ -365,28 +362,32 @@ export function fulfill(
 
       if (!(fulfillTimestamp <= queries.fulfilled.get(queryResponse))) {
         queries.fulfilling.add(queryResponse);
+        let filter = queries.filter.get(queryResponse);
+        filter = unpackExpression(filter);
         allPromises.push(
-          new Promise((resolve, reject) => {
+          xhrRequest({
-            updated = true;
+            method: "HEAD",
-            let filter = queries.filter.get(queryResponse);
+            url:
-            filter = unpackExpression(filter);
+              `api/${resourceType}/?` +
-            xhrRequest({
+              m.buildQueryString({
-              method: "HEAD",
+                filter: memoizedStringify(filter),
-              url:
+              }),
-                `api/${resourceType}/?` +
+            extract: (xhr) => {
-                m.buildQueryString({
+              if (xhr.status === 403) throw new Error("Not authorized");
-                  filter: memoizedStringify(filter),
+              if (!xhr.status) {
-                }),
+                throw new Error("Server is unreachable");
-              extract: (xhr) => +xhr.getResponseHeader("x-total-count"),
+              } else if (xhr.status !== 200) {
-              background: true,
+                throw new Error(
-            })
+                  `Unexpected response status code ${xhr.status}`
-              .then((c) => {
+                );
-                queries.value.set(queryResponse, c);
+              }
-                queries.fulfilled.set(queryResponse, fulfillTimestamp);
+              return +xhr.getResponseHeader("x-total-count");
-                queries.fulfilling.delete(queryResponse);
+            },
-                resolve();
+            background: false,
-              })
+          }).then((c) => {
-              .catch((err) => reject(err));
+            queries.value.set(queryResponse, c);
+            queries.fulfilled.set(queryResponse, fulfillTimestamp);
+            queries.fulfilling.delete(queryResponse);
           })
         );
       }
@@ -411,45 +412,39 @@ export function fulfill(
         const limit = queries.limit.get(queryResponse);
         const sort = queries.sort.get(queryResponse);
         if (limit) {
+          let filter = queries.filter.get(queryResponse);
+          filter = unpackExpression(filter);
           allPromises.push(
-            new Promise((resolve, reject) => {
+            xhrRequest({
-              updated = true;
+              method: "GET",
-              let filter = queries.filter.get(queryResponse);
+              url:
-              filter = unpackExpression(filter);
+                `api/${resourceType}/?` +
-              xhrRequest({
+                m.buildQueryString({
-                method: "GET",
+                  filter: memoizedStringify(filter),
-                url:
+                  limit: 1,
-                  `api/${resourceType}/?` +
+                  skip: limit - 1,
-                  m.buildQueryString({
+                  sort: JSON.stringify(sort),
-                    filter: memoizedStringify(filter),
+                  projection: Object.keys(sort).join(","),
-                    limit: 1,
+                }),
-                    skip: limit - 1,
+              background: true,
-                    sort: JSON.stringify(sort),
+            }).then((res) => {
-                    projection: Object.keys(sort).join(","),
+              if ((res as any[]).length) {
-                  }),
+                // Generate bookmark object
-              })
+                const bm = Object.keys(sort).reduce((b, k) => {
-                .then((res) => {
+                  if (res[0][k] != null) {
-                  if ((res as any[]).length) {
+                    if (typeof res[0][k] === "object") {
-                    // Generate bookmark object
+                      if (res[0][k].value != null) b[k] = res[0][k].value[0];
-                    const bm = Object.keys(sort).reduce((b, k) => {
+                    } else {
-                      if (res[0][k] != null) {
+                      b[k] = res[0][k];
-                        if (typeof res[0][k] === "object") {
+                    }
-                          if (res[0][k].value != null)
-                            b[k] = res[0][k].value[0];
-                        } else {
-                          b[k] = res[0][k];
-                        }
-                      }
-
-                      return b;
-                    }, {});
-                    queries.bookmark.set(queryResponse, bm);
-                  } else {
-                    queries.bookmark.delete(queryResponse);
                   }
-                  resolve();
+
-                })
+                  return b;
-                .catch(reject);
+                }, {});
+                queries.bookmark.set(queryResponse, bm);
+              } else {
+                queries.bookmark.delete(queryResponse);
+              }
             })
           );
         }
@@ -457,31 +452,82 @@ export function fulfill(
     }
   }
 
-  return new Promise((resolve, reject) => {
+  Promise.all(allPromises)
-    Promise.all(allPromises)
+    .then(() => {
-      .then(() => {
+      let updated = false;
-        const allPromises2 = [];
+      const allPromises2 = [];
-        for (const [resourceType, toFetch] of Object.entries(toFetchAll)) {
+      for (const [resourceType, toFetch] of Object.entries(toFetchAll)) {
-          let combinedFilter = null;
+        let combinedFilter = null;
 
+        for (const queryResponse of toFetch) {
+          let filter = queries.filter.get(queryResponse);
+          filter = memoizedEvaluate(filter, null, fulfillTimestamp);
+          const bookmark = queries.bookmark.get(queryResponse);
+          const sort = queries.sort.get(queryResponse);
+          if (bookmark) filter = limitFilter(filter, sort, bookmark);
+          combinedFilter = or(combinedFilter, filter);
+        }
+
+        const [union, diff] = unionDiff(
+          resources[resourceType].combinedFilter,
+          combinedFilter
+        );
+
+        if (!diff) {
           for (const queryResponse of toFetch) {
             let filter = queries.filter.get(queryResponse);
             filter = memoizedEvaluate(filter, null, fulfillTimestamp);
+            const limit = queries.limit.get(queryResponse);
             const bookmark = queries.bookmark.get(queryResponse);
             const sort = queries.sort.get(queryResponse);
             if (bookmark) filter = limitFilter(filter, sort, bookmark);
-            combinedFilter = or(combinedFilter, filter);
+
+            queries.value.set(
+              queryResponse,
+              findMatches(resourceType, filter, sort, limit)
+            );
+            queries.fulfilled.set(queryResponse, fulfillTimestamp);
+            queries.fulfilling.delete(queryResponse);
+            updated = true;
           }
+          continue;
+        }
 
-          const [union, diff] = unionDiff(
+        let deleted = new Set<string>();
-            resources[resourceType].combinedFilter,
+        if (!resources[resourceType].combinedFilter)
-            combinedFilter
+          deleted = new Set(resources[resourceType].objects.keys());
-          );
+
+        const combinedFilterDiff = diff;
+        resources[resourceType].combinedFilter = union;
+
+        allPromises2.push(
+          xhrRequest({
+            method: "GET",
+            url:
+              `api/${resourceType}/?` +
+              m.buildQueryString({
+                filter: memoizedStringify(combinedFilterDiff),
+              }),
+            background: false,
+          }).then((res) => {
+            for (const r of res as any[]) {
+              const id =
+                resourceType === "devices"
+                  ? r["DeviceID.ID"].value[0]
+                  : r["_id"];
+              resources[resourceType].objects.set(id, r);
+              deleted.delete(id);
+            }
+
+            for (const d of deleted) {
+              const obj = resources[resourceType].objects.get(d);
+              if (evaluate(combinedFilterDiff, obj, fulfillTimestamp))
+                resources[resourceType].objects.delete(d);
+            }
 
-          if (!diff) {
             for (const queryResponse of toFetch) {
               let filter = queries.filter.get(queryResponse);
-              filter = memoizedEvaluate(filter, null, fulfillTimestamp);
+              filter = unpackExpression(filter);
               const limit = queries.limit.get(queryResponse);
               const bookmark = queries.bookmark.get(queryResponse);
               const sort = queries.sort.get(queryResponse);
@@ -494,76 +540,29 @@ export function fulfill(
               queries.fulfilled.set(queryResponse, fulfillTimestamp);
               queries.fulfilling.delete(queryResponse);
             }
-            continue;
+          })
-          }
+        );
-
+      }
-          const combinedFilterDiff = diff;
+      if (updated) m.redraw();
-          resources[resourceType].combinedFilter = union;
+      return Promise.all(allPromises2);
-
+    })
-          updated = true;
+    .catch((err) => {
-          let deleted = new Set<string>();
+      notifications.push("error", err.message);
-          if (!resources[resourceType].combinedFilter)
+    });
-            deleted = new Set(resources[resourceType].objects.keys());
-
-          allPromises2.push(
-            new Promise((resolve2, reject2) => {
-              xhrRequest({
-                method: "GET",
-                url:
-                  `api/${resourceType}/?` +
-                  m.buildQueryString({
-                    filter: memoizedStringify(combinedFilterDiff),
-                  }),
-              })
-                .then((res) => {
-                  for (const r of res as any[]) {
-                    const id =
-                      resourceType === "devices"
-                        ? r["DeviceID.ID"].value[0]
-                        : r["_id"];
-                    resources[resourceType].objects.set(id, r);
-                    deleted.delete(id);
-                  }
-
-                  for (const d of deleted) {
-                    const obj = resources[resourceType].objects.get(d);
-                    if (evaluate(combinedFilterDiff, obj, fulfillTimestamp))
-                      resources[resourceType].objects.delete(d);
-                  }
-
-                  for (const queryResponse of toFetch) {
-                    let filter = queries.filter.get(queryResponse);
-                    filter = unpackExpression(filter);
-                    const limit = queries.limit.get(queryResponse);
-                    const bookmark = queries.bookmark.get(queryResponse);
-                    const sort = queries.sort.get(queryResponse);
-                    if (bookmark) filter = limitFilter(filter, sort, bookmark);
-
-                    queries.value.set(
-                      queryResponse,
-                      findMatches(resourceType, filter, sort, limit)
-                    );
-                    queries.fulfilled.set(queryResponse, fulfillTimestamp);
-                    queries.fulfilling.delete(queryResponse);
-                  }
-                  resolve2();
-                })
-                .catch(reject2);
-            })
-          );
-        }
-        Promise.all(allPromises2)
-          .then(() => resolve(updated))
-          .catch(reject);
-      })
-      .catch(reject);
-  });
 }
 
 export function getTimestamp(): number {
   return fulfillTimestamp;
 }
 
+export function setTimestamp(t: number): void {
+  if (t > fulfillTimestamp) {
+    fulfillTimestamp = t;
+    for (const resource of Object.values(resources))
+      resource.combinedFilter = null;
+  }
+}
+
 export function postTasks(
   deviceId: string,
   tasks: QueueTask[]
@@ -578,6 +577,8 @@ export function postTasks(
     url: `api/devices/${encodeURIComponent(deviceId)}/tasks`,
     body: tasks,
     extract: (xhr) => {
+      if (xhr.status === 403) throw new Error("Not authorized");
+      if (!xhr.status) throw new Error("Server is unreachable");
       if (xhr.status !== 200) throw new Error(xhr.response);
       const connectionRequestStatus = xhr.getResponseHeader(
         "Connection-Request"
@@ -647,7 +648,13 @@ export function resourceExists(resource: string, id: string): Promise<number> {
       m.buildQueryString({
         filter: memoizedStringify(filter),
       }),
-    extract: (xhr) => +xhr.getResponseHeader("x-total-count"),
+    extract: (xhr) => {
+      if (xhr.status === 403) throw new Error("Not authorized");
+      if (!xhr.status) throw new Error("Server is unreachable");
+      else if (xhr.status !== 200)
+        throw new Error(`Unexpected response status code ${xhr.status}`);
+      return +xhr.getResponseHeader("x-total-count");
+    },
     background: true,
   });
 }

ui/task-queue.ts

@@ -19,23 +19,41 @@
 
 import * as store from "./store";
 import { Task } from "../lib/types";
+import * as notifications from "./notifications";
 
 export interface QueueTask extends Task {
   status?: string;
-  device?: string;
+  device: string;
 }
 
-const queue: Set<QueueTask> = new Set();
+export interface StageTask extends Task {
-const staging: Set<QueueTask> = new Set();
+  devices: string[];
+}
 
-export function queueTask(task: QueueTask): void {
+const MAX_QUEUE = 100;
-  staging.delete(task);
+
-  task.status = "queued";
+const queue: Set<QueueTask> = new Set();
-  queue.add(task);
+const staging: Set<StageTask> = new Set();
+
+function canQueue(tasks: QueueTask[]): boolean {
+  let count = queue.size;
+  for (const task of tasks) if (!queue.has(task)) ++count;
+  return count <= MAX_QUEUE;
+}
+
+export function queueTask(...tasks: QueueTask[]): void {
+  if (!canQueue(tasks)) {
+    notifications.push("error", "Too many tasks in queue");
+    return;
+  }
+
+  for (const task of tasks) {
+    task.status = "queued";
+    queue.add(task);
+  }
 }
 
 export function deleteTask(task: QueueTask): void {
-  staging.delete(task);
   queue.delete(task);
 }
 
@@ -47,7 +65,7 @@ export function clear(): void {
   queue.clear();
 }
 
-export function getStaging(): Set<QueueTask> {
+export function getStaging(): Set<StageTask> {
   return staging;
 }
 
@@ -55,11 +73,19 @@ export function clearStaging(): void {
   staging.clear();
 }
 
-export function stageSpv(task: QueueTask): void {
+export function stageSpv(task: StageTask): void {
+  if (queue.size + task.devices.length > MAX_QUEUE) {
+    notifications.push("error", "Too many tasks in queue");
+    return;
+  }
   staging.add(task);
 }
 
-export function stageDownload(task: QueueTask): void {
+export function stageDownload(task: StageTask): void {
+  if (queue.size + task.devices.length > MAX_QUEUE) {
+    notifications.push("error", "Too many tasks in queue");
+    return;
+  }
   staging.add(task);
 }
 
@@ -69,14 +95,19 @@ export function commit(
     deviceId: string,
     err: Error,
     conReqStatus: string,
-    tasks: QueueTask[]
+    _tasks: QueueTask[]
   ) => void
 ): Promise<void> {
   const devices: { [deviceId: string]: any[] } = {};
+
+  if (!canQueue(tasks))
+    return Promise.reject(new Error("Too many tasks in queue"));
+
   for (const t of tasks) {
     devices[t.device] = devices[t.device] || [];
     devices[t.device].push(t);
-    queueTask(t);
+    t.status = "queued";
+    queue.add(t);
   }
 
   return new Promise((resolve) => {

ui/ui-config-component.ts

@@ -55,7 +55,8 @@ function putActionHandler(prefix: string[], dataYaml: string): Promise<any> {
           for (const f of res) current[f._id] = f.value;
 
           const diff = configFunctions.diffConfig(current, updated);
-          if (!diff.add.length && !diff.remove.length) return void resolve();
+          if (!diff.add.length && !diff.remove.length)
+            return void resolve(null);
 
           const promises = [];
 
@@ -74,7 +75,7 @@ function putActionHandler(prefix: string[], dataYaml: string): Promise<any> {
 
           Promise.all(promises)
             .then(() => {
-              resolve();
+              resolve(null);
             })
             .catch(reject);
         })
@@ -102,12 +103,7 @@ const component: ClosureComponent = (): Component => {
 
       const yamlString =
         config && Object.values(config).length
-          ? yaml.stringify(config, {
+          ? yaml.stringify(config, { schema: "failsafe" })
-              schema: "failsafe",
-              tags: (args) => {
-                return args;
-              },
-            })
           : "";
 
       const attrs = {
@@ -120,6 +116,7 @@ const component: ClosureComponent = (): Component => {
         },
         onChange: (value) => {
           vnode.state["updatedYaml"] = value;
+          vnode.state["modified"] = true;
         },
       };
 

ui/users-page.ts

@@ -85,12 +85,12 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
         .resourceExists("users", id)
         .then((exists) => {
           if (exists && isNew) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ _id: "User already exists" });
           }
 
           if (!exists && !isNew) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ _id: "User does not exist" });
           }
 
@@ -102,14 +102,14 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
                   .changePassword(id, password)
                   .then(() => {
                     notifications.push("success", "User created");
-                    store.fulfill(0, Date.now());
+                    store.setTimestamp(Date.now());
-                    resolve();
+                    resolve(null);
                   })
                   .catch(reject);
               } else {
                 notifications.push("success", "User updated");
-                store.fulfill(0, Date.now());
+                store.setTimestamp(Date.now());
-                resolve();
+                resolve(null);
               }
             })
             .catch(reject);
@@ -120,11 +120,11 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
         .deleteResource("users", object["_id"])
         .then(() => {
           notifications.push("success", "User deleted");
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
-          resolve();
+          resolve(null);
         })
         .catch((err) => {
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
           reject(err);
         });
     } else {
@@ -237,46 +237,46 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
+                let cb: () => Children = null;
-                  const children = [
+                const comp = m(
-                    m(
+                  putFormComponent,
-                      putFormComponent,
+                  Object.assign(
-                      Object.assign(
+                    {
-                        {
+                      base: {
-                          base: {
+                        _id: user._id,
-                            _id: user._id,
+                        roles: user.roles.split(","),
-                            roles: user.roles.split(","),
+                      },
-                          },
+                      actionHandler: (action, object) => {
-                          actionHandler: (action, object) => {
+                        return new Promise<void>((resolve) => {
-                            return new Promise((resolve) => {
+                          putActionHandler(action, object, false)
-                              putActionHandler(action, object, false)
+                            .then((errors) => {
-                                .then((errors) => {
+                              const errorList = errors
-                                  const errorList = errors
+                                ? Object.values(errors)
-                                    ? Object.values(errors)
+                                : [];
-                                    : [];
+                              if (errorList.length) {
-                                  if (errorList.length) {
+                                for (const err of errorList)
-                                    for (const err of errorList)
+                                  notifications.push("error", err);
-                                      notifications.push("error", err);
+                              } else {
-                                  } else {
+                                overlay.close(cb);
-                                    overlay.close(cb);
+                              }
-                                  }
+                              resolve();
-                                  resolve();
+                            })
-                                })
+                            .catch((err) => {
-                                .catch((err) => {
+                              notifications.push("error", err.message);
-                                  notifications.push("error", err.message);
+                              resolve();
-                                  resolve();
-                                });
                             });
-                          },
+                        });
-                        },
+                      },
-                        {
+                    },
-                          resource: "users",
+                    {
-                          attributes: attributes,
+                      resource: "users",
-                        }
+                      attributes: attributes,
-                      )
+                    }
-                    ),
+                  )
-                  ];
+                );
 
+                cb = () => {
+                  const children: Children = [comp];
                   if (canWrite) {
                     children.push(m("hr"));
                     const _attrs = {
@@ -292,7 +292,13 @@ export const component: ClosureComponent = (): Component => {
 
                   return children;
                 };
-                overlay.open(cb);
+
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
               },
             },
             "Show"
@@ -317,38 +323,43 @@ export const component: ClosureComponent = (): Component => {
               {
                 title: "Create new user",
                 onclick: () => {
-                  const cb = (): Children => {
+                  let cb: () => Children = null;
-                    return m(
+                  const comp = m(
-                      putFormComponent,
+                    putFormComponent,
-                      Object.assign(
+                    Object.assign(
-                        {
+                      {
-                          actionHandler: (action, object) => {
+                        actionHandler: (action, object) => {
-                            return new Promise((resolve) => {
+                          return new Promise<void>((resolve) => {
-                              putActionHandler(action, object, true)
+                            putActionHandler(action, object, true)
-                                .then((errors) => {
+                              .then((errors) => {
-                                  const errorList = errors
+                                const errorList = errors
-                                    ? Object.values(errors)
+                                  ? Object.values(errors)
-                                    : [];
+                                  : [];
-                                  if (errorList.length) {
+                                if (errorList.length) {
-                                    for (const err of errorList)
+                                  for (const err of errorList)
-                                      notifications.push("error", err);
+                                    notifications.push("error", err);
-                                  } else {
+                                } else {
-                                    overlay.close(cb);
+                                  overlay.close(cb);
-                                  }
+                                }
-                                  resolve();
+                                resolve();
-                                })
+                              })
-                                .catch((err) => {
+                              .catch((err) => {
-                                  notifications.push("error", err.message);
+                                notifications.push("error", err.message);
-                                  resolve();
+                                resolve();
-                                });
+                              });
-                            });
+                          });
-                          },
                         },
-                        formData
+                      },
-                      )
+                      formData
-                    );
+                    )
-                  };
+                  );
-                  overlay.open(cb);
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"
@@ -376,11 +387,11 @@ export const component: ClosureComponent = (): Component => {
                         "success",
                         `${res.length} users deleted`
                       );
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     })
                     .catch((err) => {
                       notifications.push("error", err.message);
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     });
                 },
               },
@@ -398,7 +409,11 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing users"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs),
+        m(
+          "loading",
+          { queries: [users, count] },
+          m(indexTableComponent, attrs)
+        ),
       ];
     },
   };

ui/virtual-parameters-page.ts

@@ -66,12 +66,12 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
         .resourceExists("virtualParameters", id)
         .then((exists) => {
           if (exists && isNew) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ _id: "Virtual parameter already exists" });
           }
 
           if (!exists && !isNew) {
-            store.fulfill(0, Date.now());
+            store.setTimestamp(Date.now());
             return void resolve({ _id: "Virtual parameter does not exist" });
           }
 
@@ -82,10 +82,16 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
                 "success",
                 `Virtual parameter ${exists ? "updated" : "created"}`
               );
-              store.fulfill(0, Date.now());
+              store.setTimestamp(Date.now());
-              resolve();
+              resolve(null);
             })
-            .catch(reject);
+            .catch((err) => {
+              if (err["code"] === 400 && err["response"]) {
+                reject(new Error(err["response"]));
+                return;
+              }
+              reject(err);
+            });
         })
         .catch(reject);
     } else if (action === "delete") {
@@ -93,11 +99,11 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
         .deleteResource("virtualParameters", object["_id"])
         .then(() => {
           notifications.push("success", "Virtual parameter deleted");
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
-          resolve();
+          resolve(null);
         })
         .catch((err) => {
-          store.fulfill(0, Date.now());
+          store.setTimestamp(Date.now());
           reject(err);
         });
     } else {
@@ -207,15 +213,67 @@ export const component: ClosureComponent = (): Component => {
             "a",
             {
               onclick: () => {
-                const cb = (): Children => {
+                let cb: () => Children = null;
-                  return m(
+                const comp = m(
+                  putFormComponent,
+                  Object.assign(
+                    {
+                      base: virtualParameter,
+                      actionHandler: (action, object) => {
+                        return new Promise<void>((resolve) => {
+                          putActionHandler(action, object, false)
+                            .then((errors) => {
+                              const errorList = errors
+                                ? Object.values(errors)
+                                : [];
+                              if (errorList.length) {
+                                for (const err of errorList)
+                                  notifications.push("error", err);
+                              } else {
+                                overlay.close(cb);
+                              }
+                              resolve();
+                            })
+                            .catch((err) => {
+                              notifications.push("error", err.message);
+                              resolve();
+                            });
+                        });
+                      },
+                    },
+                    formData
+                  )
+                );
+                cb = () => comp;
+                overlay.open(
+                  cb,
+                  () =>
+                    !comp.state["current"]["modified"] ||
+                    confirm("You have unsaved changes. Close anyway?")
+                );
+              },
+            },
+            "Show"
+          ),
+        ];
+      };
+
+      if (window.authorizer.hasAccess("virtualParameters", 3)) {
+        attrs["actionsCallback"] = (selected: Set<string>): Children => {
+          return [
+            m(
+              "button.primary",
+              {
+                title: "Create new virtual parameter",
+                onclick: () => {
+                  let cb: () => Children = null;
+                  const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
-                        base: virtualParameter,
                         actionHandler: (action, object) => {
-                          return new Promise((resolve) => {
+                          return new Promise<void>((resolve) => {
-                            putActionHandler(action, object, false)
+                            putActionHandler(action, object, true)
                               .then((errors) => {
                                 const errorList = errors
                                   ? Object.values(errors)
@@ -238,55 +296,13 @@ export const component: ClosureComponent = (): Component => {
                       formData
                     )
                   );
-                };
+                  cb = () => comp;
-                overlay.open(cb);
+                  overlay.open(
-              },
+                    cb,
-            },
+                    () =>
-            "Show"
+                      !comp.state["current"]["modified"] ||
-          ),
+                      confirm("You have unsaved changes. Close anyway?")
-        ];
+                  );
-      };
-
-      if (window.authorizer.hasAccess("virtualParameters", 3)) {
-        attrs["actionsCallback"] = (selected: Set<string>): Children => {
-          return [
-            m(
-              "button.primary",
-              {
-                title: "Create new virtual parameter",
-                onclick: () => {
-                  const cb = (): Children => {
-                    return m(
-                      putFormComponent,
-                      Object.assign(
-                        {
-                          actionHandler: (action, object) => {
-                            return new Promise((resolve) => {
-                              putActionHandler(action, object, true)
-                                .then((errors) => {
-                                  const errorList = errors
-                                    ? Object.values(errors)
-                                    : [];
-                                  if (errorList.length) {
-                                    for (const err of errorList)
-                                      notifications.push("error", err);
-                                  } else {
-                                    overlay.close(cb);
-                                  }
-                                  resolve();
-                                })
-                                .catch((err) => {
-                                  notifications.push("error", err.message);
-                                  resolve();
-                                });
-                            });
-                          },
-                        },
-                        formData
-                      )
-                    );
-                  };
-                  overlay.open(cb);
                 },
               },
               "New"
@@ -316,11 +332,11 @@ export const component: ClosureComponent = (): Component => {
                         "success",
                         `${res.length} virtual parameters deleted`
                       );
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     })
                     .catch((err) => {
                       notifications.push("error", err.message);
-                      store.fulfill(0, Date.now());
+                      store.setTimestamp(Date.now());
                     });
                 },
               },
@@ -338,7 +354,11 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing virtual parameters"),
         m(filterComponent, filterAttrs),
-        m(indexTableComponent, attrs),
+        m(
+          "loading",
+          { queries: [virtualParameters, count] },
+          m(indexTableComponent, attrs)
+        ),
       ];
     },
   };

ui/wizard-page.ts

@@ -87,16 +87,24 @@ export const component: ClosureComponent = (vnode): Component => {
               })
                 .then(() => {
                   setTimeout(() => {
-                    m.request({ url: "init" }).then((o) => {
+                    m.request({ url: "init" })
-                      e.target.disabled = false;
+                      .then((o) => {
-                      options = o;
+                        e.target.disabled = false;
-                      notifications.push("success", "Initialization complete", {
+                        options = o;
-                        "Open Sesame!": () => {
+                        notifications.push(
-                          m.route.set("/login");
+                          "success",
-                          window.location.reload();
+                          "Initialization complete",
-                        },
+                          {
+                            "Open Sesame!": () => {
+                              m.route.set("/login");
+                              window.location.reload();
+                            },
+                          }
+                        );
+                      })
+                      .catch((err) => {
+                        notifications.push("error", err.message);
                       });
-                    });
                   }, 3000);
                   if (opts["users"]) {
                     alert(