Fix XMPP stream not getting closed properly

The XMPP server may not flag the bind feature as required but we're
still required to bind resource to the stream in order to be able to
send IQ stanzas to other clients.

.eslintignore

@@ -0,0 +1,2 @@
+!/bin/.*.ts
+!/tools/.*.ts

.eslintrc.json

@@ -8,29 +8,19 @@
     "project": "./tsconfig.json"
   },
   "plugins": ["@typescript-eslint"],
-  "extends": [
-    "eslint:recommended",
-    "plugin:@typescript-eslint/recommended",
-    "prettier"
-  ],
+  "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended", "prettier", "prettier/@typescript-eslint"],
   "rules": {
     "no-shadow": ["error", { "allow": ["err"] }],
     "handle-callback-err": "error",
     "prefer-arrow-callback": "error",
     "curly": ["error", "multi-or-nest", "consistent"],
     "no-buffer-constructor": "error",
-    "prefer-const": ["error", { "destructuring": "all" }],
-    "eqeqeq": ["error", "always", { "null": "ignore" }],
+    "prefer-const": ["error", {"destructuring": "all"}],
+    "eqeqeq": ["error", "always", {"null": "ignore"}],
     "no-return-await": "error",
     "@typescript-eslint/no-explicit-any": 0,
-    "@typescript-eslint/no-use-before-define": [
-      "error",
-      { "functions": false }
-    ],
-    "@typescript-eslint/explicit-function-return-type": [
-      "error",
-      { "allowExpressions": true }
-    ],
+    "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
+    "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
     "@typescript-eslint/no-floating-promises": "error",
     "require-atomic-updates": 0,
     "no-prototype-builtins": 0

.gitignore

@@ -1,4 +1,6 @@
 *~
 node_modules
 dist
+bin/.*.ts
+tools/.*.ts
 docs/_build

.npmignore

@@ -0,0 +1,25 @@
+*~
+.npmignore
+
+config/config.json
+config/cwmp.key
+config/cwmp.crt
+config/cwmp.ca-bundle
+config/nbi.key
+config/nbi.crt
+config/nbi.ca-bundle
+config/fs.key
+config/fs.crt
+config/fs.ca-bundle
+config/ext
+
+debug/*
+!debug/.gitkeep
+
+dist
+.eslintrc.json
+.eslintignore
+ava.config.js
+tsconfig.json
+bin/.*
+tools/.*

.prettierignore

@@ -1,2 +0,0 @@
-dist
-npm-shrinkwrap.json

CHANGELOG.md

@@ -1,114 +1,20 @@
 # Change Log
 
-## 1.2.8 (2021-10-27)
-
-- Fix a remote code execution security vulnerability in genieacs-ui.
-- All UI components can now be configured using fully dynamic expressions.
-  Previously some component types only accept fixed string values as properties.
-- The `container` UI component can now be configured with an `element` property
-  that's either a string or a nested pair of `tag` and `attributes` properties.
-  The various attributes under the `attributes` property can now make use of a
-  new function `ENCODEURICOMPONENT()` to facilitate creating custom hyperlinks
-  in the UI.
-- Improve sorting buttons' behavior in the device listing page and other pages.
-  Sorting by multiple columns should now feel more intuitive.
-- Support the modulo (%) operator in expressions.
-- Fix a regression in the previous release where the config option 'ui.pageSize'
-  no longer works.
-- Fix process crash when a CPE sends an unsupported ACS method.
-
-## 1.2.7 (2021-09-18)
-
-- Fix regression causing frequent invalid session errors.
-- Fix regression breaking digest authentication in connection requests.
-
-## 1.2.6 (2021-09-16)
-
-- New config option `cwmp.downloadSuccessOnTimeout` to enable a workaround for
-  CPEs that neglect to send a TransferComplete message after a successful
-  download.
-- Display a progress bar when uploading new files.
-- Default to dual stack interface binding (i.e. `::` instead of `0.0.0.0`).
-  Unless the binding interface is explicitly set, this will cause IPv4 addresses
-  in the logs to be displayed as IPv4-mapped IPv6 addresses (e.g.
-  `::ffff:127.0.0.1` instead of `127.0.0.1`).
-- Detect and correct for client side (browser) clock skew that would otherwise
-  alter the numbers in the pie charts.
-- Various improvements relating to dealing with buggy TR-069 client
-  implementations.
-- Fix a bug causing "lock expired" exceptions when a CWMP session remains open
-  for a very long time due to slow clients.
-- Fix metadata of uploaded files going missing due to nginx stripping away what
-  it considers to be invalid headers. The nginx directive
-  `ignore_invalid_headers` is no longer required.
-- Fix crash when a CPE is assigned a tag containing a dot.
-- Fix bug preventing the user from closing the preset pop-up after being
-  presented with unsupported preset message.
-- Fix exceptions raised from ext scripts manifesting as timeout faults.
-- Fix download getting triggered repeatedly when the value passed to `Download`
-  parameter is greater than the current timestamp.
-- Fix crash when passing invalid attributes to `declare`.
-- Fix crash in NBI when pushing tasks to non-existing devices.
-- Fix crash in NBI when passing invalid JSON to various API endpoints.
-- Fix crash when the output from `ping` command cannot be parsed in some rare
-  edge cases.
-- A number of other fixes and stability improvements.
-
-## 1.2.5 (2021-03-12)
-
-- Support specifying custom types when uploading files.
-- Fix JS compatibility issue with Safari browser.
-
-## 1.2.4 (2021-02-24)
-
-- The data model state of a CPE is no longer forgotten after unsuccessful
-  session termination (e.g. timeout). This addresses a number of undesired side
-  effects that arise when a CPE does not terminate the session properly.
-- Executing tasks that take a long time to complete (e.g. refreshing the entire
-  data model) no longer shows a timeout error while the task is still being
-  processed.
-- New function `ROUND()` available to expressions. It works similar to the
-  function by the same name in SQLite and PostgreSQL.
-- Log access events for `genieacs-nbi` service.
-- Pipe stdout/stderr from extension scripts to the `genieacs-cwmp` process log.
-- Parameter values of type `xsd:dateTime` are now displayed in the UI and CSV
-  downloads as a date string rather than a numeric value.
-- Add file download link in the files listing page.
-- Display spinner loading animation throughout the UI.
-- Display GenieACS version and build number underneath the logo.
-- New option to specify how many parameters are displayed at a time in the
-  all-parameters component. Simply set `limit` property in the component config.
-- Reduce overly strong Brotli compression level which was causing significant
-  page load slowdown when Brotli is used.
-- Retire dump-data-model tool. `genieacs-sim` can now use a CSV file as its data
-  model.
-- Reduce the number of concurrent database connections from each process.
-- Remove dependency on 'hostInfo' MongoDB command which is a privileged action.
-  It is now possible to use shared MongoDB instances with limited privileged.
-- Fix bug in NBI where querying files returns 404 error.
-- Fix ping not working for devices with an IPv6 address.
-- Fix an elusive memory leak in `genieacs-fs` that slowly eats up memory and can
-  go unnoticed for long periods of time.
-- Fix a rare edge case where a `declare()` call to set a parameter value may not
-  work as intended if the parameter was originally received as part of the
-  Inform message.
-- A number of other fixes and stability improvements.
-
 ## 1.2.3 (2020-10-26)
 
 - New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
-  report writable parameters as non-writable. When set to true, the scripts will
-  no longer respect the 'writable' attribute of the CPE parameters and will send
-  a SetParamteerValues, AddObject, or DeleteObject request anyway.
+report writable parameters as non-writable. When set to true, the scripts will
+no longer respect the 'writable' attribute of the CPE parameters and will send
+a SetParamteerValues, AddObject, or DeleteObject request anyway.
 - Tags no longer restrict what characters are allowed. Any character other than
-  alphanumeric characters, hyphen, or underscore is now encoded in the data
-  model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
+alphanumeric characters, hyphen, or underscore is now encoded in the data
+model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
 - Ask for a confirmation before closing a pop-up dialog with unsaved changes.
 - Better XML validation to avoid crashes caused by invalid CPE requests.
 - Fix confusing 404 error message when the user attempts to modify a resource
-  when they don't have the necessary permissions.
+when they don't have the necessary permissions.
 - Fix a rare issue where genieacs-cwmp stops accepting new connections after
-  running for a few weeks.
+running for a few weeks.
 - Fix exception when IS NULL operator is used in certain situations.
 
 ## 1.2.2 (2020-10-03)
@@ -118,80 +24,81 @@
 - Fix exception that can happen and persist after a Download request.
 - Fix validation bug preventing running refreshObject task on data model root.
 - Fix invalid arguments fault in refresh preset configuration when upgrading
-  from v1.1.
+from v1.1.
 
 ## 1.2.1 (2020-09-08)
 
 - Fix bug causing faults to not be displayed in the UI.
 - Fix bug where deleting objects does not get reflected immediately in the UI.
 - Improve conversion between filters written in the expression format and
-  MongoDB queries. There should now be fewer edge cases where the two are not
-  equisatisfiable.
+MongoDB queries. There should now be fewer edge cases where the two are not
+equisatisfiable.
 
 ## 1.2.0 (2020-09-01)
 
 - Support GetParameterAttributes and SetParameterAttributes TR-069 methods.
 - Support CASE statement and COALESCE function in expressions.
 - Provision arguments can now be a list of expressions that are dynamically
-  evaluated.
+evaluated.
 - Support Forwarded HTTP header to display in the logs the correct IP of CPEs
-  behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
+behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
 - Config expressions can now access all available device parameters, not only
-  serial number, product class, and OUI.
-- Use relative URLs throughout the UI to allow serving from a subdirectory using
-  a reverse proxy.
+serial number, product class, and OUI.
+- Use relative URLs throughout the UI to allow serving from a subdirectory
+using a reverse proxy.
 - Make Date.parse() and Date.UTC() available to provision scripts.
 - libxmljs has been entirely removed in favor of our bespoke XML parser.
-- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now used
-  to determine the TCP connection timeout.
-- The all-parameters component now limits the number of parameters displayed for
-  better performance.
+- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now
+used to determine the TCP connection timeout.
+- The all-parameters component now limits the number of parameters displayed
+for better performance.
 - The process genieacs-cwmp is now much less likely to throw exceptions as a
-  result of invalid requests from CPE.
+result of invalid requests from CPE.
 - A large number of bug fixes and stability improvements.
 
 ## 1.2.0-beta.0 (2019-07-30)
 
 - A brand new UI superseding genieacs-gui.
 - New initialization wizard on first run.
-- New expression/query language used in search filters and preset preconditions.
+- New expression/query language used in search filters and preset
+preconditions.
 - CPE -> ACS authentication is now supported.
 - New config option (CWMP_KEEP_ALIVE_TIMEOUT) to specify how long to wait for a
-  reply from the CPE before closing the TCP connection.
+reply from the CPE before closing the TCP connection.
 - Debug logging has been reimplemented utilizing YAML format for logs.
 - Handle 9005 faults (Invalid Parameter Name) gracefully by attempting to
-  rediscover the path of the missing parameter recursively.
+rediscover the path of the missing parameter recursively.
 - declare() statements not followed by an explicit commit() are now deferred
-  until all currently active scripts have been executed.
+until all currently active scripts have been executed.
 - FS_HOSTNAME now defaults to the server's hostname or IP.
 - The API now validates the structure of task objects before saving.
 - New XML parser implementation for better performance. You can revert to the
-  old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
-  v10.
+old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
+v10.
 - Performance optimizations. While performance has improved for the majority of
-  use cases, there may be situations where performance has degraded. It's
-  recommended to revisit your hardware requirements.
+use cases, there may be situations where performance has degraded. It's
+recommended to revisit your hardware requirements.
 - Connection request authentication no longer uses 'auth.js' file. Instead, the
-  connection request authentication behavior can now be customized using an
-  'expression'.
+connection request authentication behavior can now be customized using an
+'expression'.
 - The config file (config.json) has been deprecated. System configuration (e.g.
-  listen ports, worker count) are now recommended to be passed as environment.
-  variables. Other general configuration options are stored in the database so
-  as to not require service restart for changes to take effect.
+listen ports, worker count) are now recommended to be passed as environment.
+variables. Other general configuration options are stored in the database so as
+to not require service restart for changes to take effect.
 - Optional redis dependency has been removed completely.
 - Tags now allow only alphanumeric characters and underscore.
-- Supported versions of Node.js and MongoDB are 10.x and up and 2.6 and up
-  respectively.
+- Supported versions of NodeJs and MongoDB are 10.x and up and 2.6 and up
+respectively.
 
 ## 1.1.3 (2018-10-23)
 
 - New config option (MAX_COMMIT_ITERATIONS) to avoid max commit iterations
-  faults for more complex scripts.
+faults for more complex scripts.
 - Support base64 and hexBinary parameter types.
 - Strict parsing of number values in queries (e.g. "123abc" no longer accepted
-  as 123).
+as 123).
 - Mixing $ne and $not operators is not allowed. Now it throws an error instead
-  of returning incorrect results.
+of returning incorrect results.
 - When a task expires, any associated fault is also deleted.
 - API now accepts 'timeout' argument when posting a task.
 - A number of stability fixes.
@@ -201,68 +108,66 @@
 - A large number of bug fixes as well as stability and performance improvements.
 - Three security vulnerabilities disclosed by Maximilian Hils have been patched.
 - New config option UDP_CONNECTION_REQUEST_PORT to specify binding port for UDP
-  connection requests.
+connection requests.
 - New config option DATETIME_MILLISECONDS to strip milliseconds from dateTime
-  values.
+values.
 - New config option BOOLEAN_LITERAL to use 1/0 or true/false for boolean values.
 - Parameter values that cannot be parsed according to the reported type now show
-  a warning message.
+a warning message.
 - Virtual parameter scripts now use the variable 'args' instead of the special
-  'TIMESTAMPS' and 'VALUES' variables. The content of the args array is:
-  {declare timestamps}, {declare values}, {current timestamps}, {current
-  values}.
+'TIMESTAMPS' and 'VALUES' variables. The content of the args array is: {declare
+timestamps}, {declare values}, {current timestamps}, {current values}.
 - Virtual parameter value types are now inferred from the JavaScript type if the
-  returned value attribute is not a value-type pair.
+returned value attribute is not a value-type pair.
 - Show a fault when a virtual parameter script doesn't return the required
-  attributes.
+attributes.
 - Redis is now optional (and disabled by default), reducing the complexity of
-  scalable deployments.
+scalable deployments.
 - Better detection of cyclical presets resulting in fewer faults for complex
-  provisioning scripts.
+provisioning scripts.
 - Math.random() is now deterministic on per-device basis. A function has been
-  added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
+added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
 - Overload spikes are now handled gracefully by refusing to accept new sessions
-  temporarily when under abnormal load.
+temporarily when under abnormal load.
 - Added log messages for session timeouts, connection drops, and XML parsing
-  errors.
+errors.
 - Date.now() now takes an optional argument to specify "time steps" (in
-  milliseconds). This can be used to ensure a group of parameters are all
-  refreshed at the same time intervals.
+milliseconds). This can be used to ensure a group of parameters are all
+refreshed at the same time intervals.
 - Only the non-default configuration options are now logged at process start.
 - Faults caused by errors from extensions now show a cleaner stack trace.
 - Exit main process if there are too many worker crashes (e.g. when DB is down).
 - Updated dependencies and included a lockfile to ensure installations get the
-  exact dependencies it was tested against.
+exact dependencies it was tested against.
 
 ## 1.1.1 (2017-03-23)
 
 - Avoid crashing when connection request credentials are missing.
 - Show a warning instead of crashing when failing to parse parameter values
-  according to the expected value type.
+according to the expected value type.
 - Add missing "Registered" event.
 - Fix bug where in certain cases many more instances than declared are created.
 - Fix parameter discovery bug when declared path timestamp is 1 or is not set.
 - Fix preset precondition failing when testing against datetime parameters and
-  certain other parameters like \_deviceId.\_ProductClass.
+certain other parameters like \_deviceId.\_ProductClass.
 
 ## 1.1.0 (2017-03-10)
 
 - Provisions enable implementing dynamic device configuration or complex device
-  provisioning work flow using arbitrary scripts.
+provisioning work flow using arbitrary scripts.
 - Virtual parameters are user-defined parameters whose values are evaluated from
-  a custom script.
+a custom script.
 - Extensions are sandboxed Node.js scripts that are accessible from provision
-  and virtual parameter scripts to facilitate integration with external
-  entities.
-- Support for UDP/STUN based connection requests for reaching devices behind NAT
-  (TR-069 Annex G).
+and virtual parameter scripts to facilitate integration with external entities.
+- Support for UDP/STUN based connection requests for reaching devices behind
+NAT (TR-069 Annex G).
 - Presets can now be scheduled using a cron-like expression.
 - Presets can now be tied to specific device events (e.g. boot).
-- Presets precondition queries no longer support "\$or" or other MongoDB logical
-  operators.
+- Presets precondition queries no longer support "$or" or other MongoDB logical
+operators.
 - Faults are no longer a part of tasks but are now first class objects.
 - Presets are now assigned to channels. A fault in one channel only blocks
-  presets in that channel.
+presets in that channel.
 - New API CRUD functions for provisions, virtual parameters, and faults.
 - New config options for XML output.
 - API responses now include "GenieACS-Version" header.
@@ -271,19 +176,19 @@
 - Supported Node.js versions are 6.x and 7.x.
 - Supported MongoDB versions are 2.6 through 3.4.
 - Expect performance differences due to major under the hood changes. Some
-  operations are faster and some are slower. Overall performance is improved.
+operations are faster and some are slower. Overall performance is improved.
 - GenieACS will no longer fetch the entire device data model upon first contact
-  but will instead only fetch the parameters it needs to fulfill the presets.
+but will instead only fetch the parameters it needs to fulfill the presets.
 - Logs have been overhauled and split into two streams: process log (stderr) and
-  access log (stdout). Also added config options to dump logs to files rather
-  than standard streams.
+access log (stdout). Also added config options to dump logs to files rather than
+standard streams.
 - Connection request authentication credentials are picked up from the device
-  data model if available. config/auth.js is still supported as a fallback and
-  now supports an optional callback argument.
+data model if available. config/auth.js is still supported as a fallback and now
+supports an optional callback argument.
 - Custom commands have been removed. Use virtual parameters and/or extensions.
 - Aliases and value normalizers (config/parameters.json) have been removed. Use
-  virtual parameters.
+virtual parameters.
 - The API /devices/<device_id>/preset has been removed.
 - Rarely used RequestDownload method no longer supported.
 - The TR-069 client simulator has moved to its own repo at
-  https://github.com/zaidka/genieacs-sim
+https://github.com/zaidka/genieacs-sim

CONTRIBUTING.md

@@ -5,8 +5,8 @@ tracker is strictly for bug reports.
 
 # Issues
 
-We use GitHub issues to track bugs. Please ensure your bug description is clear
-and has sufficient instructions to be able to reproduce the issue.
+We use GitHub issues to track bugs. Please ensure your bug description is
+clear and has sufficient instructions to be able to reproduce the issue.
 
 If you face interoperability issues with your CPE, more often than not it's a
 device related issue. Please consult the [forum] before submitting an Issue.
@@ -26,14 +26,13 @@ By submitting a pull request to this repository, you acknowledge that, while
 maintaining copyright, you grant Zaid Abdulla a perpetual, worldwide, non-
 exclusive, no-charge, royalty-free, irrevocable license to reproduce, prepare
 derivative works of, publicly display, publicly perform, sublicense, and
-distribute your contributions and such derivative works under the AGPLv3 license
-or any other license terms, including, but not limited to, proprietary or
-commercial license terms.
+distribute your contributions and such derivative works under the AGPLv3
+license or any other license terms, including, but not limited to, proprietary
+or commercial license terms.
 
-You confirm that you own or have rights to distribute and sublicense the source
-code contained therein, and that your content does not infringe upon the
-intellectual property rights of a third party.
+You confirm that you own or have rights to distribute and sublicense the
+source code contained therein, and that your content does not infringe upon
+the intellectual property rights of a third party.
 
-[commit-guidelines]:
-  http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
+[commit-guidelines]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
 [forum]: https://forum.genieacs.com

README.md

@@ -17,10 +17,10 @@ supported versions are:
 
 Install GenieACS from NPM:
 
-    sudo npm install -g genieacs
+    sudo npm install -g --unsafe-perm genieacs
 
 To build from source instead, clone this repo or download the source archive
-then _cd_ into the source directory then run:
+then *cd* into the source directory then run:
 
     npm install
     npm run build
@@ -30,8 +30,8 @@ source):
 
 ### genieacs-cwmp
 
-This is the service that the CPEs will communicate with. It listens on port 7547
-by default. Configure the ACS URL in your devices accordingly.
+This is the service that the CPEs will communicate with. It listens on port
+7547 by default. Configure the ACS URL in your devices accordingly.
 
 You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
 as a dummy TR-069 simulator if you don't have a CPE at hand.
@@ -50,7 +50,7 @@ such. It listens on port 7567 by default.
 ### genieacs-ui
 
 This serves the web based user interface. It listens on port 3000 by default.
-You must pass _--ui-jwt-secret_ argument to supply the secret key used for
+You must pass *--ui-jwt-secret* argument to supply the secret key used for
 signing browser cookies:
 
     genieacs-ui --ui-jwt-secret secret
@@ -64,13 +64,14 @@ a complete installation guide for production deployments.
 
 ## Support
 
-The [forum](https://forum.genieacs.com) is a good place to get guidance and help
-from the community. Head on over and join the conversation!
+The [forum](https://forum.genieacs.com) is a good place to get guidance and
+help from the community. Head on over and join the conversation!
 
 For commercial support options, please visit
 [genieacs.com](https://genieacs.com/support/).
 
 ## License
 
-Copyright 2013-2021 GenieACS Inc. GenieACS is released under the
-[AGPLv3 license terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).
+Copyright 2013-2020 GenieACS Inc. GenieACS is released under the [AGPLv3
+license
+terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

ava.config.js

@@ -1,4 +1,5 @@
 export default {
-  extensions: ["ts"],
-  require: ["esm", "ts-node/register/transpile-only"],
+  "extensions": ["ts"],
+  "require": ["esm", "ts-node/register/transpile-only"]
 };
+

bin/genieacs-fs.ts

@@ -94,7 +94,7 @@ if (!cluster.worker) {
 
   const _listener = (req, res): void => {
     if (stopping) res.setHeader("Connection", "close");
-    void listener(req, res);
+    listener(req, res);
   };
 
   const initPromise = db

build/build.ts

@@ -25,11 +25,12 @@ import typescript from "@rollup/plugin-typescript";
 import { terser } from "rollup-plugin-terser";
 import nodeResolve from "@rollup/plugin-node-resolve";
 import commonjs from "@rollup/plugin-commonjs";
+import webpack, { Configuration } from "webpack";
 import postcss from "postcss";
 import postcssImport from "postcss-import";
 import postcssPresetEnv from "postcss-preset-env";
 import cssnano from "cssnano";
-import { optimize } from "svgo";
+import SVGO from "svgo";
 import * as xmlParser from "../lib/xml-parser";
 
 const MODE = process.env["NODE_ENV"] || "production";
@@ -42,7 +43,7 @@ const BUILD_METADATA = new Date()
 const INPUT_DIR = path.resolve(__dirname, "..");
 const OUTPUT_DIR = path.resolve(__dirname, "../dist");
 
-const builtins = [
+const externals = [
   "path",
   "fs",
   "cluster",
@@ -50,16 +51,38 @@ const builtins = [
   "tls",
   "http",
   "https",
+  "net",
+  "events",
   "zlib",
   "crypto",
   "util",
+  "mongodb",
   "vm",
+  "later",
+  "parsimmon",
+  "seedrandom",
   "querystring",
   "child_process",
   "dgram",
   "url",
-  "readline",
+  "iconv-lite",
+  "koa",
+  "koa-router",
+  "koa-compress",
+  "koa-bodyparser",
+  "koa-jwt",
+  "koa-static",
+  "jsonwebtoken",
   "stream",
+  "mithril",
+  "parsimmon",
+  "yaml",
+  "codemirror",
+  "codemirror/mode/javascript/javascript",
+  "codemirror/mode/yaml/yaml",
+  "ipaddr.js",
+  "jsbi",
+  "espresso-iisojs",
 ];
 
 function rmDirSync(dirPath): void {
@@ -74,7 +97,6 @@ function rmDirSync(dirPath): void {
   fs.rmdirSync(dirPath);
 }
 
-// For lockfileVersion = 1
 function stripDevDeps(deps): void {
   if (!deps["dependencies"]) return;
   for (const [k, v] of Object.entries(deps["dependencies"])) {
@@ -84,15 +106,6 @@ function stripDevDeps(deps): void {
   if (!Object.keys(deps["dependencies"]).length) delete deps["dependencies"];
 }
 
-// For lockfileVersion = 2
-function stripDevDeps2(deps): void {
-  if (!deps["packages"]) return;
-  for (const [k, v] of Object.entries(deps["packages"])) {
-    delete v["devDependencies"];
-    if (v["dev"]) delete deps["packages"][k];
-  }
-}
-
 function xmlTostring(xml): string {
   const children = [];
   for (const c of xml.children || []) children.push(xmlTostring(c));
@@ -119,7 +132,7 @@ function generateSymbol(id: string, svgStr: string): string {
   return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
 }
 
-async function init(): Promise<string[]> {
+async function init(): Promise<void> {
   // Delete any old output directory
   rmDirSync(OUTPUT_DIR);
 
@@ -127,13 +140,13 @@ async function init(): Promise<string[]> {
   fs.mkdirSync(OUTPUT_DIR);
   fs.mkdirSync(OUTPUT_DIR + "/bin");
   fs.mkdirSync(OUTPUT_DIR + "/public");
+  fs.mkdirSync(OUTPUT_DIR + "/tools");
 
   // Create package.json
   const packageJson = JSON.parse(
     fs.readFileSync(path.resolve(INPUT_DIR, "package.json")).toString()
   );
   delete packageJson["devDependencies"];
-  delete packageJson["private"];
   packageJson["scripts"] = {
     install: packageJson["scripts"].install,
     configure: packageJson["scripts"].configure,
@@ -150,12 +163,10 @@ async function init(): Promise<string[]> {
   );
   npmShrinkwrapJson["version"] = packageJson["version"];
   stripDevDeps(npmShrinkwrapJson);
-  stripDevDeps2(npmShrinkwrapJson);
   fs.writeFileSync(
     path.resolve(OUTPUT_DIR, "npm-shrinkwrap.json"),
     JSON.stringify(npmShrinkwrapJson, null, 2)
   );
-  return Object.keys(packageJson["dependencies"]);
 }
 
 async function copyStatic(): Promise<void> {
@@ -193,7 +204,41 @@ async function generateCss(): Promise<void> {
   fs.writeFileSync(cssOutPath, cssOut.css);
 }
 
-async function generateBackendJs(externals: string[]): Promise<void> {
+async function generateToolsJs(): Promise<void> {
+  for (const bin of ["dump-data-model"]) {
+    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}.ts`);
+    const outputFile = path.resolve(OUTPUT_DIR, `tools/${bin}`);
+    const bundle = await rollup({
+      input: inputFile,
+      external: externals,
+      acorn: {
+        allowHashBang: true,
+      },
+      plugins: [
+        typescript({
+          tsconfig: "./tsconfig.json",
+          include: [`tools/${bin}.ts`, "lib/**/*.ts"],
+        }),
+        MODE === "production" ? terser() : null,
+      ],
+    });
+
+    await bundle.write({
+      format: "cjs",
+      preferConst: true,
+      sourcemap: "inline",
+      sourcemapExcludeSources: true,
+      banner: "#!/usr/bin/env node",
+      file: outputFile,
+    });
+
+    // Mark as executable
+    const mode = fs.statSync(outputFile).mode;
+    fs.chmodSync(outputFile, mode | 73);
+  }
+}
+
+async function generateBackendJs(): Promise<void> {
   for (const bin of [
     "genieacs-cwmp",
     "genieacs-ext",
@@ -205,7 +250,7 @@ async function generateBackendJs(externals: string[]): Promise<void> {
     const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
     const bundle = await rollup({
       input: inputFile,
-      external: [...builtins, ...externals],
+      external: externals,
       acorn: {
         allowHashBang: true,
       },
@@ -249,26 +294,26 @@ async function generateBackendJs(externals: string[]): Promise<void> {
   }
 }
 
-async function generateFrontendJs(externals: string[]): Promise<void> {
+async function generateFrontendJs(): Promise<void> {
   const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
-  const outputDir = path.resolve(OUTPUT_DIR, "public");
+  const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
 
-  const inlineDeps = ["parsimmon", "espresso-iisojs"];
+  const inlineDeps = ["mithril", "parsimmon", "jsbi", "espresso-iisojs"];
   const bundle = await rollup({
     input: inputFile,
-    external: [
-      ...builtins,
-      ...externals.filter((e) => !inlineDeps.includes(e)),
-    ],
+    external: externals.filter((e) => !inlineDeps.includes(e)),
     plugins: [
       rollupJson({ preferConst: true }),
       {
         name: "",
         resolveId: function (importee, importer) {
-          if (importee.endsWith("/package.json")) {
-            const p = path.resolve(path.dirname(importer), importee);
-            if (p === path.resolve(INPUT_DIR, "package.json"))
-              return path.resolve(OUTPUT_DIR, "package.json");
+          if (importee.endsWith("/bigint")) {
+            return this.resolve(importee + "-jsbi", importer);
+          } else if (importee === "espresso-iisojs") {
+            return this.resolve(
+              "espresso-iisojs/dist/espresso-iisojs-jsbi.mjs",
+              importer
+            );
           }
           return null;
         },
@@ -276,9 +321,8 @@ async function generateFrontendJs(externals: string[]): Promise<void> {
       typescript({ tsconfig: "./tsconfig.json" }),
       nodeResolve(),
       commonjs(),
-      MODE === "production" ? terser() : null,
     ],
-    preserveEntrySignatures: false,
+    inlineDynamicImports: true,
     treeshake: {
       propertyReadSideEffects: false,
       moduleSideEffects: false,
@@ -290,37 +334,52 @@ async function generateFrontendJs(externals: string[]): Promise<void> {
   });
 
   await bundle.write({
-    manualChunks: (id) => {
-      if (id.includes("node_modules/codemirror")) return "codemirror";
-      else if (id.includes("node_modules/yaml")) return "yaml";
-      return "app";
-    },
     preferConst: true,
     format: "es",
-    sourcemap: true,
+    sourcemap: "inline",
     sourcemapExcludeSources: true,
-    dir: outputDir,
+    file: outputFile,
+  });
+
+  const webpackConf: Configuration = {
+    mode: MODE as "development" | "production",
+    entry: outputFile,
+    resolve: {
+      aliasFields: ["module"],
+    },
+    devtool: "nosources-source-map",
+    module: {
+      rules: [
+        {
+          test: /\.js$/,
+          use: ["source-map-loader"],
+          enforce: "pre",
+        },
+      ],
+    },
+    output: {
+      path: path.resolve(OUTPUT_DIR, "public"),
+      filename: "app.js",
+    },
+  };
+
+  return new Promise((resolve, reject) => {
+    webpack(webpackConf, (err, stats) => {
+      if (err) return reject(err);
+      process.stdout.write(stats.toString({ colors: true }) + "\n");
+      resolve();
+    });
   });
 }
 
 async function generateIconsSprite(): Promise<void> {
+  const svgo = new SVGO({ plugins: [{ removeViewBox: false }] });
   const symbols = [];
   const iconsDir = path.resolve(INPUT_DIR, "ui/icons");
   for (const file of fs.readdirSync(iconsDir)) {
     const id = path.parse(file).name;
     const filePath = path.join(iconsDir, file);
-    const { data } = await optimize(fs.readFileSync(filePath).toString(), {
-      plugins: [
-        {
-          name: "preset-default",
-          params: {
-            overrides: {
-              removeViewBox: false,
-            },
-          },
-        },
-      ],
-    });
+    const { data } = await svgo.optimize(fs.readFileSync(filePath).toString());
     symbols.push(generateSymbol(id, data));
   }
   fs.writeFileSync(
@@ -330,13 +389,14 @@ async function generateIconsSprite(): Promise<void> {
 }
 
 init()
-  .then((externals) => {
+  .then(() => {
     Promise.all([
       copyStatic(),
       generateCss(),
       generateIconsSprite(),
-      generateBackendJs(externals),
-      generateFrontendJs(externals),
+      generateToolsJs(),
+      generateBackendJs(),
+      generateFrontendJs(),
     ])
       .then(() => {
         // Ignore

build/spellcheck-dict.pws

@@ -1,172 +0,0 @@
-personal_ws-1.1 en 0
-genieacs
-GenieACS
-javascript
-sudo
-config
-cwmp
-CPE
-systemctl
-nbi
-CWMP
-NBI
-ACS
-SSL
-xsd
-args
-DeviceID
-GENIEACS
-env
-param
-arg
-AUTH
-igd
-TCP
-toctree
-auth
-EnvironmentFile
-JSON
-json
-parameterValues
-setParameterValues
-stderr
-stdout
-attr
-cfg
-CPEs
-ExecStart
-faq
-FileType
-GenieACS's
-JWT
-maxdepth
-npm
-objectName
-refreshObject
-SerialNumber
-systemd
-usr
-WantedBy
-yaml
-chown
-Config
-getParameterValues
-https
-InternetGatewayDevice
-mkdir
-OUI
-sql
-TODO
-UDP
-addObject
-AGPLv
-const
-cpe
-cron
-deleteObject
-FactoryReset
-factoryReset
-FileName
-fileType
-Github
-hostname
-HTTPS
-jwt
-latlong
-NPM
-oui
-parameterNames
-ProductClass
-productClass
-repo
-SSID
-sublicense
-TLS
-WANIPConnection
-WPA
-YourProvisionName
-abc
-Abdulla
-accessors
-AddObject
-AddressingType
-api
-APIs
-chmod
-cwmp's
-dateext
-DATETIME
-dateTime
-datetime
-declaratively
-delaycompress
-DeleteObject
-deviceId
-DHCP
-dir
-distro
-equisatisfiable
-ExternalIPAddress
-failover
-favor
-func
-GetParameterAttributes
-getPassword
-gte
-gui
-hexBinary
-Hils
-HOSTNAME
-init
-journalctl
-journald
-LastFileName
-LastFileType
-lastInform
-LIBXMLJS
-libxmljs
-literalinclude
-lockfile
-logrotate
-lte
-mipsbe
-normalizers
-pre
-quickstart
-README
-redis
-reimplemented
-RequestDownload
-RESTful
-rollout
-sandboxed
-scalable
-SetParameterAttributes
-SetParamteerValues
-SIGINT
-SIGTERM
-skipWritableCheck
-stringify
-subdirectory
-TargetFileName
-useradd
-VirtualParameters
-vparam
-xml
-YAML
-Zaid
-cacheExpire
-iss
-http
-statusCode
-rawData
-pos
-Brotli
-CSV
-hostInfo
-IPv
-downloadSuccessOnTimeout
-TransferComplete
-nginx
-ENCODEURICOMPONENT
-pageSize

build/spellcheck.sh

@@ -1,12 +0,0 @@
-#! /bin/sh
-
-cd "$(dirname "$0")"
-
-FILES=`ls ../docs/*.rst ../docs/*.js ../*.md`
-
-for FILE in $FILES
-do
-    echo $FILE
-    cat $FILE | aspell list --lang=en --add-extra-dicts=./spellcheck-dict.pws --ignore 2
-    echo
-done

docs/api-reference.rst

@@ -1,7 +1,7 @@
 API Reference
 =============
 
-GenieACS exposes a rich RESTful API through its NBI component. This document
+GenieACS expoeses a rich RESTful API through its NBI component. This document
 serves as a reference for the available APIs.
 
 This API makes use of MongoDB's query language in some of its endpoints. Refer
@@ -16,7 +16,7 @@ to MongoDB's documentation for details.
 
 .. warning::
 
-  A common pitfall is not properly percent-encoding special characters in the
+  A common pitfll is not properly percent-encoding special characters in the
   device ID or query in the URL.
 
 Endpoints
@@ -41,11 +41,11 @@ Examples
 
 .. code:: javascript
 
-  query = {"_id": "202BC1-BM632w-000000"}
+  query = {"_id": "202BC1-BM632w-0000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D'
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D'
 
 - Find a device by its MAC address:
 
@@ -77,21 +77,21 @@ Examples
 
 .. code:: javascript
 
-  query = {"device": "202BC1-BM632w-000000"}
+  query = {"device": "202BC1-BM632w-0000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-000000%22%7D'
+  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-0000000%22%7D'
 
 - Return specific parameters for a given device:
 
 .. code:: javascript
 
-  query = {"_id": "202BC1-BM632w-000000"}
+  query = {"_id": "202BC1-BM632w-0000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
+  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
 
 The ``projection`` URL param is a comma-separated list of the parameters to receive.
 
@@ -106,7 +106,7 @@ if the tasks have been queued to be executed at the next inform.
 *device_id*: The ID of the device.
 
 *connection_request*: Indicates that a connection request will be triggered to
-execute the tasks immediately. Otherwise, the tasks will be queued and be
+execute the tasks immediatly. Otherwise, the tasks will be queued and be
 processed at the next inform.
 
 The response body is the task object as it is inserted in the database. The
@@ -120,7 +120,7 @@ Examples
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
   -X POST \
   --data '{"name": "refreshObject", "objectName": ""}'
 
@@ -138,7 +138,7 @@ Examples
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
   -X POST \
   --data '{"name":"setParameterValues", "parameterValues": [["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]]}'
 
@@ -183,7 +183,7 @@ Example
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/faults/202BC1-BM632w-000000:default' -X DELETE
+  curl -i 'http://localhost:7557/faults/202BC1-BM632w-0000000:default' -X DELETE
 
 DELETE /devices/\<device_id\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -218,7 +218,7 @@ Assign the tag "testing" to a device:
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X POST
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X POST
 
 DELETE /devices/\<device_id\>/tags/\<tag\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -236,7 +236,7 @@ Remove the tag "testing" from a device:
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X DELETE
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X DELETE
 
 PUT /presets/\<preset_name\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -299,7 +299,7 @@ The following file metadata may be sent as request headers:
 - ``fileType``: For firmware images it should be "1 Firmware Upgrade Image".
   Other common types are "2 Web Content" and "3 Vendor Configuration File".
 
-- ``oui``: The OUI of the device model that this file belongs to.
+- ``oui``: The OUI of the device model that this file belogs to.
 
 - ``productClass``: The product class of the device.
 
@@ -344,7 +344,7 @@ Find files using a query.
 Tasks
 -----
 
-Find the different available tasks and their object structure.
+Find the different availabe tasks and their object structure.
 
 ``getParameterValues``
 ~~~~~~~~~~~~~~~~~~~~~~

docs/cpe-authentication.rst

@@ -3,70 +3,4 @@
 CPE Authentication
 ==================
 
-CPE to ACS
-----------
-
-.. note::
-
-  By default GenieACS will accept any incoming connection via HTTP/HTTPS and
-  respond to it.
-
-The following parameters are used to set and get (password is redacted but
-can be set) the username/password used to authenticate against the ACS:
-
-Username: ``Device.ManagementServer.Username`` or ``InternetGatewayDevice.ManagementServer.Username``
-
-Password: ``Device.ManagementServer.Password`` or ``InternetGatewayDevice.ManagementServer.Password``
-
-Enable CPE to ACS Authentication
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-CPE to ACS authentication can be configured in the web interface by using the
-`Config` option in the `Admin` tab.
-
-Go to the `Admin` -> `Config` page and click on `New config` button at the
-bottom of the page. This will open pop-up which requires you to fill in a key
-and value. The key should be ``cwmp.auth``. The value accepts a boolean.
-Setting the value to ``true`` makes it so that GenieACS accepts any incoming
-connection, setting it to ``false`` makes GenieACS deny all incoming
-connections. This can be further configured using the ``AUTH()`` and ``EXT()``
-functions.
-
-The ``AUTH()`` function
-~~~~~~~~~~~~~~~~~~~~~~~
-
-The ``AUTH()`` function accepts two parameters, username and password. It
-checks the given username and password with the incoming request to determine
-whether to return true or false.
-
-Basic usage of the ``AUTH()`` function could be as follows:
-
-.. code:: sql
-
-   AUTH("fixed-username", "fixed-password")
-
-This will only accept incoming request who authenticate with
-"fixed-username" and "fixed-password".
-
-The various device parameters can be referenced from within the ``cwmp.auth``
-expression. For example:
-
-.. code:: sql
-
-   AUTH(Device.ManagementServer.Username, Device.ManagementServer.Password)
-
-The ``EXT()`` function
-~~~~~~~~~~~~~~~~~~~~~~
-
-The ``EXT()`` function makes it possible to call an :ref:`extension
-<extensions>` script from the auth expression. This can be used to fetch
-the credentials from an external source:
-
-.. code:: sql
-
-   AUTH(DeviceID.SerialNumber, EXT("authenticate", "getPassword", DeviceID.SerialNumber))
-
-ACS to CPE
-----------
-
 TODO

docs/environment-variables.rst

@@ -64,7 +64,7 @@ CWMP_PORT
 CWMP_INTERFACE
   The network interface that genieacs-cwmp binds to.
 
-  Default: ``::``
+  Default: ``0.0.0.0``
 
 CWMP_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
@@ -86,7 +86,7 @@ CWMP_ACCESS_LOG_FILE
   File to log incoming requests for genieacs-cwmp. If omitted, logs will go to
   stdout.
 
-  Default: unset
+  Default: unest
 
 NBI_WORKER_PROCESSES
   The number of worker processes to spawn for genieacs-nbi. A value of 0 means
@@ -102,12 +102,12 @@ NBI_PORT
 NBI_INTERFACE
   The network interface that genieacs-nbi binds to.
 
-  Default: ``::``
+  Default: ``0.0.0.0``
 
 NBI_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
 
-  Default: unset
+  Default: unest
 
 NBI_SSL_KEY
   Path to certificate key file. If omitted, non-secure HTTP will be used.
@@ -140,7 +140,7 @@ FS_PORT
 FS_INTERFACE
   The network interface that genieacs-fs binds to.
 
-  Default: ``::``
+  Default: ``0.0.0.0``
 
 FS_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
@@ -186,7 +186,7 @@ UI_PORT
 UI_INTERFACE
   The network interface that genieacs-ui binds to.
 
-  Default: ``::``
+  Default: ``0.0.0.0``
 
 UI_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.

docs/ext-sample.js

@@ -1,4 +1,4 @@
-// This is an example GenieACS extension to get the current latitude/longitude
+// This is an exmaple GenieACS extension to get the current latitude/longitude
 // of the International Space Station. Why, you ask? Because why not.
 // To install, copy this file to config/ext/iss.js.
 
@@ -10,28 +10,25 @@ let cache = null;
 let cacheExpire = 0;
 
 function latlong(args, callback) {
-  if (Date.now() < cacheExpire) return callback(null, cache);
+  if (Date.now() < cacheExpire)
+    return callback(null, cache);
 
-  http
-    .get("http://api.open-notify.org/iss-now.json", (res) => {
-      if (res.statusCode !== 200)
-        return callback(
-          new Error(`Request failed (status code: ${res.statusCode})`)
-        );
+  http.get("http://api.open-notify.org/iss-now.json", (res) => {
+    if (res.statusCode !== 200)
+      return callback(new Error(`Request failed (status code: ${res.statusCode})`));
 
-      let rawData = "";
-      res.on("data", (chunk) => (rawData += chunk));
+    let rawData = "";
+    res.on("data", (chunk) => rawData += chunk);
 
-      res.on("end", () => {
-        let pos = JSON.parse(rawData)["iss_position"];
-        cache = [+pos["latitude"], +pos["longitude"]];
-        cacheExpire = Date.now() + 10000;
-        callback(null, cache);
-      });
-    })
-    .on("error", (err) => {
-      callback(err);
+    res.on("end", () => {
+      let pos = JSON.parse(rawData)["iss_position"];
+      cache = [+pos["latitude"], +pos["longitude"]];
+      cacheExpire = Date.now() + 10000;
+      callback(null, cache);
     });
+  }).on("error", (err) => {
+    callback(err);
+  });
 }
 
 exports.latlong = latlong;

docs/installation-guide.rst

@@ -4,7 +4,7 @@ Installation Guide
 This guide is for installing GenieACS on a single server on any Linux distro
 that uses *systemd* as its init system.
 
-The various GenieACS services are independent of each other and may be
+The various GenieACS services are independant of each other and may be
 installed on different servers. You may also run multiple instances of each in
 a load-balancing/failover setup.
 
@@ -19,7 +19,7 @@ Prerequisites
 
 .. topic:: Node.js
 
-  GenieACS requires Node.js 12.13 and up. Refer to https://nodejs.org/ for
+  GenieACS requires Node.js 10.x and up. Refer to https://nodejs.org/ for
   instructions.
 
 .. topic:: MongoDB
@@ -34,7 +34,7 @@ Install GenieACS
 
   .. parsed-literal::
 
-    sudo npm install -g genieacs@\ |release|
+    sudo npm install -g --unsafe-perm genieacs@\ |release|
 
 .. topic:: Installing from source
 
@@ -73,7 +73,6 @@ Configure systemd
     GENIEACS_FS_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-fs-access.log
     GENIEACS_UI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-ui-access.log
     GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
-    NODE_OPTIONS=--enable-source-maps
     GENIEACS_EXT_DIR=/opt/genieacs/ext
     GENIEACS_UI_JWT_SECRET=secret
   

lib/api-functions.ts

@@ -17,6 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+import { parse } from "url";
 import * as db from "./db";
 import * as common from "./common";
 import * as cache from "./cache";
@@ -28,10 +29,14 @@ import {
 import {
   httpConnectionRequest,
   udpConnectionRequest,
+  xmppConnectionRequest,
 } from "./connection-request";
 import { Expression, Task } from "./types";
 import { flattenDevice } from "./mongodb-functions";
 import { evaluate } from "./common/expression";
+import * as config from "../lib/config";
+
+const XMPP_CONFIGURED = !!config.get("XMPP_JID");
 
 export async function connectionRequest(
   deviceId: string,
@@ -43,7 +48,12 @@ export async function connectionRequest(
     device = flattenDevice(res);
   }
 
-  let connectionRequestUrl, udpConnectionRequestAddress, username, password;
+  let connectionRequestUrl,
+    udpConnectionRequestAddress,
+    stunEnable,
+    connReqJabberId,
+    username,
+    password;
 
   if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
     connectionRequestUrl = (device[
@@ -54,6 +64,12 @@ export async function connectionRequest(
         "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
       ] || {}
     ).value || [""])[0];
+    stunEnable = ((
+      device["InternetGatewayDevice.ManagementServer.STUNEnable"] || {}
+    ).value || [""])[0];
+    connReqJabberId = ((
+      device["InternetGatewayDevice.ManagementServer.ConnReqJabberID"] || {}
+    ).value || [""])[0];
     username = ((
       device[
         "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
@@ -71,6 +87,10 @@ export async function connectionRequest(
     udpConnectionRequestAddress = ((
       device["Device.ManagementServer.UDPConnectionRequestAddress"] || {}
     ).value || [""])[0];
+    stunEnable = ((device["Device.ManagementServer.STUNEnable"] || {})
+      .value || [""])[0];
+    connReqJabberId = ((device["Device.ManagementServer.ConnReqJabberID"] || {})
+      .value || [""])[0];
     username = ((
       device["Device.ManagementServer.ConnectionRequestUsername"] || {}
     ).value || [""])[0];
@@ -79,7 +99,7 @@ export async function connectionRequest(
     ).value || [""])[0];
   }
 
-  const remoteAddress = new URL(connectionRequestUrl).hostname;
+  const remoteAddress = parse(connectionRequestUrl).host;
 
   const evalCallback = (exp): Expression => {
     if (!Array.isArray(exp)) return exp;
@@ -94,7 +114,7 @@ export async function connectionRequest(
       else if (name === "password") return password;
 
       const p = device[name];
-      if (p?.value) return p.value[0];
+      if (p && p.value) return p.value[0];
     } else if (exp[0] === "FUNC") {
       if (exp[1] === "REMOTE_ADDRESS") return remoteAddress;
       else if (exp[1] === "USERNAME") return username;
@@ -144,36 +164,42 @@ export async function connectionRequest(
 
   const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
 
-  let udpProm = Promise.resolve(false);
-  if (udpConnectionRequestAddress) {
-    try {
-      const u = new URL("udp://" + udpConnectionRequestAddress);
-      udpProm = udpConnectionRequest(
-        u.hostname,
-        parseInt(u.port || "80"),
-        authExp,
-        UDP_CONNECTION_REQUEST_PORT,
-        debug,
-        deviceId
-      ).then(
-        () => true,
-        () => false
-      );
-    } catch (err) {
-      // Ignore invalid address
-    }
+  let udpProm;
+  if (udpConnectionRequestAddress && +stunEnable) {
+    udpProm = udpConnectionRequest(
+      udpConnectionRequestAddress,
+      authExp,
+      UDP_CONNECTION_REQUEST_PORT,
+      debug,
+      deviceId
+    );
   }
 
-  const status = await httpConnectionRequest(
-    connectionRequestUrl,
-    authExp,
-    CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
-    CONNECTION_REQUEST_TIMEOUT,
-    debug,
-    deviceId
-  );
+  let status;
 
-  if (await udpProm) return "";
+  if (connReqJabberId && XMPP_CONFIGURED) {
+    status = await xmppConnectionRequest(
+      connReqJabberId,
+      authExp,
+      CONNECTION_REQUEST_TIMEOUT,
+      debug,
+      deviceId
+    );
+  } else {
+    status = await httpConnectionRequest(
+      connectionRequestUrl,
+      authExp,
+      CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
+      CONNECTION_REQUEST_TIMEOUT,
+      debug,
+      deviceId
+    );
+
+    if (udpProm) {
+      await udpProm;
+      return "";
+    }
+  }
 
   return status;
 }
@@ -301,15 +327,6 @@ function sanitizeTask(task): void {
       )
         throw new Error("Invalid 'provisions' property");
       break;
-
-    case "reboot":
-      break;
-
-    case "factoryReset":
-      break;
-
-    default:
-      throw new Error("Invalid task name");
   }
 
   return task;
@@ -317,7 +334,7 @@ function sanitizeTask(task): void {
 
 export async function insertTasks(tasks: any[]): Promise<Task[]> {
   if (tasks && !Array.isArray(tasks)) tasks = [tasks];
-  else if (!tasks?.length) return tasks || [];
+  else if (!tasks || tasks.length === 0) return tasks || [];
 
   for (const task of tasks) {
     sanitizeTask(task);

lib/auth.ts

@@ -52,9 +52,9 @@ function parseHeaderFeilds(str: string): Record<string, string> {
   return res;
 }
 
-export function parseAuthorizationHeader(authHeader: string): {
-  method: string;
-} {
+export function parseAuthorizationHeader(
+  authHeader: string
+): { method: string } {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };

lib/cache.ts

@@ -59,7 +59,7 @@ export async function set(
 
 export async function pop(key: string): Promise<any> {
   const res = await cacheCollection.findOneAndDelete({ _id: key });
-  if (res?.["value"]) return res["value"]["value"];
+  if (res && res["value"]) return res["value"]["value"];
   return null;
 }
 
@@ -78,9 +78,9 @@ export async function acquireLock(
           value: token,
           expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
         },
-        $currentDate: { timestamp: true as unknown as Date },
+        $currentDate: { timestamp: true },
       },
-      { upsert: true, returnDocument: "after" }
+      { upsert: true, returnOriginal: false }
     );
     const v = r.value;
     if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
@@ -114,5 +114,5 @@ export async function releaseLock(
     _id: lockName,
     value: token,
   });
-  if (res.deletedCount !== 1) throw new Error("Lock expired");
+  if (res["result"]["n"] !== 1) throw new Error("Lock expired");
 }

lib/cluster.ts

@@ -17,14 +17,14 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import cluster, { Worker } from "cluster";
+import cluster from "cluster";
 import { cpus } from "os";
 import * as logger from "./logger";
 
 let respawnTimestamp = 0;
 let crashes: number[] = [];
 
-function fork(): Worker {
+function fork(): cluster.Worker {
   const w = cluster.fork();
   w.on("error", (err: NodeJS.ErrnoException) => {
     // Avoid exception when attempting to kill the process just as it's exiting

lib/common.ts

@@ -51,7 +51,7 @@ export function escapeRegExp(str: string): string {
 export function encodeTag(tag: string): string {
   return encodeURIComponent(tag)
     .replace(
-      /[!~*'().]/g,
+      /[!~*'()]/g,
       (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
     )
     .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")

lib/common/authorizer.ts

@@ -21,13 +21,13 @@ import { PermissionSet, Expression } from "../types";
 import { evaluate, or } from "./expression";
 
 export default class Authorizer {
-  private declare permissionSets: PermissionSet[];
-  private declare validatorCache: WeakMap<
+  private permissionSets: PermissionSet[];
+  private validatorCache: WeakMap<
     any,
     (mutationType, mutation, any) => boolean
   >;
-  private declare hasAccessCache: Map<string, boolean>;
-  private declare getFilterCache: Map<string, Expression>;
+  private hasAccessCache: Map<string, boolean>;
+  private getFilterCache: Map<string, Expression>;
 
   public constructor(permissionSets: PermissionSet[]) {
     this.permissionSets = permissionSets;

lib/common/bigint-jsbi.ts

@@ -0,0 +1,108 @@
+/**
+ * Copyright 2013-2020  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import JSBI from "jsbi";
+
+export function and(a: JSBI, b: JSBI): JSBI {
+  return JSBI.bitwiseAnd(a, b);
+}
+
+export function or(a: JSBI, b: JSBI): JSBI {
+  return JSBI.bitwiseOr(a, b);
+}
+
+export function xor(a: JSBI, b: JSBI): JSBI {
+  return JSBI.bitwiseXor(a, b);
+}
+
+export function not(a: JSBI): JSBI {
+  return JSBI.bitwiseNot(a);
+}
+
+export function lshift(a: JSBI, b: JSBI): JSBI {
+  return JSBI.leftShift(a, b);
+}
+
+export function rshift(a: JSBI, b: JSBI): JSBI {
+  return JSBI.signedRightShift(a, b);
+}
+
+export function add(a: JSBI, b: JSBI): JSBI {
+  return JSBI.add(a, b);
+}
+
+export function sub(a: JSBI, b: JSBI): JSBI {
+  return JSBI.subtract(a, b);
+}
+
+export function mul(a: JSBI, b: JSBI): JSBI {
+  return JSBI.multiply(a, b);
+}
+
+export function div(a: JSBI, b: JSBI): JSBI {
+  return JSBI.divide(a, b);
+}
+
+export function exp(a: JSBI, b: JSBI): JSBI {
+  return JSBI.exponentiate(a, b);
+}
+
+export function rem(a: JSBI, b: JSBI): JSBI {
+  return JSBI.remainder(a, b);
+}
+
+export function toNumber(a: JSBI): number {
+  return JSBI.toNumber(a);
+}
+
+export function eq(a: JSBI, b: JSBI): boolean {
+  return JSBI.equal(a, b);
+}
+
+export function ne(a: JSBI, b: JSBI): boolean {
+  return JSBI.notEqual(a, b);
+}
+
+export function lt(a: JSBI, b: JSBI): boolean {
+  return JSBI.lessThan(a, b);
+}
+
+export function lte(a: JSBI, b: JSBI): boolean {
+  return JSBI.lessThanOrEqual(a, b);
+}
+
+export function gt(a: JSBI, b: JSBI): boolean {
+  return JSBI.greaterThan(a, b);
+}
+
+export function gte(a: JSBI, b: JSBI): boolean {
+  return JSBI.greaterThanOrEqual(a, b);
+}
+
+export function asUintN(a: number, b: JSBI): JSBI {
+  return JSBI.asUintN(a, b);
+}
+
+export function asIntN(a: number, b: JSBI): JSBI {
+  return JSBI.asIntN(a, b);
+}
+
+const _BigInt = JSBI.BigInt;
+type _bigint = JSBI;
+export { _BigInt as BigInt, _bigint as bigint };

lib/common/boolean-expression.ts

@@ -29,8 +29,8 @@ const TWO = BI.BigInt(2);
 const NEGATIVE_ONE = BI.BigInt(-1);
 
 class Indeterminates {
-  public declare map: Map<string, number>;
-  public declare sortedKeys: string[];
+  public map: Map<string, number>;
+  public sortedKeys: string[];
 
   public constructor(str?: string) {
     this.map = new Map();
@@ -118,7 +118,7 @@ function findGcd(a: BI.bigint, b: BI.bigint): BI.bigint {
 }
 
 class Polynomial {
-  public declare terms: Term[];
+  public terms: Term[];
 
   public constructor(terms: Term[]) {
     this.terms = terms;
@@ -298,11 +298,12 @@ class Polynomial {
 }
 
 type Minterm = number[];
+type Context = Map<string, number>;
 
 abstract class BoolExprSynth {
-  public abstract true(context: SynthContext): Minterm[];
-  public abstract false(context: SynthContext): Minterm[];
-  public abstract null(context: SynthContext): Minterm[];
+  public abstract true(context: Context): Minterm[];
+  public abstract false(context: Context): Minterm[];
+  public abstract null(context: Context): Minterm[];
 }
 
 class TrueSynth extends BoolExprSynth {
@@ -342,8 +343,8 @@ class NullSynth extends BoolExprSynth {
 }
 
 class VarSynth extends BoolExprSynth {
-  private declare exp: Expression;
-  private declare negate: boolean;
+  private expStr: string;
+  private negate: boolean;
   public constructor(exp: Expression) {
     super();
     this.negate = false;
@@ -367,49 +368,52 @@ class VarSynth extends BoolExprSynth {
         this.negate = true;
       }
     }
-    this.exp = exp;
+    this.expStr = JSON.stringify(exp);
   }
-  public true(context: SynthContext): Minterm[] {
-    const v = context.getVariable(this.exp);
+  public true(context: Context): Minterm[] {
+    let v = context.get(this.expStr);
+    if (v == null) context.set(this.expStr, (v = context.size));
     return [[(v << 2) ^ (this.negate ? 1 : 3)]];
   }
-  public false(context: SynthContext): Minterm[] {
-    const v = context.getVariable(this.exp);
+  public false(context: Context): Minterm[] {
+    let v = context.get(this.expStr);
+    if (v == null) context.set(this.expStr, (v = context.size));
     return [[(v << 2) ^ (this.negate ? 3 : 1)]];
   }
-  public null(context: SynthContext): Minterm[] {
-    const v = context.getVariable(this.exp);
+  public null(context: Context): Minterm[] {
+    let v = context.get(this.expStr);
+    if (v == null) context.set(this.expStr, (v = context.size));
     return [[v << 2, (v << 2) ^ 2]];
   }
 }
 
 class NotSynth extends BoolExprSynth {
-  private declare exprSynth: BoolExprSynth;
+  private exprSynth: BoolExprSynth;
   public constructor(e: BoolExprSynth) {
     super();
     this.exprSynth = e;
   }
-  public true(context: SynthContext): Minterm[] {
+  public true(context: Context): Minterm[] {
     return this.exprSynth.false(context);
   }
-  public false(context: SynthContext): Minterm[] {
+  public false(context: Context): Minterm[] {
     return this.exprSynth.true(context);
   }
-  public null(context: SynthContext): Minterm[] {
+  public null(context: Context): Minterm[] {
     return this.exprSynth.null(context);
   }
 }
 
 class IsNullSynth extends BoolExprSynth {
-  private declare exprSynth: BoolExprSynth;
+  private exprSynth: BoolExprSynth;
   public constructor(e: BoolExprSynth) {
     super();
     this.exprSynth = e;
   }
-  public true(context: SynthContext): Minterm[] {
+  public true(context: Context): Minterm[] {
     return this.exprSynth.null(context);
   }
-  public false(context: SynthContext): Minterm[] {
+  public false(context: Context): Minterm[] {
     return [...this.exprSynth.true(context), ...this.exprSynth.false(context)];
   }
   public null(): Minterm[] {
@@ -418,7 +422,7 @@ class IsNullSynth extends BoolExprSynth {
 }
 
 class OrSynth extends BoolExprSynth {
-  private declare exprSynths: BoolExprSynth[];
+  private exprSynths: BoolExprSynth[];
   public constructor(...e: BoolExprSynth[]) {
     super();
     this.exprSynths = e.filter((ee) => !(ee instanceof FalseSynth));
@@ -432,13 +436,13 @@ class OrSynth extends BoolExprSynth {
     });
     this.exprSynths.push(...unpacked);
   }
-  public true(context: SynthContext): Minterm[] {
+  public true(context: Context): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].true(context);
     if (this.exprSynths.some((e) => e instanceof TrueSynth)) return [[]];
     return this.exprSynths.map((e) => e.true(context)).flat();
   }
-  public false(context: SynthContext): Minterm[] {
+  public false(context: Context): Minterm[] {
     if (this.exprSynths.length === 0) return [[]];
     if (this.exprSynths.length === 1) return this.exprSynths[0].false(context);
     if (
@@ -451,7 +455,7 @@ class OrSynth extends BoolExprSynth {
       this.exprSynths.map((e) => complement(e.false(context))).flat()
     );
   }
-  public null(context: SynthContext): Minterm[] {
+  public null(context: Context): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].null(context);
     const n = this.exprSynths.map((e) => e.null(context)).flat();
@@ -461,7 +465,7 @@ class OrSynth extends BoolExprSynth {
 }
 
 class AndSynth extends BoolExprSynth {
-  private declare exprSynths: BoolExprSynth[];
+  private exprSynths: BoolExprSynth[];
   public constructor(...e: BoolExprSynth[]) {
     super();
     this.exprSynths = e.filter((ee) => !(ee instanceof TrueSynth));
@@ -475,7 +479,7 @@ class AndSynth extends BoolExprSynth {
     });
     this.exprSynths.push(...unpacked);
   }
-  public true(context: SynthContext): Minterm[] {
+  public true(context: Context): Minterm[] {
     if (this.exprSynths.length === 0) return [[]];
     if (this.exprSynths.length === 1) return this.exprSynths[0].true(context);
     if (
@@ -488,13 +492,13 @@ class AndSynth extends BoolExprSynth {
       this.exprSynths.map((e) => complement(e.true(context))).flat()
     );
   }
-  public false(context: SynthContext): Minterm[] {
+  public false(context: Context): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].false(context);
     if (this.exprSynths.some((e) => e instanceof FalseSynth)) return [[]];
     return this.exprSynths.map((e) => e.false(context)).flat();
   }
-  public null(context: SynthContext): Minterm[] {
+  public null(context: Context): Minterm[] {
     if (this.exprSynths.length === 0) return [];
     if (this.exprSynths.length === 1) return this.exprSynths[0].null(context);
     const n = this.exprSynths.map((e) => e.null(context)).flat();
@@ -504,12 +508,12 @@ class AndSynth extends BoolExprSynth {
 }
 
 class CaseSynth extends BoolExprSynth {
-  private declare exprSynths: BoolExprSynth[];
+  private exprSynths: BoolExprSynth[];
   public constructor(e: BoolExprSynth[]) {
     super();
     this.exprSynths = e;
   }
-  public true(context: SynthContext): Minterm[] {
+  public true(context: Context): Minterm[] {
     const minterms: Minterm[] = [];
     const cumulative: Minterm[] = [];
     for (let i = 0; i < this.exprSynths.length; i += 2) {
@@ -529,7 +533,7 @@ class CaseSynth extends BoolExprSynth {
     }
     return minterms;
   }
-  public false(context: SynthContext): Minterm[] {
+  public false(context: Context): Minterm[] {
     const minterms: Minterm[] = [];
     const cumulative: Minterm[] = [];
     for (let i = 0; i < this.exprSynths.length; i += 2) {
@@ -549,7 +553,7 @@ class CaseSynth extends BoolExprSynth {
     }
     return minterms;
   }
-  public null(context: SynthContext): Minterm[] {
+  public null(context: Context): Minterm[] {
     const minterms: Minterm[] = [];
     const cumulative: Minterm[] = [];
     for (let i = 0; i < this.exprSynths.length; i += 2) {
@@ -662,10 +666,10 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return args.reduce(
+    return (args.reduce(
       (previousValue, currentValue) => previousValue.add(currentValue),
       ADDITIVE_IDENTITY
-    ) as unknown as Expression;
+    ) as unknown) as Expression;
   } else if (op === "*") {
     const args: Polynomial[] = [];
     for (let i = 1; i < exp.length; ++i) {
@@ -673,10 +677,10 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return args.reduce(
+    return (args.reduce(
       (previousValue, currentValue) => previousValue.multiply(currentValue),
       MULTIPLICATIVE_IDENTITY
-    ) as unknown as Expression;
+    ) as unknown) as Expression;
   } else if (op === "-") {
     const args: Polynomial[] = [];
     for (let i = 1; i < exp.length; ++i) {
@@ -684,9 +688,9 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return args.reduce((previousValue, currentValue) =>
+    return (args.reduce((previousValue, currentValue) =>
       previousValue.subtract(currentValue)
-    ) as unknown as Expression;
+    ) as unknown) as Expression;
   } else if (op === "/") {
     const args: Polynomial[] = [];
     for (let i = 1; i < exp.length; ++i) {
@@ -694,9 +698,9 @@ function normalizeCallback(exp: Expression): Expression {
       if (p == null) return null;
       args.push(p);
     }
-    return args.reduce((previousValue, currentValue) =>
+    return (args.reduce((previousValue, currentValue) =>
       previousValue.divide(currentValue)
-    ) as unknown as Expression;
+    ) as unknown) as Expression;
   } else if (["=", "<>", ">", ">=", "<", "<="].includes(op)) {
     if (exp[1] == null || exp[2] == null) return null;
     let lhs: Polynomial, rhs: Polynomial;
@@ -794,14 +798,17 @@ function toBoolExprSynth(e: Expression | BoolExprSynth): BoolExprSynth {
   return new FalseSynth();
 }
 
-function sopToExpression(sop: number[][], context: SynthContext): Expression {
+function sopToExpression(
+  sop: number[][],
+  expressions: Map<number, Expression>
+): Expression {
   if (!sop.length) return false;
   const res: Expression[] = [];
   for (const s of sop) {
     if (!s.length) return true;
     const conjs: Expression[] = [];
     for (const i of s) {
-      let expr = context.getExpression(i >>> 2);
+      let expr = expressions.get(i >>> 2);
       if (!(i & 1) !== !(i & 2)) expr = ["NOT", expr];
       if (Array.isArray(expr) && expr[0] === "NOT" && Array.isArray(expr[1])) {
         const e: Expression[] = expr[1];
@@ -825,196 +832,163 @@ function sopToExpression(sop: number[][], context: SynthContext): Expression {
   return res[0];
 }
 
-function findIsNullDeps(exp: Expression): Expression[] {
-  if (!Array.isArray(exp)) return [];
-  const op = exp[0];
-  if (op === "IS NULL" || op === "IS NOT NULL") return [];
-
-  if (op === "FUNC") {
-    if (exp[1] === "NOW") {
-      return [];
-    } else if (exp[1] === "LOWER" || exp[1] === "UPPER") {
-      return findIsNullDeps(exp[2]);
-    } else if (exp[1] === "ROUND") {
-      return exp
-        .slice(2, 4)
-        .map((e) => findIsNullDeps(e))
-        .flat();
-    }
-  } else if (op !== "PARAM") {
-    return exp
-      .slice(1)
-      .map((e) => findIsNullDeps(e))
-      .flat();
-  }
-  return [exp];
-}
-
-class SynthContext {
-  private declare _dcSet: number[][];
-  private declare _variables: Map<string, number>;
-  private declare _expressions: Map<number, Expression>;
-  private declare _isNullRelations: Map<number, number[]>;
-  private declare _isNullVars: Set<number>;
-  private declare _comparisons: Map<
+function generateDcSetAndIsNull(
+  variables: Map<number, Expression>
+): { dcSet: number[][]; isNull: Map<number, number> } {
+  const relations: Map<
     string,
-    Map<string, { eq: number; gt: number; lt: number }>
-  >;
+    { op: string; rhs: boolean | number | string; var: number }[]
+  > = new Map();
 
-  constructor() {
-    this._dcSet = [];
-    this._variables = new Map();
-    this._expressions = new Map();
-    this._isNullRelations = new Map();
-    this._isNullVars = new Set();
-    this._comparisons = new Map();
-  }
-
-  getVariable(exp: Expression): number {
-    const expStr = JSON.stringify(exp);
-    if (this._variables.has(expStr)) return this._variables.get(expStr);
-    const v = this._variables.size;
-    this._variables.set(expStr, v);
-    this._expressions.set(v, exp);
-    this.generateIsNullDc(v);
-    this.generateComparisionDc(v);
-
-    if (
-      expStr === '["IS NULL",["PARAM","DeviceID.ID"]]' ||
-      expStr === '["IS NULL",["PARAM","_id"]]'
-    )
-      this._dcSet.push([(v << 2) ^ 3]);
-
-    return v;
-  }
-
-  private generateIsNullDc(v: number): void {
-    this._isNullRelations.set(v, []);
-    const exp = this._expressions.get(v);
-
-    const isNull = new Set(
-      findIsNullDeps(exp).map((e) => this.getVariable(["IS NULL", e]))
-    );
-
-    if (!isNull.size) return;
-
-    for (const n of isNull) {
-      this._isNullVars.add(n);
-      this._isNullRelations.get(n).push(v);
-      this._isNullRelations.get(v).push(n);
-      this._dcSet.push([(n << 2) ^ 3, (v << 2) ^ 1]);
-      this._dcSet.push([(n << 2) ^ 3, (v << 2) ^ 3]);
-    }
-    this._dcSet.push(
-      [...isNull].map((n) => (n << 2) ^ 2).concat([(v << 2) ^ 0, (v << 2) ^ 2])
-    );
-  }
-
-  private generateComparisionDc(v: number): void {
-    const exp = this._expressions.get(v);
-    if (!Array.isArray(exp)) return;
-    const op = exp[0];
-    if (![">", "<", "="].includes(op)) return;
-    const rhs = exp[2];
-    const rhsStr = JSON.stringify(rhs);
-    const lhsStr = JSON.stringify(exp[1]);
-    let allComps = this._comparisons.get(lhsStr);
-    if (!allComps) this._comparisons.set(lhsStr, (allComps = new Map()));
-    if (allComps.has(rhsStr)) return;
-
-    const comp = {
-      eq: -1,
-      gt: -1,
-      lt: -1,
-    };
-
-    allComps.set(rhsStr, comp);
-
-    comp.eq = this.getVariable(["=", exp[1], exp[2]]);
-    comp.gt = this.getVariable([">", exp[1], exp[2]]);
-    comp.lt = this.getVariable(["<", exp[1], exp[2]]);
-
-    this._dcSet.push([
-      (comp.eq << 2) ^ 1,
-      (comp.gt << 2) ^ 1,
-      (comp.lt << 2) ^ 1,
-    ]);
-    this._dcSet.push([(comp.eq << 2) ^ 3, (comp.gt << 2) ^ 3]);
-    this._dcSet.push([(comp.eq << 2) ^ 3, (comp.lt << 2) ^ 3]);
-    this._dcSet.push([(comp.gt << 2) ^ 3, (comp.lt << 2) ^ 3]);
-
-    const type1 = typeof exp[2];
-    if (!["boolean", "number", "string"].includes(type1)) return;
-
-    for (const [rhs2Str, comp2] of allComps) {
-      if (comp2 === comp) continue;
-      const rhs2 = JSON.parse(rhs2Str);
-      const type2 = typeof rhs2;
-
-      if (!["boolean", "number", "string"].includes(type2)) continue;
-      let cmp = 0;
-      if (type1 === type2) cmp = rhs > rhs2 ? 1 : -1;
-      else if (type1 === "string") cmp = 1;
-      else if (type2 === "string") cmp = -1;
-      else cmp = +rhs - +rhs2;
-
-      const c1 = cmp > 0 ? comp : comp2;
-      const c2 = cmp > 0 ? comp2 : comp;
-
-      // This is the minimum clauses required if all relavent vars
-      // were included in the DC set.
-      // this._dcSet.push([(c1.eq << 2) ^ 3, (c2.eq << 2) ^ 3]);
-      // this._dcSet.push([(c1.lt << 2) ^ 1, (c2.gt << 2) ^ 1]);
-
-      // But we use non-minimal set because intermediate vars
-      // between any two may not be present in the DC set.
-      this._dcSet.push([(c1.lt << 2) ^ 1, (c2.gt << 2) ^ 1]);
-      this._dcSet.push([(c1.lt << 2) ^ 1, (c2.eq << 2) ^ 3]);
-      this._dcSet.push([(c1.lt << 2) ^ 1, (c2.lt << 2) ^ 3]);
-      this._dcSet.push([(c1.eq << 2) ^ 3, (c2.gt << 2) ^ 1]);
-      this._dcSet.push([(c1.eq << 2) ^ 3, (c2.eq << 2) ^ 3]);
-      this._dcSet.push([(c1.eq << 2) ^ 3, (c2.lt << 2) ^ 3]);
-      this._dcSet.push([(c1.gt << 2) ^ 3, (c2.gt << 2) ^ 1]);
-      this._dcSet.push([(c1.gt << 2) ^ 3, (c2.eq << 2) ^ 3]);
-      this._dcSet.push([(c1.gt << 2) ^ 3, (c2.lt << 2) ^ 3]);
+  for (const [n, e] of variables) {
+    const op = e[0] as string;
+    if (["=", ">", "<"].includes(op) && !Array.isArray(e[2])) {
+      const lhs = JSON.stringify(e[1]);
+      let r = relations.get(lhs);
+      if (!r) relations.set(lhs, (r = []));
+      r.push({ op: op, rhs: e[2], var: n });
+    } else if (op === "LIKE" && !Array.isArray(e[2]) && !Array.isArray(e[3])) {
+      const lhs = JSON.stringify(e[1]);
+      let r = relations.get(lhs);
+      if (!r) relations.set(lhs, (r = []));
+      r.push({ op: "", rhs: null, var: n });
+    } else {
+      const lhs = JSON.stringify(e);
+      let r = relations.get(lhs);
+      if (!r) relations.set(lhs, (r = []));
+      r.push({ op: "", rhs: null, var: n });
     }
   }
 
-  getExpression(v: number): Expression {
-    return this._expressions.get(v);
-  }
+  const dcSet: number[][] = [];
+  const isNull: Map<number, number> = new Map();
 
-  getIsNullRelations(v: number): number[] {
-    return this._isNullRelations.get(v);
-  }
+  for (const [key, rels] of relations) {
+    const n = variables.size;
+    variables.set(n, ["IS NULL", JSON.parse(key)]);
 
-  isIsNull(v: number): boolean {
-    return this._isNullVars.has(v);
-  }
+    if (key === '["PARAM","DeviceID.ID"]' || key === '["PARAM","_id"]')
+      dcSet.push([(n << 2) ^ 3]);
 
-  getDcSet(minterms?: Minterm[]): number[][] {
-    if (!minterms) return this._dcSet;
-    const vars = new Set(minterms.flat().map((v) => v >> 2));
+    const existingOps = new Map() as Map<string, Map<string, number>>;
+    for (const rel of rels) {
+      if (!rel.op) continue;
+      const rhsStr = JSON.stringify(rel.rhs);
+      const s = existingOps.get(rhsStr);
+      if (s) s.set(rel.op, rel.var);
+      else existingOps.set(rhsStr, new Map([[rel.op, rel.var]]));
+    }
 
-    for (const comps of this._comparisons.values()) {
-      for (const comp of comps.values()) {
-        const c = Object.values(comp).filter((v) => !vars.has(v));
-        if (c.length === 1) vars.add(c[0]);
+    for (const [k, v] of existingOps) {
+      if (v.size > 1) {
+        if (!v.has("=")) {
+          const va = variables.size;
+          variables.set(va, ["=", JSON.parse(key), JSON.parse(k)]);
+          rels.push({ op: "=", rhs: JSON.parse(k), var: va });
+          v.set("=", va);
+        }
+
+        if (!v.has(">")) {
+          const va = variables.size;
+          variables.set(va, [">", JSON.parse(key), JSON.parse(k)]);
+          rels.push({ op: ">", rhs: JSON.parse(k), var: va });
+          v.set(">", va);
+        }
+
+        if (!v.has("<")) {
+          const va = variables.size;
+          variables.set(va, ["<", JSON.parse(key), JSON.parse(k)]);
+          rels.push({ op: "<", rhs: JSON.parse(k), var: va });
+          v.set("<", va);
+        }
+
+        dcSet.push([
+          (v.get("=") << 2) ^ 1,
+          (v.get(">") << 2) ^ 1,
+          (v.get("<") << 2) ^ 1,
+        ]);
       }
     }
 
-    for (const n of this._isNullVars)
-      if (this._isNullRelations.get(n).some((v) => vars.has(v))) vars.add(n);
+    for (let i = 0; i < rels.length; ++i) {
+      const rel1 = rels[i];
+      isNull.set(rel1.var, n);
 
-    const dcSet = this._dcSet.filter((m) => m.every((v) => vars.has(v >> 2)));
+      dcSet.push([(n << 2) ^ 3, (rel1.var << 2) ^ 1]);
+      dcSet.push([(n << 2) ^ 3, (rel1.var << 2) ^ 3]);
+      dcSet.push([(rel1.var << 2) ^ 0, (rel1.var << 2) ^ 2, (n << 2) ^ 2]);
 
-    return dcSet;
+      if (!rel1.op) continue;
+      for (let j = i + 1; j < rels.length; ++j) {
+        const rel2 = rels[j];
+        if (!rel2.op) continue;
+        let comp = 0;
+        if (Array.isArray(rel1.rhs) || Array.isArray(rel2.rhs)) {
+          if (JSON.stringify(rel1.rhs) !== JSON.stringify(rel2.rhs)) continue;
+        } else if (typeof rel1.rhs === "string") {
+          if (typeof rel2.rhs === "string") {
+            if (rel1.rhs > rel2.rhs) comp = 1;
+            else if (rel1.rhs < rel2.rhs) comp = -1;
+          } else {
+            comp = 1;
+          }
+        } else if (typeof rel2.rhs === "string") {
+          comp = -1;
+        } else {
+          comp = +rel1.rhs - +rel2.rhs;
+        }
+
+        if (rel1.op === rel2.op) {
+          if (comp === 0)
+            dcSet.push([(rel1.var << 2) ^ 1, (rel2.var << 2) ^ 1]);
+          else if (rel1.op === "=")
+            dcSet.push([(rel1.var << 2) ^ 3, (rel2.var << 2) ^ 3]);
+          else if (
+            (rel1.op === ">" && comp > 0) ||
+            (rel1.op === "<" && comp < 0)
+          )
+            dcSet.push([(rel1.var << 2) ^ 3, (rel2.var << 2) ^ 1]);
+          else if (
+            (rel1.op === "<" && comp > 0) ||
+            (rel1.op === ">" && comp < 0)
+          )
+            dcSet.push([(rel1.var << 2) ^ 1, (rel2.var << 2) ^ 3]);
+        } else if (rel1.op === "=" || rel2.op === "=") {
+          let r1 = rel1,
+            r2 = rel2,
+            cmp = comp;
+          if (rel1.op !== "=") {
+            r2 = rel1;
+            r1 = rel2;
+            cmp = -comp;
+          }
+          if (r2.op === ">" || r2.op === "<") {
+            if (cmp === 0) {
+              dcSet.push([(r1.var << 2) ^ 3, (r2.var << 2) ^ 3]);
+            } else {
+              if ((cmp > 0 && r2.op === ">") || (cmp < 0 && r2.op === "<"))
+                dcSet.push([(r1.var << 2) ^ 3, (r2.var << 2) ^ 1]);
+              else dcSet.push([(r1.var << 2) ^ 3, (r2.var << 2) ^ 3]);
+            }
+          }
+        } else if (
+          [">", "<"].includes(rel1.op) &&
+          [">", "<"].includes(rel2.op)
+        ) {
+          if ((rel1.op === ">" && comp < 0) || (rel1.op === "<" && comp > 0))
+            dcSet.push([(rel1.var << 2) ^ 1, (rel2.var << 2) ^ 1]);
+          else dcSet.push([(rel1.var << 2) ^ 3, (rel2.var << 2) ^ 3]);
+        }
+      }
+    }
   }
+
+  return { dcSet, isNull };
 }
 
 function sanitizeMinterms(
   minterms: number[][],
-  context: SynthContext
+  isNull: Map<number, number>
 ): number[][] {
   const res = [] as number[][];
   loop: for (const m of minterms) {
@@ -1025,20 +999,13 @@ function sanitizeMinterms(
     const perms: number[][] = [];
     for (const [k, v] of merged) {
       if (v === 0b1010) continue loop;
-      const isNullVars = context.getIsNullRelations(k);
+      const n = isNull.get(k) << 2;
       const t = k << 2;
-      if (v === 0b0101) {
-        if (isNullVars.length === 1) minterm.push((isNullVars[0] << 2) ^ 3);
-        else perms.push(isNullVars.map((n) => (n << 2) ^ 3));
-      } else if (v === 0b0001) {
-        perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 3]);
-      } else if (v === 0b0100) {
-        perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 1]);
-      } else if (v & 0b1000) {
-        minterm.push(t ^ 3);
-      } else if (v & 0b0010) {
-        minterm.push(t ^ 1);
-      }
+      if (v === 0b0101) minterm.push(n ^ 3);
+      else if (v === 0b0001) perms.push([n ^ 3, t ^ 3]);
+      else if (v === 0b0100) perms.push([n ^ 3, t ^ 1]);
+      else if (v & 0b1000) minterm.push(t ^ 3);
+      else if (v & 0b0010) minterm.push(t ^ 1);
     }
     let ms = [minterm];
     while (perms.length) {
@@ -1053,13 +1020,17 @@ function sanitizeMinterms(
 }
 
 function boolExprSynthToExpression(boolExpr: BoolExprSynth): Expression {
-  const context = new SynthContext();
+  const context: Map<string, number> = new Map();
   let minterms = boolExpr.true(context);
-  minterms = sanitizeMinterms(minterms, context);
-  const canRaise = getCanRaiseCallback(context);
-  const dcSet = context.getDcSet(minterms);
+  const variables = new Map(
+    Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
+  );
+  const { dcSet, isNull } = generateDcSetAndIsNull(variables);
+  minterms = sanitizeMinterms(minterms, isNull);
+
+  const canRaise = getCanRaiseCallback(isNull);
   minterms = espresso(minterms, dcSet, { canRaise });
-  return sopToExpression(minterms, context);
+  return sopToExpression(minterms, variables);
 }
 
 function mapCallback(exp: Expression | BoolExprSynth): Expression {
@@ -1072,22 +1043,22 @@ function mapCallback(exp: Expression | BoolExprSynth): Expression {
 
   const op = exp[0];
   if (op === "IS NULL")
-    return new IsNullSynth(toBoolExprSynth(exp[1])) as unknown as Expression;
+    return (new IsNullSynth(toBoolExprSynth(exp[1])) as unknown) as Expression;
 
   if (op === "IS NOT NULL") {
-    return new NotSynth(
+    return (new NotSynth(
       new IsNullSynth(toBoolExprSynth(exp[1]))
-    ) as unknown as Expression;
+    ) as unknown) as Expression;
   } else if (op === "NOT") {
-    return new NotSynth(toBoolExprSynth(exp[1])) as unknown as Expression;
+    return (new NotSynth(toBoolExprSynth(exp[1])) as unknown) as Expression;
   } else if (op === "OR") {
-    return new OrSynth(
+    return (new OrSynth(
       ...exp.slice(1).map((a) => toBoolExprSynth(a))
-    ) as unknown as Expression;
+    ) as unknown) as Expression;
   } else if (op === "AND") {
-    return new AndSynth(
+    return (new AndSynth(
       ...exp.slice(1).map((a) => toBoolExprSynth(a))
-    ) as unknown as Expression;
+    ) as unknown) as Expression;
   }
   for (let i = 1; i < exp.length; ++i) {
     if (exp[i] instanceof BoolExprSynth) {
@@ -1103,20 +1074,19 @@ function mapCallback(exp: Expression | BoolExprSynth): Expression {
 }
 
 function getCanRaiseCallback(
-  context: SynthContext
+  isNull: Map<number, number>
 ): (idx: number, set: Set<number>) => boolean {
   return (idx: number, set: Set<number>): boolean => {
     const i = idx >> 2;
-    const vars = context.getIsNullRelations(i);
-    if (!context.isIsNull(i)) {
+    const n = isNull.get(i);
+    if (n != null) {
       if (!(idx & 1)) return true;
-      for (const n of vars) if (set.has((n << 2) ^ 3)) return true;
-      if (set.has(idx ^ 3)) return false;
-      for (const n of vars) if (!set.has((n << 2) ^ 2)) return false;
-      return true;
+      if (set.has((n << 2) ^ 2)) return true;
+      if (set.has((n << 2) ^ 3)) return true;
+      return !set.has(idx ^ 3);
     }
-
-    for (const k of vars) {
+    for (const [k, v] of isNull) {
+      if (v !== i) continue;
       if (set.has((k << 2) ^ 1)) continue;
       if (set.has((k << 2) ^ 3)) continue;
       if (set.has((k << 2) ^ 0)) return false;
@@ -1131,31 +1101,33 @@ export function minimize(expr: Expression, boolean = false): Expression {
   expr = map(expr, mapCallback);
   if (Array.isArray(expr) && expr[0] === "CASE") {
     if (!boolean) {
-      const context = new SynthContext();
+      const context: Map<string, number> = new Map();
       const whens = expr.filter((e, i) => i % 2).map((e) => e.true(context));
-      const caseDcSet: Minterm[] = [];
-      const canRaise = getCanRaiseCallback(context);
+      const variables = new Map(
+        Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
+      );
+      const { dcSet, isNull } = generateDcSetAndIsNull(variables);
+      const canRaise = getCanRaiseCallback(isNull);
       const res: Expression = ["CASE"];
       for (let i = 1; i < expr.length; i += 2) {
-        let minterms = sanitizeMinterms(whens[(i - 1) / 2], context);
-        const dcSet = context.getDcSet(minterms.concat(caseDcSet));
+        let minterms = sanitizeMinterms(whens[(i - 1) / 2], isNull);
         minterms = espresso(minterms, dcSet, { canRaise });
         if (!minterms.length) continue;
-        const w = sopToExpression(minterms, context);
+        const w = sopToExpression(minterms, variables);
         let t = expr[i + 1];
         if (t instanceof BoolExprSynth) t = boolExprSynthToExpression(t);
         res.push(w, t);
         if (w === true) break;
-        caseDcSet.push(...minterms);
+        dcSet.push(...minterms);
       }
       while (res[res.length - 1] == null) res.splice(-2);
       if (res.length < 3) return null;
       return res;
     }
-    expr = toBoolExprSynth(expr) as unknown as Expression;
+    expr = (toBoolExprSynth(expr) as unknown) as Expression;
   }
 
-  if (boolean) expr = toBoolExprSynth(expr) as unknown as Expression;
+  if (boolean) expr = (toBoolExprSynth(expr) as unknown) as Expression;
 
   if (expr instanceof BoolExprSynth) expr = boolExprSynthToExpression(expr);
   return expr;
@@ -1173,7 +1145,7 @@ export function unionDiff(
 
   if (!expr1) {
     if (Array.isArray(expr2) && expr2[0] === "CASE")
-      expr2 = toBoolExprSynth(expr2) as unknown as Expression;
+      expr2 = (toBoolExprSynth(expr2) as unknown) as Expression;
     if (expr2 instanceof BoolExprSynth)
       expr2 = boolExprSynthToExpression(expr2);
     return [expr2, expr2];
@@ -1184,37 +1156,35 @@ export function unionDiff(
   expr1 = map(expr1, mapCallback);
   const b1 = toBoolExprSynth(expr1);
 
-  const context = new SynthContext();
+  const context: Map<string, number> = new Map();
 
   const expr2Minterms = b2.true(context);
   const expr1Minterms = b1.true(context);
   const expr1NullMinterms = b1.null(context);
   const expr1FalseMinterms = b1.false(context);
+  const variables = new Map(
+    Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
+  );
+  const { dcSet, isNull } = generateDcSetAndIsNull(variables);
 
   const unionMinterms = sanitizeMinterms(
     [...expr1Minterms, ...expr2Minterms],
-    context
+    isNull
   );
-
   const diffMinterms = sanitizeMinterms(
     complement([
       ...complement([...expr1NullMinterms, ...expr1FalseMinterms]),
       ...complement(expr2Minterms),
     ]),
-    context
+    isNull
   );
 
-  const canRaise = getCanRaiseCallback(context);
+  const canRaise = getCanRaiseCallback(isNull);
 
-  const union = espresso(unionMinterms, context.getDcSet(unionMinterms), {
-    canRaise,
-  });
+  const union = espresso(unionMinterms, dcSet, { canRaise });
+  const diff = espresso(diffMinterms, dcSet, { canRaise });
 
-  const diff = espresso(diffMinterms, context.getDcSet(diffMinterms), {
-    canRaise,
-  });
-
-  return [sopToExpression(union, context), sopToExpression(diff, context)];
+  return [sopToExpression(union, variables), sopToExpression(diff, variables)];
 }
 
 export function covers(expr1: Expression, expr2: Expression): boolean {
@@ -1228,13 +1198,13 @@ export function covers(expr1: Expression, expr2: Expression): boolean {
   expr2 = map(expr2, mapCallback);
   const b2 = toBoolExprSynth(expr2);
 
-  const context = new SynthContext();
+  const context: Map<string, number> = new Map();
   const expr1Minterms = b1.true(context);
   const expr2Minterms = b2.true(context);
+  const variables = new Map(
+    Array.from(context).map(([k, v]) => [v, JSON.parse(k) as Expression])
+  );
+  const { dcSet } = generateDcSetAndIsNull(variables);
 
-  return tautology([
-    ...complement(expr2Minterms),
-    ...context.getDcSet(),
-    ...expr1Minterms,
-  ]);
+  return tautology([...complement(expr2Minterms), ...dcSet, ...expr1Minterms]);
 }

lib/common/expression-parser.ts

@@ -29,7 +29,7 @@ function interpretEscapes(str): string {
     r: "\r",
     t: "\t",
   };
-  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/g, (_, escape) => {
+  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
     const type = escape.charAt(0);
     const hex = escape.slice(1);
     if (type === "u") return String.fromCharCode(parseInt(hex, 16));
@@ -87,6 +87,8 @@ function binaryLeft(
       rest.reduce((acc, ch) => {
         const [op, another] = ch;
         if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
+        if (Array.isArray(another) && op === another[0])
+          return [op, acc].concat(another.slice(1));
         return [op, acc, another];
       }, first)
   );
@@ -285,11 +287,7 @@ const lang = parsimmon.createLanguage({
           .skip(parsimmon.optWhitespace),
         binaryLeft(
           parsimmon
-            .alt(
-              parsimmon.string("*"),
-              parsimmon.string("/"),
-              parsimmon.string("%")
-            )
+            .alt(parsimmon.string("*"), parsimmon.string("/"))
             .skip(parsimmon.optWhitespace),
           parsimmon.alt(
             r.Value,
@@ -388,7 +386,6 @@ export function stringify(exp: Expression, level = 0): string {
     "-": 31,
     "*": 32,
     "/": 32,
-    "%": 32,
   };
 
   const op = exp[0].toUpperCase();

lib/common/expression.ts

@@ -130,15 +130,6 @@ export function evaluateCallback(exp: Expression): Expression {
     } else if (exp[1] === "LOWER") {
       if (exp[2] == null) return null;
       if (!isArray(exp[2])) return toString(exp[2]).toLowerCase();
-    } else if (exp[1] === "ROUND") {
-      const n = exp[2];
-      const p = exp.length > 3 ? exp[3] : 0;
-      if (n == null || p == null) return null;
-      if (!isArray(n) && !isArray(p)) {
-        const d = 10 ** p;
-        const m = n * d * (1 + Number.EPSILON);
-        return Math.round(m) / d;
-      }
     }
   } else if (exp[0] === "PARAM") {
     if (exp[1] == null) return null;
@@ -178,11 +169,11 @@ export function evaluateCallback(exp: Expression): Expression {
   } else if (exp[0] === "IS NULL") {
     if (isArray(exp[1])) return exp;
     else if (exp[1] == null) return true;
-    else return false;
+    else return null;
   } else if (exp[0] === "IS NOT NULL") {
     if (isArray(exp[1])) return exp;
     else if (exp[1] != null) return true;
-    else return false;
+    else return null;
   } else if (exp[0] === "LIKE") {
     if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
     else if (
@@ -244,12 +235,9 @@ export function evaluateCallback(exp: Expression): Expression {
   } else if (exp[0] === "/") {
     return reduce(exp, (a, b, i) => {
       if (a == null || b == null) return null;
-      if (isArray(a) || isArray(b)) return REDUCE_SKIP;
-      const n1 = toNumber(a);
-      const n2 = toNumber(b);
-      if (i !== 0) return n1 * n2;
-      if (n2 === 0) return null;
-      return n1 / n2;
+      if (!isArray(a) && !isArray(b))
+        return i === 0 ? toNumber(a) / toNumber(b) : toNumber(a) * toNumber(b);
+      return REDUCE_SKIP;
     });
   } else if (exp[0] === "+") {
     return reduce(exp, (a, b) => {
@@ -264,15 +252,6 @@ export function evaluateCallback(exp: Expression): Expression {
         return i === 0 ? toNumber(a) - toNumber(b) : toNumber(a) + toNumber(b);
       return REDUCE_SKIP;
     });
-  } else if (exp[0] === "%") {
-    return reduce(exp, (a, b, i) => {
-      if (a == null || b == null) return null;
-      if (isArray(a) || isArray(b) || i !== 0) return REDUCE_SKIP;
-      const n1 = toNumber(a);
-      const n2 = Math.trunc(toNumber(b));
-      if (n2 === 0) return null;
-      return n1 % n2;
-    });
   } else if (exp[0] === "||") {
     return reduce(exp, (a, b) => {
       if (a == null || b == null) return null;
@@ -346,7 +325,15 @@ export async function evaluateAsync(
     if (!isArray(e)) return e;
 
     if (e[0] === "FUNC") {
-      if (e[1] === "NOW") if (now) return now;
+      if (e[1] === "NOW") {
+        if (now) return now;
+      } else if (e[1] === "UPPER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
+      } else if (e[1] === "LOWER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
+      }
     } else if (e[0] === "PARAM") {
       if (e[1] == null) return null;
       if (obj && !isArray(e[1])) {

lib/common/memoize.ts

@@ -47,18 +47,9 @@ export default function memoize<T extends (...args: any[]) => any>(func: T): T {
     if (cache1.has(key)) return cache1.get(key);
 
     let r;
-    if (cache2.has(key)) {
-      cache1.set(key, (r = cache2.get(key)));
-    } else {
-      cache1.set(key, (r = func(...args)));
-      // Evict rejected promises
-      if (r instanceof Promise) {
-        r.catch(() => {
-          cache1.delete(key);
-          cache2.delete(key);
-        });
-      }
-    }
+    if (cache2.has(key)) r = cache2.get(key);
+    else r = func(...args);
+    cache1.set(key, r);
     return r;
   }) as any;
 }

lib/common/path-set.ts

@@ -20,9 +20,9 @@
 import Path from "./path";
 
 export default class PathSet {
-  private declare lengthIndex: Set<Path>[];
-  private declare fragmentIndex: Map<string, Set<Path>>[];
-  private declare stringIndex: Map<string, Path>;
+  private lengthIndex: Set<Path>[];
+  private fragmentIndex: Map<string, Set<Path>>[];
+  private stringIndex: Map<string, Path>;
 
   public constructor() {
     this.lengthIndex = [];

lib/common/path.ts

@@ -23,102 +23,44 @@ type Alias = [Path, string][];
 let cache1 = new Map<string, Path>();
 let cache2 = new Map<string, Path>();
 
-const CHAR_SPACE = 32;
-const CHAR_BACKSLASH = 92;
-const CHAR_COLON = 58;
-const CHAR_QUOTE = 34;
-const CHAR_OPEN_BRACKET = 91;
-const CHAR_CLOSE_BRACKET = 93;
-const CHAR_ASTERISK = 42;
-const CHAR_COMMA = 44;
-const CHAR_DOT = 46;
-
-function charCodeAt(str: string, index: number): number {
-  if (index >= str.length) return 0;
-  return str.charCodeAt(index);
-}
-
-function legalChar(c: number): boolean {
-  return (
-    (c >= 97 && c <= 122) ||
-    (c >= 65 && c <= 90) ||
-    (c >= 48 && c <= 57) ||
-    c === 95 ||
-    c === 45
-  );
-}
-
 export default class Path {
-  public declare readonly segments: Segments;
-  public declare readonly wildcard: number;
-  public declare readonly alias: number;
-  protected declare _string: string;
-  protected declare _stringIndex: number[];
-
-  protected static parseAliasValue(
-    pattern: string,
-    index: number
-  ): { index: number; value: string } {
-    let i = index;
-    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
-    if (charCodeAt(pattern, i) === CHAR_QUOTE) {
-      for (let j = i + 1; j < pattern.length; ++j) {
-        if (
-          pattern.charCodeAt(j) === CHAR_QUOTE &&
-          pattern.charCodeAt(j - 1) !== CHAR_BACKSLASH
-        ) {
-          try {
-            ++j;
-            const v = JSON.parse(pattern.slice(i, j));
-            return { index: j, value: v };
-          } catch {
-            return { index, value: null };
-          }
-        }
-      }
-      return { index, value: null };
-    }
-
-    for (; i < pattern.length; ++i) {
-      const c = pattern.charCodeAt(i);
-      if (c === CHAR_CLOSE_BRACKET || c === CHAR_COMMA) break;
-    }
-
-    return { index: i, value: pattern.slice(index, i).trim() };
-  }
-
-  protected static parseAliasPath(
-    pattern: string,
-    index: number
-  ): { index: number; path: Path } {
-    let i = index;
-    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
-    const { index: idx, segments } = Path.parsePath(pattern, i);
-    if (idx === i) return { index, path: null };
-    i = idx;
-    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
-    if (charCodeAt(pattern, i) !== CHAR_COLON) return { index, path: null };
-    return { index: i + 1, path: new Path(segments) };
-  }
+  public readonly segments: Segments;
+  public readonly wildcard: number;
+  public readonly alias: number;
+  protected _string: string;
+  protected _stringIndex: number[];
 
   protected static parseAlias(
     pattern: string,
     index: number
   ): { index: number; alias: Alias } {
     const alias: Alias = [];
-    let i = index;
-    for (;;) {
-      let path: Path;
-      let value: string;
-      ({ index: i, path } = Path.parseAliasPath(pattern, i));
-      if (!path) break;
-      ({ index: i, value } = Path.parseAliasValue(pattern, i));
-      if (value == null) break;
-      alias.push([path, value]);
-      index = i;
+    while (index < pattern.length && pattern[index] !== "]") {
+      const { index: idx, segments } = Path.parsePath(pattern, index);
+      let j = (index = idx + 1);
+      while (pattern[j] !== "]" && pattern[j] !== ",") {
+        if (pattern[j] === '"' && index === j) {
+          ++j;
+          while (pattern[j] !== '"' || pattern[j - 1] === "\\") {
+            if (++j >= pattern.length)
+              throw new Error("Invalid alias expression");
+          }
+        }
+        if (++j >= pattern.length) throw new Error("Invalid alias expression");
+      }
 
-      if (charCodeAt(pattern, index) !== CHAR_COMMA) break;
-      ++i;
+      let value = pattern.slice(index, j).trim();
+      index = j;
+      if (value[0] === '"') {
+        try {
+          value = JSON.parse(value);
+        } catch (error) {
+          throw new Error("Invalid alias expression");
+        }
+      }
+
+      alias.push([new Path(segments), value]);
+      if (pattern[index] === ",") ++index;
     }
 
     // Ensure identical expressions have idential string representation
@@ -139,26 +81,30 @@ export default class Path {
     index: number
   ): { index: number; segments: Segments } {
     const segments = [];
+    // Colon separator is needed for parseAlias
+    if (index < pattern.length && pattern[index] !== ":") {
+      for (;;) {
+        if (pattern[index] === "[") {
+          const { index: idx, alias } = Path.parseAlias(pattern, index + 1);
+          index = idx + 1;
+          segments.push(alias);
+        } else {
+          const j = index;
+          while (
+            index < pattern.length &&
+            pattern[index] !== ":" &&
+            pattern[index] !== "."
+          )
+            ++index;
+          const s = pattern.slice(j, index).trim();
+          if (!s) throw new Error("Invalid parameter path");
+          segments.push(s);
+        }
 
-    let segStart = index;
-    let wildcard = -1;
-    for (let i = index; i <= pattern.length; ++i) {
-      const c = charCodeAt(pattern, i);
-      if (c === CHAR_OPEN_BRACKET && i === segStart) {
-        const { index: idx, alias } = Path.parseAlias(pattern, i + 1);
-        if (charCodeAt(pattern, idx) !== CHAR_CLOSE_BRACKET) break;
-        segments.push(alias);
-        index = i = idx + 1;
-        segStart = i + 1;
-      } else if (c === CHAR_ASTERISK && i === segStart) {
-        wildcard = i + 1;
-      } else if (!legalChar(c) || i === wildcard) {
-        if (i === segStart) break;
-        const s = pattern.slice(segStart, i);
-        segments.push(s);
-        index = i;
-        if (c !== CHAR_DOT) break;
-        segStart = i + 1;
+        if (index >= pattern.length || pattern[index] === ":") break;
+        else if (pattern[index] !== ".")
+          throw new Error("Invalid alias expression");
+        ++index;
       }
     }
 
@@ -197,8 +143,7 @@ export default class Path {
     if (!path) {
       path = cache2.get(str);
       if (!path) {
-        const { index, segments } = Path.parsePath(str, 0);
-        if (index < str.length) throw new Error("Invalid parameter path");
+        const { segments } = Path.parsePath(str, 0);
         path = new Path(segments);
         if (path.toString() !== str) cache1.set(path.toString(), path);
       }

lib/common/yaml.ts

@@ -1,218 +0,0 @@
-const LINE_WIDTH = 80;
-const INDENTATION = "  ";
-
-const STRING_RESERVED = new Set([
-  "true",
-  "True",
-  "TRUE",
-  "false",
-  "False",
-  "FALSE",
-  "null",
-  "Null",
-  "NULL",
-]);
-
-function isPrintable(str: string): boolean {
-  return !/[^\t\n\x20-\x7e\x85\u{a0}-\u{d7ff}\u{e000}-\u{fffd}\u{10000}-\u{10ffff}]/u.test(
-    str
-  );
-}
-
-function stringifyKey(str: string): string {
-  if (!str || !isPrintable(str)) return JSON.stringify(str);
-  if (/^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|[\n,[\]{}]|\s$/.test(str))
-    return JSON.stringify(str);
-  return str;
-}
-
-function foldString(str: string): string[] {
-  if (str.length <= LINE_WIDTH) return [str];
-  if (str.startsWith(" ")) return [str];
-  const lines: string[] = [];
-
-  let idx = 0;
-  let cand = 0;
-  for (let i = 1; i < str.length - 1; ++i) {
-    if (str[i] !== " ") continue;
-
-    if (str[i + 1] === " ") {
-      i += 2;
-      while (str[i] === " ") ++i;
-      continue;
-    }
-
-    if (i <= idx + LINE_WIDTH) {
-      cand = i;
-      continue;
-    }
-
-    const c = cand > idx ? cand : i;
-    lines.push(str.slice(idx, c));
-    idx = c + 1;
-    cand = i;
-  }
-
-  if (cand > idx && str.length > idx + LINE_WIDTH) {
-    lines.push(str.slice(idx, cand));
-    idx = cand + 1;
-  }
-
-  lines.push(str.slice(idx));
-
-  return lines;
-}
-
-function stringifyString(str: string, res: string[], prefix1, prefix2): void {
-  if (/^\s*$/.test(str) || STRING_RESERVED.has(str) || !isPrintable(str)) {
-    res.push(prefix1 + JSON.stringify(str));
-    return;
-  }
-
-  if (!prefix2) prefix2 = INDENTATION;
-
-  const lines = str.split("\n");
-  if (lines.length > 1) {
-    let idt = "";
-    let chmp = "-";
-    if ((lines.find((l) => l) || "").startsWith(" "))
-      idt = `${INDENTATION.length}`;
-
-    if (!lines[lines.length - 1]) {
-      lines.pop();
-      if (lines[lines.length - 1]) chmp = "";
-      else chmp = "+";
-    }
-
-    if (/^\s+$/.test(lines[lines.length - 1])) {
-      res.push(prefix1 + JSON.stringify(str));
-      return;
-    }
-
-    let isFolded = false;
-    const folded = lines.map((l) => {
-      const ls = foldString(l);
-      if (ls.length > 1) isFolded = true;
-      return ls;
-    });
-
-    if (!isFolded) {
-      res.push(
-        `${prefix1}|${idt}${chmp}`,
-        ...lines.map((l) => (l ? prefix2 + l : l))
-      );
-      return;
-    }
-
-    res.push(`${prefix1}>${idt}${chmp}`);
-    res.push(...folded[0].map((f) => prefix2 + f));
-    for (let i = 1; i < folded.length; ++i) {
-      const prevLine = folded[i - 1][0];
-      if (prevLine && !folded[i - 1][0].startsWith(" ")) res.push("");
-      res.push(...folded[i].map((f) => prefix2 + f));
-    }
-    return;
-  }
-
-  if (
-    /^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|\s$/.test(str) ||
-    parseFloat(str) === +str
-  ) {
-    res.push(prefix1 + JSON.stringify(str));
-    return;
-  }
-
-  res.push(prefix1 + str);
-}
-
-function stringifyAny(
-  obj: unknown,
-  res: string[],
-  prefix1 = "",
-  prefix2 = ""
-): void {
-  if (obj == null) {
-    res.push(`${prefix1}null`);
-    return;
-  }
-  if (typeof obj === "number" || typeof obj === "boolean") {
-    res.push(`${prefix1}${JSON.stringify(obj)}`);
-    return;
-  }
-  if (obj instanceof Date) {
-    res.push(`${prefix1}${obj.toJSON()}`);
-    return;
-  }
-
-  if (typeof obj === "string") {
-    stringifyString(obj, res, prefix1, prefix2);
-    return;
-  }
-
-  if (Array.isArray(obj)) {
-    if (!obj.length) {
-      res.push(prefix1 + "[]");
-      return;
-    }
-
-    if (!prefix1 || prefix1.endsWith("- ")) {
-      stringifyAny(obj[0], res, prefix1 + "- ", prefix2 + INDENTATION);
-      prefix1 = prefix2 + "- ";
-      prefix2 = prefix2 + INDENTATION;
-      for (let i = 1; i < obj.length; ++i)
-        stringifyAny(obj[i], res, prefix1, prefix2);
-    } else {
-      res.push(prefix1);
-      prefix1 = prefix2 + "- ";
-      prefix2 = prefix2 + INDENTATION;
-      for (let i = 0; i < obj.length; ++i)
-        stringifyAny(obj[i], res, prefix1, prefix2);
-    }
-    return;
-  }
-
-  const entries = Object.entries(obj).filter((e) => e[1] !== undefined);
-
-  if (!entries.length) {
-    res.push(prefix1 + "{}");
-    return;
-  }
-
-  if (!prefix1 || prefix1.endsWith("- ")) {
-    stringifyAny(
-      entries[0][1],
-      res,
-      prefix1 + `${stringifyKey(entries[0][0])}: `,
-      prefix2 + INDENTATION
-    );
-    prefix1 = prefix2;
-    prefix2 = prefix2 + INDENTATION;
-    for (let i = 1; i < entries.length; ++i) {
-      stringifyAny(
-        entries[i][1],
-        res,
-        prefix1 + `${stringifyKey(entries[i][0])}: `,
-        prefix2
-      );
-    }
-  } else {
-    res.push(prefix1);
-    prefix1 = prefix2;
-    prefix2 = prefix2 + INDENTATION;
-    for (let i = 0; i < entries.length; ++i) {
-      stringifyAny(
-        entries[i][1],
-        res,
-        prefix1 + `${stringifyKey(entries[i][0])}: `,
-        prefix2
-      );
-    }
-  }
-}
-
-export function stringify(obj: unknown): string {
-  if (obj === undefined) return undefined;
-  const lines: string[] = [];
-  stringifyAny(obj, lines);
-  return lines.join("\n") + "\n";
-}

lib/config.ts

@@ -43,7 +43,7 @@ const options = {
 
   CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
   CWMP_PORT: { type: "int", default: 7547 },
-  CWMP_INTERFACE: { type: "string", default: "::" },
+  CWMP_INTERFACE: { type: "string", default: "0.0.0.0" },
   CWMP_SSL_CERT: { type: "string", default: "" },
   CWMP_SSL_KEY: { type: "string", default: "" },
   CWMP_LOG_FILE: { type: "path", default: "" },
@@ -51,7 +51,7 @@ const options = {
 
   NBI_WORKER_PROCESSES: { type: "int", default: 0 },
   NBI_PORT: { type: "int", default: 7557 },
-  NBI_INTERFACE: { type: "string", default: "::" },
+  NBI_INTERFACE: { type: "string", default: "0.0.0.0" },
   NBI_SSL_CERT: { type: "string", default: "" },
   NBI_SSL_KEY: { type: "string", default: "" },
   NBI_LOG_FILE: { type: "path", default: "" },
@@ -59,7 +59,7 @@ const options = {
 
   FS_WORKER_PROCESSES: { type: "int", default: 0 },
   FS_PORT: { type: "int", default: 7567 },
-  FS_INTERFACE: { type: "string", default: "::" },
+  FS_INTERFACE: { type: "string", default: "0.0.0.0" },
   FS_SSL_CERT: { type: "string", default: "" },
   FS_SSL_KEY: { type: "string", default: "" },
   FS_URL_PREFIX: { type: "string", default: "" },
@@ -68,7 +68,7 @@ const options = {
 
   UI_WORKER_PROCESSES: { type: "int", default: 0 },
   UI_PORT: { type: "int", default: 3000 },
-  UI_INTERFACE: { type: "string", default: "::" },
+  UI_INTERFACE: { type: "string", default: "0.0.0.0" },
   UI_SSL_CERT: { type: "string", default: "" },
   UI_SSL_KEY: { type: "string", default: "" },
   UI_LOG_FILE: { type: "path", default: "" },
@@ -101,6 +101,9 @@ const options = {
 
   // Should probably never be changed
   DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
+
+  XMPP_JID: { type: "string", default: "" },
+  XMPP_PASSWORD: { type: "string", default: "" },
 };
 
 const allConfig: { [name: string]: string | number } = {};

lib/connection-request.ts

@@ -19,13 +19,17 @@
 
 import * as crypto from "crypto";
 import * as dgram from "dgram";
-import { URL } from "url";
+import { parse } from "url";
 import * as http from "http";
 import { evaluateAsync } from "./common/expression";
 import { Expression } from "./types";
 import * as auth from "./auth";
 import * as extensions from "./extensions";
 import * as debug from "./debug";
+import XmppClient from "./xmpp-client";
+import * as config from "../lib/config";
+import { encodeEntities, parseAttrs, Element } from "./xml-parser";
+import * as logger from "../lib/logger";
 
 async function extractAuth(
   exp: Expression,
@@ -61,26 +65,24 @@ async function extractAuth(
 }
 
 function httpGet(
-  url: URL,
   options: http.RequestOptions,
   _debug: boolean,
   deviceId: string
 ): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
   return new Promise((resolve, reject) => {
     const req = http
-      .get(url, options, (res) => {
+      .get(options, (res) => {
         res.resume();
         resolve({ statusCode: res.statusCode, headers: res.headers });
         if (_debug) {
-          debug.outgoingHttpRequest(req, deviceId, "GET", url, null);
+          debug.outgoingHttpRequest(req, deviceId, options, null);
           debug.incomingHttpResponse(res, deviceId, null);
         }
       })
       .on("error", (err) => {
         req.destroy();
         reject(err);
-        if (_debug)
-          debug.outgoingHttpRequestError(req, deviceId, "GET", url, err);
+        if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
       })
       .on("timeout", () => {
         req.destroy();
@@ -96,13 +98,15 @@ export async function httpConnectionRequest(
   _debug: boolean,
   deviceId: string
 ): Promise<string> {
-  const url = new URL(address);
-  if (url.protocol !== "http:")
+  const options: http.RequestOptions = parse(address);
+  if (options.protocol !== "http:")
     return "Invalid connection request URL or protocol";
 
-  const options: http.RequestOptions = {
-    agent: new http.Agent({ maxSockets: 1, keepAlive: true, timeout: timeout }),
-  };
+  options.agent = new http.Agent({
+    maxSockets: 1,
+    keepAlive: true,
+    timeout: timeout,
+  });
 
   let authHeader: Record<string, string>;
   let username: string;
@@ -129,7 +133,7 @@ export async function httpConnectionRequest(
               Authorization: auth.solveDigest(
                 username,
                 password,
-                url.pathname + url.search,
+                options.path,
                 "GET",
                 null,
                 authHeader
@@ -145,12 +149,12 @@ export async function httpConnectionRequest(
 
     let res: { statusCode: number; headers: http.IncomingHttpHeaders };
     try {
-      res = await httpGet(url, opts, _debug, deviceId);
+      res = await httpGet(opts, _debug, deviceId);
     } catch (err) {
       // Workaround for some devices unexpectedly closing the connection
       if (authHeader) {
         try {
-          res = await httpGet(url, opts, _debug, deviceId);
+          res = await httpGet(opts, _debug, deviceId);
         } catch (err) {
           return `Connection request error: ${err.message}`;
         }
@@ -169,13 +173,9 @@ export async function httpConnectionRequest(
     if (res.statusCode === 503) return "Device is offline";
 
     if (res.statusCode === 401 && res.headers["www-authenticate"]) {
-      try {
-        authHeader = auth.parseWwwAuthenticateHeader(
-          res.headers["www-authenticate"]
-        );
-      } catch (err) {
-        return "Connection request error: Error parsing www-authenticate header";
-      }
+      authHeader = auth.parseWwwAuthenticateHeader(
+        res.headers["www-authenticate"]
+      );
       [username, password, authExp] = await extractAuth(authExp, false);
     } else {
       return `Connection request error: Unexpected status code ${res.statusCode}`;
@@ -186,13 +186,14 @@ export async function httpConnectionRequest(
 }
 
 export async function udpConnectionRequest(
-  host: string,
-  port: number,
+  address: string,
   authExp: Expression,
   sourcePort = 0,
   _debug: boolean,
   deviceId: string
 ): Promise<void> {
+  const [host, portStr] = address.split(":", 2);
+  const port = portStr ? parseInt(portStr) : 80;
   const now = Date.now();
 
   const client = dgram.createSocket({ type: "udp4", reuseAddr: true });
@@ -219,12 +220,12 @@ export async function udpConnectionRequest(
       .createHmac("sha1", password)
       .update(`${ts}${id}${username}${cn}`)
       .digest("hex");
-    const uri = `http://${host}:${port}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
-    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${host}:${port}\r\n\r\n`;
+    const uri = `http://${address}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
+    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${address}\r\n\r\n`;
     const message = Buffer.from(msg);
 
     for (let i = 0; i < 3; ++i) {
-      await new Promise<void>((resolve, reject) => {
+      await new Promise((resolve, reject) => {
         client.send(message, 0, message.length, port, host, (err: Error) => {
           if (err) reject(err);
           else resolve();
@@ -237,3 +238,84 @@ export async function udpConnectionRequest(
   }
   client.close();
 }
+
+const XMPP_JID = config.get("XMPP_JID") as string;
+const XMPP_PASSWORD = config.get("XMPP_PASSWORD") as string;
+const XMPP_RESOURCE = crypto.randomBytes(8).toString("hex");
+
+let xmppClient: XmppClient;
+
+function xmppClientOnError(err: Error): void {
+  xmppClient = null;
+  logger.error({
+    message: "XMPP exception",
+    exception: err,
+    pid: process.pid,
+  });
+}
+
+function xmppClientOnClose(): void {
+  xmppClient = null;
+}
+
+export async function xmppConnectionRequest(
+  jid: string,
+  authExp: Expression,
+  timeout: number,
+  _debug: boolean,
+  deviceId: string
+): Promise<string> {
+  if (!xmppClient) {
+    const [host, username] = XMPP_JID.split("@").reverse();
+    xmppClient = await XmppClient.connect({
+      host,
+      username,
+      resource: XMPP_RESOURCE,
+      password: XMPP_PASSWORD,
+      timeout: 120000,
+    });
+    xmppClient.on("error", xmppClientOnError);
+    xmppClient.on("close", xmppClientOnClose);
+    xmppClient.unref();
+  }
+
+  let username: string;
+  let password: string;
+
+  [username, password, authExp] = await extractAuth(authExp, null);
+  while (username != null && password != null) {
+    const msg = `<connectionRequest xmlns="urn:broadband-forum-org:cwmp:xmppConnReq-1-0"><username>${encodeEntities(
+      username
+    )}</username><password>${encodeEntities(
+      password
+    )}</password></connectionRequest>`;
+    let res: Element, rawRes: string, rawReq: string;
+    try {
+      ({ res, rawRes, rawReq } = await xmppClient.sendIqStanza(
+        XMPP_JID,
+        jid,
+        "get",
+        msg,
+        timeout
+      ));
+    } catch (err) {
+      return err.message;
+    }
+    if (_debug) {
+      debug.outgoingXmppStanza(deviceId, rawReq);
+      debug.incomingXmppStanza(deviceId, rawRes);
+    }
+    const attrs = parseAttrs(res.attrs);
+    const type = attrs.find((a) => a.name === "type");
+    if (type && type.value === "result") return "";
+    const error = res.children.find((c) => c.name === "error");
+    if (!error || !error.children[0])
+      return "Unexpected XMPP connection request response";
+    if (error.children[0].name === "service-unavailable")
+      return "Device is offline";
+    if (error.children[0].name !== "not-authorized")
+      return "Unexpected XMPP connection request response";
+    [username, password, authExp] = await extractAuth(authExp, null);
+  }
+  return "Incorrect connection request credentials";
+}

lib/cwmp.ts

@@ -45,7 +45,6 @@ import {
   InformRequest,
   Preset,
   GetRPCMethodsResponse,
-  CpeFault,
 } from "./types";
 import { IncomingMessage, ServerResponse } from "http";
 import { Readable } from "stream";
@@ -80,21 +79,17 @@ async function authenticate(
     sessionContext.cacheSnapshot,
     "cwmp.auth"
   );
-  if (authExpression == null) return true;
+  if (!authExpression) return true;
 
   let authentication;
 
   if (sessionContext.httpRequest.headers["authorization"]) {
-    try {
-      authentication = auth.parseAuthorizationHeader(
-        sessionContext.httpRequest.headers["authorization"]
-      );
-    } catch (err) {
-      return false;
-    }
+    authentication = auth.parseAuthorizationHeader(
+      sessionContext.httpRequest.headers["authorization"]
+    );
   }
 
-  if (authentication?.method === "Digest") {
+  if (authentication && authentication.method === "Digest") {
     const sessionNonce = sessionsNonces.get(sessionContext.httpRequest.socket);
 
     if (
@@ -188,8 +183,7 @@ async function writeResponse(
   }
 
   const httpResponse = sessionContext.httpResponse;
-  // Don't use httpResponse.socket as it may be null, even before end() is called
-  const connection = sessionContext.httpRequest.socket;
+  const connection = httpResponse.socket;
 
   httpResponse.setHeader("Content-Length", Buffer.byteLength(data));
   httpResponse.writeHead(res.code, res.headers);
@@ -214,7 +208,7 @@ async function writeResponse(
       sessionContext.extendLock = now + 10000;
       await cache.acquireLock(
         `cwmp_session_${sessionContext.deviceId}`,
-        sessionContext.timeout * 1000 + 15000,
+        sessionContext.timeout + 10000,
         0,
         sessionContext.sessionId
       );
@@ -436,7 +430,7 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
   const deviceEvents = {};
   for (const p of deviceData.paths.find(Path.parse("Events.*"), false, true)) {
     const attrs = deviceData.attributes.get(p);
-    if (attrs?.value && attrs.value[1][0] >= sessionContext.timestamp)
+    if (attrs && attrs.value && attrs.value[1][0] >= sessionContext.timestamp)
       deviceEvents[p.segments[1] as string] = true;
   }
 
@@ -460,7 +454,7 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
 
     if (!eventsMatch) continue;
 
-    if (preset.schedule?.schedule) {
+    if (preset.schedule && preset.schedule.schedule) {
       const r = scheduling.cron(
         sessionContext.timestamp,
         preset.schedule.schedule
@@ -499,11 +493,10 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
     defer: true,
   }));
 
-  const {
-    fault: flt,
-    rpcId: reqId,
-    rpc: acsReq,
-  } = await session.rpcRequest(sessionContext, declarations);
+  const { fault: flt, rpcId: reqId, rpc: acsReq } = await session.rpcRequest(
+    sessionContext,
+    declarations
+  );
 
   if (flt) {
     recordFault(sessionContext, flt);
@@ -575,11 +568,10 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
 
   deviceData.timestamps.dirty = 0;
   deviceData.attributes.dirty = 0;
-  const {
-    fault: fault,
-    rpcId: id,
-    rpc: acsRequest,
-  } = await session.rpcRequest(sessionContext, null);
+  const { fault: fault, rpcId: id, rpc: acsRequest } = await session.rpcRequest(
+    sessionContext,
+    null
+  );
 
   if (fault) {
     recordFault(sessionContext, fault);
@@ -609,11 +601,10 @@ async function applyPresets(sessionContext: SessionContext): Promise<void> {
 }
 
 async function nextRpc(sessionContext: SessionContext): Promise<void> {
-  const {
-    fault: fault,
-    rpcId: id,
-    rpc: acsRequest,
-  } = await session.rpcRequest(sessionContext, null);
+  const { fault: fault, rpcId: id, rpc: acsRequest } = await session.rpcRequest(
+    sessionContext,
+    null
+  );
 
   if (fault) {
     recordFault(sessionContext, fault);
@@ -735,15 +726,7 @@ async function nextRpc(sessionContext: SessionContext): Promise<void> {
       );
       break;
     default:
-      if (!sessionContext.doneTasks) sessionContext.doneTasks = [];
-      sessionContext.doneTasks.push(task._id);
-      sessionContext.tasks = sessionContext.tasks.filter((t) => t !== task);
-
-      logger.accessWarn({
-        sessionContext: sessionContext,
-        message: "Invalid task",
-        taskId: task._id,
-      });
+      throw new Error("Task name not recognized");
   }
 
   return nextRpc(sessionContext);
@@ -791,7 +774,7 @@ async function endSession(sessionContext: SessionContext): Promise<void> {
     }
   }
 
-  if (sessionContext.doneTasks?.length) {
+  if (sessionContext.doneTasks && sessionContext.doneTasks.length) {
     saveCache = true;
     promises.push(
       db.clearTasks(sessionContext.deviceId, sessionContext.doneTasks)
@@ -883,6 +866,27 @@ async function sendAcsRequest(
   return writeResponse(sessionContext, res);
 }
 
+async function getSession(
+  connection: Socket,
+  sessionId: string
+): Promise<SessionContext> {
+  let sessionContext = currentSessions.get(connection);
+  if (sessionContext) {
+    currentSessions.delete(connection);
+    return sessionContext;
+  }
+
+  if (!sessionId) return null;
+
+  await new Promise((resolve) => setTimeout(resolve, 100));
+
+  const sessionContextString = await cache.pop(`session_${sessionId}`);
+  if (!sessionContextString) return null;
+  sessionContext = await session.deserialize(sessionContextString);
+  connection.setTimeout(sessionContext.timeout);
+  return sessionContext;
+}
+
 // Only needed to prevent tree shaking from removing the remoteAddress
 // workaround in onConnection function.
 const remoteAddressWorkaround = new WeakMap<Socket, string>();
@@ -897,14 +901,6 @@ export function onConnection(socket: Socket): void {
     const sessionContext = currentSessions.get(socket);
     if (!sessionContext) return;
     currentSessions.delete(socket);
-    if (sessionContext.authState !== 2) {
-      logger.accessError({
-        message: "Authentication failure",
-        sessionContext: sessionContext,
-      });
-      return;
-    }
-
     const now = Date.now();
 
     const lastActivity = sessionContext.lastActivity;
@@ -1029,13 +1025,13 @@ async function reportBadState(sessionContext: SessionContext): Promise<void> {
     sessionContext: sessionContext,
   });
   const httpResponse = sessionContext.httpResponse;
+  currentSessions.delete(httpResponse.socket);
   const body = "Bad session state";
   httpResponse.setHeader("Content-Length", Buffer.byteLength(body));
   httpResponse.writeHead(400, { Connection: "close" });
   if (sessionContext.debug)
     debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, body);
   httpResponse.end(body);
-  if (sessionContext.state) return endSession(sessionContext);
 }
 
 async function responseUnauthorized(
@@ -1087,7 +1083,7 @@ async function processRequest(
   }
 
   if (sessionContext.state === 0) {
-    if (rpc.cpeRequest?.name !== "Inform")
+    if (!rpc.cpeRequest || rpc.cpeRequest.name !== "Inform")
       return reportBadState(sessionContext);
 
     const res = await inform(sessionContext, rpc);
@@ -1134,7 +1130,7 @@ async function processRequest(
       sessionContext.extendLock = sessionContext.timestamp + 10000;
       await cache.acquireLock(
         `cwmp_session_${sessionContext.deviceId}`,
-        sessionContext.timeout * 1000 + 15000,
+        sessionContext.timeout + 10000,
         0,
         sessionContext.sessionId
       );
@@ -1232,15 +1228,7 @@ async function processRequest(
   } else if (rpc.cpeResponse) {
     if (sessionContext.state !== 2) return reportBadState(sessionContext);
 
-    const fault = await session.rpcResponse(
-      sessionContext,
-      rpc.id,
-      rpc.cpeResponse
-    );
-    if (fault) {
-      recordFault(sessionContext, fault);
-      session.clearProvisions(sessionContext);
-    }
+    await session.rpcResponse(sessionContext, rpc.id, rpc.cpeResponse);
     return nextRpc(sessionContext);
   } else if (rpc.cpeFault) {
     if (sessionContext.state !== 2) return reportBadState(sessionContext);
@@ -1257,41 +1245,6 @@ async function processRequest(
       session.clearProvisions(sessionContext);
     }
     return nextRpc(sessionContext);
-  } else if (rpc.unknownMethod) {
-    if (sessionContext.state === 1) {
-      logger.accessWarn({
-        sessionContext: sessionContext,
-        message: "Method not supported",
-        method: rpc.unknownMethod,
-      });
-
-      const f: CpeFault = {
-        faultCode: "Server",
-        faultString: "CWMP fault",
-        detail: {
-          faultCode: "8000",
-          faultString: "Method not supported",
-        },
-      };
-
-      const res = soap.response({
-        id: rpc.id,
-        acsFault: f,
-        cwmpVersion: sessionContext.cwmpVersion,
-      });
-
-      return writeResponse(sessionContext, res);
-    } else if (sessionContext.state === 2) {
-      const fault = {
-        code: "invalid_response",
-        message: "Response name does not match request name",
-      };
-      recordFault(sessionContext, fault);
-      session.clearProvisions(sessionContext);
-      return nextRpc(sessionContext);
-    } else {
-      return reportBadState(sessionContext);
-    }
   } else {
     // CPE sent empty response
     if (sessionContext.state !== 1) return reportBadState(sessionContext);
@@ -1327,13 +1280,13 @@ export function listener(
       stats.concurrentRequests -= 1;
     })
     .catch((err) => {
-      currentSessions.delete(httpRequest.socket);
+      currentSessions.delete(httpResponse.socket);
       stats.concurrentRequests -= 1;
       setTimeout(() => {
         throw err;
       });
       try {
-        httpRequest.socket.unref();
+        httpResponse.socket.unref();
         httpResponse.writeHead(500, { Connection: "close" });
         httpResponse.end(`${err.name}: ${err.message}`);
       } catch (err) {
@@ -1342,48 +1295,6 @@ export function listener(
     });
 }
 
-async function clientError(
-  httpRequest: IncomingMessage,
-  httpResponse: ServerResponse,
-  sessionContext: SessionContext,
-  body: string,
-  msg: string
-): Promise<void> {
-  let debugEnabled = false;
-  let deviceId: string = null;
-
-  if (sessionContext) {
-    debugEnabled = sessionContext.debug;
-    deviceId = sessionContext.deviceId;
-  } else {
-    const cacheSnapshot = await localCache.getCurrentSnapshot();
-    debugEnabled = !!localCache.getConfig(
-      cacheSnapshot,
-      "cwmp.debug",
-      {
-        remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
-      },
-      Date.now(),
-      (e) => {
-        if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
-          return getRequestOrigin(httpRequest).remoteAddress;
-        return e;
-      }
-    );
-  }
-
-  httpResponse.setHeader("Content-Length", Buffer.byteLength(msg));
-  httpResponse.writeHead(400, { Connection: "close" });
-
-  if (debugEnabled) {
-    debug.incomingHttpRequest(httpRequest, deviceId, body);
-    debug.outgoingHttpResponse(httpResponse, deviceId, msg);
-  }
-
-  httpResponse.end(msg);
-  if (sessionContext?.state) await endSession(sessionContext);
-}
-
 function decodeString(buffer: Buffer, charset: string): string {
   try {
     return buffer.toString(charset as BufferEncoding);
@@ -1410,8 +1321,7 @@ async function listenerAsync(
 
   let sessionId;
   // Separation by comma is important as some devices don't comform to standard
-  const COOKIE_REGEX =
-    /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g;
+  const COOKIE_REGEX = /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g;
   let match;
   while ((match = COOKIE_REGEX.exec(httpRequest.headers.cookie)))
     if (match[1] === "session") sessionId = match[2];
@@ -1472,14 +1382,15 @@ async function listenerAsync(
   // Request aborted
   if (!body) return;
 
-  let sessionContext = currentSessions.get(httpRequest.socket);
+  const newConnection = !currentSessions.has(httpRequest.socket);
+
+  const sessionContext = await getSession(httpRequest.socket, sessionId);
 
   if (sessionContext) {
-    currentSessions.delete(httpRequest.socket);
     sessionContext.httpRequest = httpRequest;
     sessionContext.httpResponse = httpResponse;
     if (
-      (sessionContext.sessionId !== sessionId && sessionContext.state) ||
+      (newConnection && sessionContext.sessionId !== sessionId) ||
       sessionContext.lastActivity + sessionContext.timeout * 1000 < Date.now()
     ) {
       logger.accessError({
@@ -1487,14 +1398,29 @@ async function listenerAsync(
         sessionContext: sessionContext,
       });
 
-      return clientError(
-        httpRequest,
-        httpResponse,
-        sessionContext,
-        body.toString(),
-        "Invalid session"
-      );
+      const _body = "Invalid session";
+      httpResponse.setHeader("Content-Length", Buffer.byteLength(_body));
+      httpResponse.writeHead(400, { Connection: "close" });
+      if (sessionContext.debug) {
+        debug.outgoingHttpResponse(
+          httpResponse,
+          sessionContext.deviceId,
+          _body
+        );
+      }
+      httpResponse.end(_body);
+      await endSession(sessionContext);
+      return;
     }
+    if (newConnection && sessionContext.authState !== 1)
+      sessionContext.authState = 0;
+  } else if (stats.concurrentRequests > MAX_CONCURRENT_REQUESTS) {
+    // Check again just in case device included old session ID
+    // from the previous session
+    httpResponse.writeHead(503, { "Retry-after": 60, Connection: "close" });
+    httpResponse.end("503 Service Unavailable");
+    stats.droppedRequests += 1;
+    return;
   }
 
   let charset: string;
@@ -1514,16 +1440,6 @@ async function listenerAsync(
   const bodyStr = decodeString(body, charset);
 
   if (bodyStr == null) {
-    if (!sessionContext && sessionId) {
-      await new Promise((resolve) => setTimeout(resolve, 100));
-      const sessionContextString = await cache.pop(`session_${sessionId}`);
-      if (sessionContextString) {
-        sessionContext = await session.deserialize(sessionContextString);
-        sessionContext.httpRequest = httpRequest;
-        sessionContext.httpResponse = httpResponse;
-      }
-    }
-
     const msg = `Unknown encoding '${charset}'`;
     logger.accessError({
       message: "XML parse error",
@@ -1533,87 +1449,130 @@ async function listenerAsync(
         httpResponse: httpResponse,
       },
     });
-    return clientError(
-      httpRequest,
-      httpResponse,
-      sessionContext,
-      body.toString(),
-      msg
-    );
+    httpResponse.setHeader("Content-Length", Buffer.byteLength(msg));
+    httpResponse.writeHead(400, { Connection: "close" });
+    if (sessionContext) {
+      if (sessionContext.debug) {
+        debug.incomingHttpRequest(
+          httpRequest,
+          sessionContext.deviceId,
+          body.toString()
+        );
+        debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, msg);
+      }
+      await endSession(sessionContext);
+    } else {
+      const cacheSnapshot = await localCache.getCurrentSnapshot();
+      const d = !!localCache.getConfig(
+        cacheSnapshot,
+        "cwmp.debug",
+        {
+          remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
+        },
+        Date.now(),
+        (e) => {
+          if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
+            return getRequestOrigin(httpRequest).remoteAddress;
+          return e;
+        }
+      );
+      if (d) {
+        debug.incomingHttpRequest(httpRequest, null, body.toString());
+        debug.outgoingHttpResponse(httpResponse, null, msg);
+      }
+    }
+    httpResponse.end(msg);
+    return;
   }
 
   const parseWarnings = [];
   let rpc: SoapMessage;
   try {
-    rpc = soap.request(bodyStr, parseWarnings);
-  } catch (err) {
-    if (!sessionContext && sessionId) {
-      await new Promise((resolve) => setTimeout(resolve, 100));
-      const sessionContextString = await cache.pop(`session_${sessionId}`);
-      if (sessionContextString) {
-        sessionContext = await session.deserialize(sessionContextString);
-        sessionContext.httpRequest = httpRequest;
-        sessionContext.httpResponse = httpResponse;
-      }
-    }
-
+    rpc = soap.request(
+      bodyStr,
+      sessionContext ? sessionContext.cwmpVersion : null,
+      parseWarnings
+    );
+  } catch (error) {
     logger.accessError({
       message: "XML parse error",
-      parseError: err.message,
+      parseError: error.message.trim(),
       sessionContext: sessionContext || {
         httpRequest: httpRequest,
         httpResponse: httpResponse,
       },
     });
-
-    return clientError(
-      httpRequest,
-      httpResponse,
-      sessionContext,
-      bodyStr,
-      err.message
-    );
-  }
-
-  if (!sessionContext && sessionId && rpc.cpeRequest?.name !== "Inform") {
-    await new Promise((resolve) => setTimeout(resolve, 100));
-    const sessionContextString = await cache.pop(`session_${sessionId}`);
-    if (sessionContextString) {
-      sessionContext = await session.deserialize(sessionContextString);
-      sessionContext.httpRequest = httpRequest;
-      sessionContext.httpResponse = httpResponse;
-      httpRequest.socket.setTimeout(sessionContext.timeout * 1000);
-      if (sessionContext.authState !== 1) sessionContext.authState = 0;
+    httpResponse.setHeader("Content-Length", Buffer.byteLength(error.message));
+    httpResponse.writeHead(400, { Connection: "close" });
+    if (sessionContext) {
+      if (sessionContext.debug) {
+        debug.incomingHttpRequest(
+          httpRequest,
+          sessionContext.deviceId,
+          bodyStr
+        );
+        debug.outgoingHttpResponse(
+          httpResponse,
+          sessionContext.deviceId,
+          error.message
+        );
+      }
+      await endSession(sessionContext);
+    } else {
+      const cacheSnapshot = await localCache.getCurrentSnapshot();
+      const d = !!localCache.getConfig(
+        cacheSnapshot,
+        "cwmp.debug",
+        {
+          remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
+        },
+        Date.now(),
+        (e) => {
+          if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
+            return getRequestOrigin(httpRequest).remoteAddress;
+          return e;
+        }
+      );
+      if (d) {
+        debug.incomingHttpRequest(httpRequest, null, bodyStr);
+        debug.outgoingHttpResponse(httpResponse, null, error.message);
+      }
     }
+    httpResponse.end(error.message);
+    return;
   }
 
   if (sessionContext)
     return processRequest(sessionContext, rpc, parseWarnings, bodyStr);
 
-  if (rpc.cpeRequest?.name !== "Inform") {
+  if (!(rpc.cpeRequest && rpc.cpeRequest.name === "Inform")) {
     logger.accessError({
       message: "Invalid session",
-      sessionContext: {
+      sessionContext: sessionContext || {
         httpRequest: httpRequest,
         httpResponse: httpResponse,
       },
     });
-
-    return clientError(
-      httpRequest,
-      httpResponse,
-      null,
-      bodyStr,
-      "Invalid session"
+    const _body = "Invalid session";
+    httpResponse.setHeader("Content-Length", Buffer.byteLength(_body));
+    httpResponse.writeHead(400, { Connection: "close" });
+    const cacheSnapshot = await localCache.getCurrentSnapshot();
+    const d = !!localCache.getConfig(
+      cacheSnapshot,
+      "cwmp.debug",
+      {
+        remoteAddress: getRequestOrigin(httpRequest).remoteAddress,
+      },
+      Date.now(),
+      (e) => {
+        if (Array.isArray(e) && e[0] === "FUNC" && e[1] === "REMOTE_ADDRESS")
+          return getRequestOrigin(httpRequest).remoteAddress;
+        return e;
+      }
     );
-  }
-
-  if (stats.concurrentRequests > MAX_CONCURRENT_REQUESTS) {
-    // Check again just in case device included old session ID
-    // from the previous session
-    httpResponse.writeHead(503, { "Retry-after": 60, Connection: "close" });
-    httpResponse.end("503 Service Unavailable");
-    stats.droppedRequests += 1;
+    if (d)
+      debug.outgoingHttpResponse(httpResponse, sessionContext.deviceId, _body);
+    httpResponse.end(_body);
     return;
   }
 

lib/db.ts

@@ -17,9 +17,9 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { MongoClient, ObjectId, Collection, Db } from "mongodb";
+import { MongoClient, ObjectID, Collection, Db } from "mongodb";
 import { get } from "./config";
-import { decodeTag, encodeTag, escapeRegExp } from "./common";
+import { encodeTag, escapeRegExp } from "./common";
 import { parse } from "./common/expression-parser";
 import {
   DeviceData,
@@ -46,8 +46,6 @@ export let tasksCollection: Collection,
 
 let clientPromise: Promise<MongoClient>;
 
-const INVALID_PATH_SUFFIX = "__invalid";
-
 function compareAccessLists(list1: string[], list2: string[]): boolean {
   if (list1.length !== list2.length) return false;
   for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
@@ -61,7 +59,10 @@ export function onConnect(callback: (db: Db) => Promise<void>): void {
 }
 
 export async function connect(): Promise<void> {
-  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"));
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
+    useNewUrlParser: true,
+    useUnifiedTopology: true,
+  });
 
   const client = await clientPromise;
   const db = client.db();
@@ -130,12 +131,7 @@ export async function fetchDevice(
   const device = await devicesCollection.findOne({ _id: id });
   if (!device) return null;
 
-  function storeParams(
-    obj,
-    path: string,
-    pathLength: number,
-    ts: number
-  ): void {
+  function storeParams(obj, path: string, pathLength: number, ts): void {
     if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
     if (obj["_attributesTimestamp"])
       obj["_attributesTimestamp"] = +obj["_attributesTimestamp"];
@@ -166,34 +162,19 @@ export async function fetchDevice(
     if (obj["_accessList"] != null)
       attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
 
-    try {
-      res.push([Path.parse(path), t, attrs]);
-    } catch (err) {
-      // The path parser is now more strict so we might be in a situation where
-      // the database contains invalid paths from before this change So here we
-      // encode the invalid characters.
-      const splits = path.split(".");
-      splits[splits.length - 1] =
-        encodeTag(splits[splits.length - 1]) + INVALID_PATH_SUFFIX;
-      path = splits.join(".");
-      res.push([Path.parse(path), t, attrs]);
-      return;
-    }
+    res.push([Path.parse(path.slice(0, -1)), t, attrs]);
 
     for (const [k, v] of Object.entries(obj)) {
       if (!k.startsWith("_")) {
         obj["_object"] = true;
-        storeParams(v, `${path}.${k}`, pathLength + 1, obj["_timestamp"]);
+        storeParams(v, path + k + ".", pathLength + 1, obj["_timestamp"]);
       }
     }
 
-    if (obj["_object"] && obj["_timestamp"])
-      res.push([Path.parse(path + ".*"), obj["_timestamp"]]);
+    if ((obj["_object"] || !pathLength) && obj["_timestamp"])
+      res.push([Path.parse(path + "*"), obj["_timestamp"]]);
   }
 
-  const ts: number = device["_timestamp"] || 0;
-  if (ts) res.push([Path.parse("*"), ts]);
-
   for (const [k, v] of Object.entries(device)) {
     switch (k) {
       case "_lastInform":
@@ -321,11 +302,10 @@ export async function fetchDevice(
             },
           ]);
         }
-        break;
-      default:
-        if (!k.startsWith("_")) storeParams(v, k, 1, ts);
     }
   }
+
+  storeParams(device, "", 0, 0);
   return res;
 }
 
@@ -460,16 +440,7 @@ export async function saveDevice(
         break;
       default:
         if (!diff[2]) {
-          let pathStr = path.toString();
-          // Paths with that suffix are encoded and need to be decoded
-          if (pathStr.endsWith(INVALID_PATH_SUFFIX)) {
-            const splits = pathStr.split(".");
-            splits[splits.length - 1] = decodeTag(
-              splits[splits.length - 1].slice(0, 0 - INVALID_PATH_SUFFIX.length)
-            );
-            pathStr = splits.join(".");
-          }
-          update["$unset"][pathStr] = 1;
+          update["$unset"][path.toString()] = 1;
           continue;
         }
 
@@ -505,7 +476,7 @@ export async function saveDevice(
 
                 break;
               case "object":
-                if (!diff[1]?.object || object2 !== object1) {
+                if (!diff[1] || !diff[1].object || object2 !== object1) {
                   update["$set"][
                     path.length ? path.toString() + "._object" : "_object"
                   ] = !!object2;
@@ -513,7 +484,7 @@ export async function saveDevice(
 
                 break;
               case "writable":
-                if (!diff[1]?.writable || writable2 !== writable1) {
+                if (!diff[1] || !diff[1].writable || writable2 !== writable1) {
                   update["$set"][
                     path.length ? path.toString() + "._writable" : "_writable"
                   ] = !!writable2;
@@ -534,8 +505,9 @@ export async function saveDevice(
                 }
 
                 if (attributesTimestamp2 !== attributesTimestamp1) {
-                  update["$set"][path.toString() + "._attributesTimestamp"] =
-                    new Date(attributesTimestamp2);
+                  update["$set"][
+                    path.toString() + "._attributesTimestamp"
+                  ] = new Date(attributesTimestamp2);
                 }
 
                 break;
@@ -553,8 +525,9 @@ export async function saveDevice(
                 }
 
                 if (attributesTimestamp2 !== attributesTimestamp1) {
-                  update["$set"][path.toString() + "._attributesTimestamp"] =
-                    new Date(attributesTimestamp2);
+                  update["$set"][
+                    path.toString() + "._attributesTimestamp"
+                  ] = new Date(attributesTimestamp2);
                 }
             }
           }
@@ -564,7 +537,7 @@ export async function saveDevice(
           for (const attrName of Object.keys(diff[1])) {
             if (
               diff[1][attrName][1] != null &&
-              diff[2]?.[attrName]?.[1] == null
+              (!diff[2] || !diff[2][attrName] || diff[2][attrName][1] == null)
             ) {
               const p = path.length ? path.toString() + "." : "";
               update["$unset"][`${p}_${attrName}`] = 1;
@@ -618,7 +591,7 @@ export async function saveDevice(
     upsert: isNew,
   });
 
-  if (!result.matchedCount && !result.upsertedCount)
+  if (result.result.n !== 1)
     throw new Error(`Device ${deviceId} not found in database`);
 
   if (update2) {
@@ -642,7 +615,7 @@ export async function getFaults(
     delete r.device;
     r.timestamp = +r.timestamp;
     r.provisions = JSON.parse(r.provisions);
-    faults[channel] = r as SessionFault;
+    faults[channel] = r;
   }
 
   return faults;
@@ -683,13 +656,13 @@ export async function getDueTasks(
     if (task.timestamp >= timestamp) return [tasks, +task.timestamp];
     task._id = String(task._id);
 
-    tasks.push(task as Task);
+    tasks.push(task);
 
     // For API compatibility
     if (task.name === "download" && task.file) {
       let q;
-      if (ObjectId.isValid(task.file))
-        q = { _id: { $in: [task.file, new ObjectId(task.file)] } };
+      if (ObjectID.isValid(task.file))
+        q = { _id: { $in: [task.file, new ObjectID(task.file)] } };
       else q = { _id: task.file };
 
       const res = await filesCollection.find(q).toArray();
@@ -710,7 +683,7 @@ export async function clearTasks(
   taskIds: string[]
 ): Promise<void> {
   await tasksCollection.deleteMany({
-    _id: { $in: taskIds.map((id) => new ObjectId(id)) },
+    _id: { $in: taskIds.map((id) => new ObjectID(id)) },
   });
 }
 
@@ -727,14 +700,14 @@ export async function getOperations(
     delete r._id;
     // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
     if (typeof r.provisions !== "string") {
-      operations[commandKey] = r as Operation;
+      operations[commandKey] = r;
       continue;
     }
     r.timestamp = +r.timestamp;
     if (r.args) r.args = JSON.parse(r.args);
     r.provisions = JSON.parse(r.provisions);
     r.retries = JSON.parse(r.retries);
-    operations[commandKey] = r as Operation;
+    operations[commandKey] = r;
   }
   return operations;
 }
@@ -802,7 +775,7 @@ interface Permission {
 }
 
 export async function getPermissions(): Promise<Permission[]> {
-  return permissionsCollection.find().toArray() as unknown as Permission[];
+  return permissionsCollection.find().toArray();
 }
 
 interface User {
@@ -813,5 +786,5 @@ interface User {
 }
 
 export async function getUsers(): Promise<User[]> {
-  return usersCollection.find().toArray() as unknown as User[];
+  return usersCollection.find().toArray();
 }

lib/debug.ts

@@ -17,10 +17,15 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { IncomingMessage, ServerResponse, ClientRequest } from "http";
+import {
+  IncomingMessage,
+  ServerResponse,
+  ClientRequest,
+  RequestOptions,
+} from "http";
 import { Socket } from "net";
 import { appendFileSync } from "fs";
-import { stringify } from "./common/yaml";
+import * as yaml from "yaml";
 import * as config from "./config";
 
 const DEBUG_FILE = "" + config.get("DEBUG_FILE");
@@ -59,7 +64,7 @@ export function incomingHttpRequest(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -85,7 +90,7 @@ export function outgoingHttpResponse(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -94,8 +99,7 @@ export function outgoingHttpResponse(
 export function outgoingHttpRequest(
   httpRequest: ClientRequest,
   deviceId: string,
-  method: "GET" | "PUT" | "POST" | "DELETE",
-  url: URL,
+  options: RequestOptions,
   body: string
 ): void {
   if (!DEBUG_FILE) return;
@@ -107,15 +111,15 @@ export function outgoingHttpRequest(
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
-    remotePort: url.port,
-    method: method,
-    url: url.pathname + url.search,
+    remotePort: options.port,
+    method: options.method || "GET",
+    url: options.path,
     headers: httpRequest.getHeaders(),
     body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -124,8 +128,7 @@ export function outgoingHttpRequest(
 export function outgoingHttpRequestError(
   httpRequest: ClientRequest,
   deviceId: string,
-  method: "GET" | "PUT" | "POST" | "DELETE",
-  url: URL,
+  options: RequestOptions,
   err: Error
 ): void {
   if (!DEBUG_FILE) return;
@@ -133,18 +136,18 @@ export function outgoingHttpRequestError(
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
-    remoteAddress: url.hostname,
+    remoteAddress: options.hostname,
     deviceId: deviceId,
     connection: null,
-    remotePort: url.port,
-    method: method,
-    url: url.pathname + url.search,
+    remotePort: options.port,
+    method: options.method,
+    url: options.path,
     headers: httpRequest.getHeaders(),
     error: err.message,
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -170,7 +173,7 @@ export function incomingHttpResponse(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -194,7 +197,41 @@ export function outgoingUdpMessage(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function incomingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "incoming XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);

lib/device.ts

@@ -54,6 +54,13 @@ export function sanitizeParameterValue(
           else parameterValue[0] = b;
         }
         break;
+      case "xsd:string":
+      case "xsd:base64":
+      case "xsd:hexBinary":
+        if (typeof parameterValue[0] !== "string")
+          parameterValue[0] = "" + parameterValue[0];
+
+        break;
       case "xsd:int":
       case "xsd:unsignedInt":
         if (typeof parameterValue[0] !== "number") {
@@ -76,7 +83,7 @@ export function sanitizeParameterValue(
         }
         break;
       default:
-        parameterValue[0] = "" + parameterValue[0];
+        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
         break;
     }
   }
@@ -201,7 +208,7 @@ export function clear(
   timestamp = timestamp || 0;
 
   let descendantsTimestamp = timestamp;
-  if (attributes?.object) {
+  if (attributes && attributes.object) {
     if (attributes.object > descendantsTimestamp)
       descendantsTimestamp = attributes.object;
     if (!(attributes.object <= attributes.value))

lib/extensions.ts

@@ -22,14 +22,20 @@ import * as crypto from "crypto";
 import * as config from "./config";
 import { Fault } from "./types";
 import { ROOT_DIR } from "./config";
-import * as logger from "./logger";
-import readline from "readline";
 
 const TIMEOUT = +config.get("EXT_TIMEOUT");
 
 const processes: { [script: string]: ChildProcess } = {};
 const jobs = new Map();
 
+function messageHandler(message): void {
+  const func = jobs.get(message[0]);
+  if (func) {
+    jobs.delete(message[0]);
+    func({ fault: message[1], value: message[2] });
+  }
+}
+
 export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
   return new Promise((resolve) => {
     const scriptName = args[0];
@@ -39,7 +45,7 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
 
     if (!processes[scriptName]) {
       const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
-        stdio: ["ignore", "pipe", "pipe", "ipc"],
+        stdio: ["inherit", "inherit", "inherit", "ipc"],
       });
       processes[scriptName] = p;
 
@@ -62,26 +68,7 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
         if (processes[scriptName] === p) delete processes[scriptName];
       });
 
-      p.on("message", (message) => {
-        const func = jobs.get(message[0]);
-        if (func) {
-          jobs.delete(message[0]);
-          // Wait for any disconnect even to fire
-          setTimeout(() => {
-            func({ fault: message[1], value: message[2] });
-          });
-        }
-      });
-
-      const rlstdout = readline.createInterface(p.stdout);
-      rlstdout.on("line", (line) => {
-        logger.info({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
-      });
-
-      const rlstderr = readline.createInterface(p.stderr);
-      rlstderr.on("line", (line) => {
-        logger.warn({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
-      });
+      p.on("message", messageHandler);
     }
 
     setTimeout(() => {

lib/fs.ts

@@ -18,12 +18,12 @@
  */
 
 import * as url from "url";
+import * as querystring from "querystring";
 import { IncomingMessage, ServerResponse } from "http";
 import { Collection, GridFSBucket } from "mongodb";
 import { onConnect } from "./db";
 import * as logger from "./logger";
 import { getRequestOrigin } from "./forwarded";
-import memoize from "./common/memoize";
 
 let filesCollection: Collection;
 let filesBucket: GridFSBucket;
@@ -33,65 +33,43 @@ onConnect(async (db) => {
   filesBucket = new GridFSBucket(db);
 });
 
-const getFile = memoize(
-  (uploadDate: number, size: number, filename: string): Promise<Buffer> => {
-    return new Promise((resolve, reject) => {
-      const buffer = Buffer.allocUnsafe(size);
-      let i = 0;
-      const downloadStream = filesBucket.openDownloadStreamByName(filename);
-      downloadStream.on("error", reject);
-      downloadStream.on("data", (data: Buffer) => {
-        data.copy(buffer, i);
-        i += data.length;
-      });
-      downloadStream.on("end", () => {
-        if (i !== size) reject(new Error("File size mismatch"));
-        else resolve(buffer);
-      });
-    });
-  }
-);
-
-export async function listener(
+export function listener(
   request: IncomingMessage,
   response: ServerResponse
-): Promise<void> {
-  if (request.method !== "GET") {
+): void {
+  const urlParts = url.parse(request.url, true);
+  if (request.method === "GET") {
+    const filename = querystring.unescape(urlParts.pathname.substring(1));
+
+    const log = {
+      message: "Fetch file",
+      filename: filename,
+      remoteAddress: getRequestOrigin(request).remoteAddress,
+    };
+
+    filesCollection.findOne({ _id: filename }, (err, file) => {
+      if (err) throw err;
+
+      if (!file) {
+        response.writeHead(404);
+        response.end();
+        log.message += " not found";
+        logger.accessError(log);
+        return;
+      }
+
+      response.writeHead(200, {
+        "Content-Type": file.contentType || "application/octet-stream",
+        "Content-Length": file.length,
+      });
+
+      const downloadStream = filesBucket.openDownloadStreamByName(filename);
+      downloadStream.pipe(response);
+
+      logger.accessInfo(log);
+    });
+  } else {
     response.writeHead(405, { Allow: "GET" });
     response.end("405 Method Not Allowed");
-    return;
   }
-
-  const urlParts = url.parse(request.url, true);
-  const filename = decodeURIComponent(urlParts.pathname.substring(1));
-
-  const log = {
-    message: "Fetch file",
-    filename: filename,
-    remoteAddress: getRequestOrigin(request).remoteAddress,
-  };
-
-  const file = await filesCollection.findOne({ _id: filename });
-
-  if (!file) {
-    response.writeHead(404);
-    response.end();
-    log.message += " not found";
-    logger.accessError(log);
-    return;
-  }
-
-  const buffer = await getFile(
-    file["uploadDate"].getTime(),
-    file.length,
-    filename
-  );
-
-  response.writeHead(200, {
-    "Content-Type": "application/octet-stream",
-    "Content-Length": file.length,
-  });
-
-  response.end(buffer);
-  logger.accessInfo(log);
 }

lib/init.ts

@@ -59,7 +59,7 @@ const INFORM_SCRIPT = `
 // Device ID as user name
 const username = declare("DeviceID.ID", {value: 1}).value[0]
 
-// Password will be fixed for a given device because Math.random() is seeded with device ID by default.
+// Password will be fixed a given device because Math.random() is seeded with devcie ID by default.
 const password = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER).toString(36);
 
 const informInterval = 300;
@@ -257,7 +257,7 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
           "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
       },
       { _id: "ui.device.3.type", value: "'parameter-table'" },
-      { _id: "ui.device.3.label", value: "'LAN hosts'" },
+      { _id: "ui.device.3.label", value: "'WLAN hosts'" },
       {
         _id: "ui.device.3.parameter",
         value: "InternetGatewayDevice.LANDevice.1.Hosts.Host",

lib/instance-set.ts

@@ -22,7 +22,7 @@ interface Instance {
 }
 
 export default class InstanceSet {
-  private declare set: Set<Instance>;
+  private set: Set<Instance>;
 
   public constructor() {
     this.set = new Set();

lib/local-cache.ts

@@ -50,8 +50,8 @@ interface Snapshot {
   ui: UiConfig;
 }
 
-const REFRESH = 5000;
-const EVICT_TIMEOUT = 120000;
+const REFRESH = 3000;
+const EVICT_TIMEOUT = 60000;
 
 const snapshots = new Map<string, Snapshot>();
 let currentSnapshot: string = null;
@@ -131,7 +131,7 @@ async function fetchPresets(): Promise<Preset[]> {
 
     // If no keys are defined, consider all parameters as keys to keep the
     // same behavior from v1.0
-    if (!obj["_keys"]?.length)
+    if (!obj["_keys"] || !obj["_keys"].length)
       obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
 
     return obj;
@@ -266,7 +266,9 @@ async function fetchProvisions(): Promise<Provisions> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    provisions[r["_id"]].script = new vm.Script(
+    provisions[
+      r["_id"]
+    ].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -285,7 +287,9 @@ async function fetchVirtualParameters(): Promise<VirtualParameters> {
       .createHash("md5")
       .update(r["script"])
       .digest("hex");
-    virtualParameters[r["_id"]].script = new vm.Script(
+    virtualParameters[
+      r["_id"]
+    ].script = new vm.Script(
       `"use strict";(function(){\n${r["script"]}\n})();`,
       { filename: r["_id"], lineOffset: -1, timeout: 50 }
     );
@@ -321,6 +325,8 @@ async function fetchFiles(): Promise<Files> {
     const id = r["filename"] || r["_id"].toString();
     files[id] = {};
     files[id].length = r["length"];
+    files[id].md5 = r["md5"];
+    files[id].contentType = r["contentType"];
   }
 
   return files;
@@ -351,7 +357,6 @@ async function fetchConfig(): Promise<[Config, UiConfig]> {
     device: {},
     index: {},
     overview: {},
-    pageSize: null,
   };
   const _config = {};
 
@@ -361,13 +366,12 @@ async function fetchConfig(): Promise<[Config, UiConfig]> {
     _config[c.id] = val;
     if (c.id.startsWith("ui.")) {
       const keys = c.id.split(".");
-      if (!(keys[1] in ui)) continue;
       // remove the first key(ui)
       keys.shift();
       let ref = ui;
       while (keys.length > 1) {
         const k = keys.shift();
-        if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
+        if (typeof ref[k] !== "object") ref[k] = {};
         ref = ref[k];
       }
       ref[keys[0]] = val;
@@ -394,7 +398,7 @@ async function refresh(): Promise<void> {
     return;
   }
 
-  const lockToken = await cache.acquireLock("presets_hash_lock", 5000, 5000);
+  const lockToken = await cache.acquireLock("presets_hash_lock", 4000, 4000);
 
   const res = await Promise.all([
     fetchPresets(),
@@ -497,7 +501,7 @@ export function getConfig(
   if (!(key in snapshot.config)) {
     if (key in oldOpts) {
       let id;
-      if (context?.["id"]) {
+      if (context && context["id"]) {
         id = context["id"];
       } else if (cb) {
         id = cb(["PARAM", "DeviceID.ID"]);

lib/logger.ts

@@ -292,9 +292,11 @@ function formatSimple(
     return `${severity}${remote}${details.message}${meta}${os.EOL}`;
   }
 
-  return `${details.timestamp} [${(
-    details.severity as string
-  ).toUpperCase()}] ${remote}${details.message}${meta}${os.EOL}`;
+  return `${
+    details.timestamp
+  } [${(details.severity as string).toUpperCase()}] ${remote}${
+    details.message
+  }${meta}${os.EOL}`;
 }
 
 function log(details: Record<string, unknown>): void {

lib/mongodb-functions.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ObjectId } from "mongodb";
+import { ObjectID } from "mongodb";
 import { map, parse, stringify, parseList } from "./common/expression-parser";
 import { likePatternToRegExp } from "./common/expression";
 import { Expression, Fault, Task } from "./types";
@@ -90,7 +90,7 @@ export function processTasksFilter(filter: Expression): Expression {
     if (!isArray(exp)) return exp;
     if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
       const e = exp.slice();
-      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectId(e[2]);
+      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectID(e[2]);
       else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
         e[2] = new Date(e[2]);
       else if (e[1][0] === "PARAM" && e[1][1] === "expiry")

lib/nbi.ts

@@ -19,6 +19,7 @@
 
 import * as url from "url";
 import { Collection, GridFSBucket, ObjectId } from "mongodb";
+import * as querystring from "querystring";
 import * as vm from "vm";
 import * as config from "./config";
 import { onConnect } from "./db";
@@ -28,13 +29,10 @@ import { IncomingMessage, ServerResponse } from "http";
 import * as cache from "./cache";
 import { version as VERSION } from "../package.json";
 import { ping } from "./ping";
-import * as logger from "./logger";
-import { flattenDevice } from "./mongodb-functions";
 
 const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
 const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
-const TAGS_REGEX =
-  /^\/devices\/([a-zA-Z0-9\-_%]+)\/tags\/([a-zA-Z0-9\-_%]+)\/?$/;
+const TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tags\/([a-zA-Z0-9\-_%]+)\/?$/;
 const PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-_%]+)\/?$/;
 const OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FILES_REGEX = /^\/files\/([a-zA-Z0-9%!*'();:@&=+$,?#[\]\-_.~]+)\/?$/;
@@ -42,16 +40,15 @@ const PING_REGEX = /^\/ping\/([a-zA-Z0-9\-_.:]+)\/?$/;
 const QUERY_REGEX = /^\/([a-zA-Z0-9_]+)\/?$/;
 const DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/?$/;
 const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
-const VIRTUAL_PARAMETERS_REGEX =
-  /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
+const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
 const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
 
-const collections: Record<string, Collection> = {
+const collections = {
   tasks: null as Collection,
   devices: null as Collection,
   presets: null as Collection,
   objects: null as Collection,
-  files: null as Collection,
+  "fs.files": null as Collection,
   provisions: null as Collection,
   virtualParameters: null as Collection,
   faults: null as Collection,
@@ -59,10 +56,7 @@ const collections: Record<string, Collection> = {
 let filesBucket: GridFSBucket;
 
 onConnect(async (db) => {
-  for (const k of Object.keys(collections)) {
-    if (k === "files") collections[k] = db.collection("fs.files");
-    else collections[k] = db.collection(k);
-  }
+  for (const k of Object.keys(collections)) collections[k] = db.collection(k);
   filesBucket = new GridFSBucket(db);
 });
 
@@ -101,27 +95,12 @@ export function listener(
   request.addListener("end", () => {
     const body = getBody();
     const urlParts = url.parse(request.url, true);
-
-    logger.accessInfo(
-      Object.assign({}, urlParts.query, {
-        remoteAddress: request.socket.remoteAddress,
-        message: `${request.method} ${urlParts.pathname}`,
-      })
-    );
-
     if (PRESETS_REGEX.test(urlParts.pathname)) {
-      const presetName = decodeURIComponent(
+      const presetName = querystring.unescape(
         PRESETS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
-        let preset;
-        try {
-          preset = JSON.parse(body.toString());
-        } catch (err) {
-          response.writeHead(400);
-          response.end(`${err.name}: ${err.message}`);
-          return;
-        }
+        const preset = JSON.parse(body.toString());
         preset._id = presetName;
         collections.presets.replaceOne(
           { _id: presetName },
@@ -164,18 +143,11 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (OBJECTS_REGEX.test(urlParts.pathname)) {
-      const objectName = decodeURIComponent(
+      const objectName = querystring.unescape(
         OBJECTS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
-        let object;
-        try {
-          object = JSON.parse(body.toString());
-        } catch (err) {
-          response.writeHead(400);
-          response.end(`${err.name}: ${err.message}`);
-          return;
-        }
+        const object = JSON.parse(body.toString());
         object._id = objectName;
         collections.objects.replaceOne(
           { _id: objectName },
@@ -218,7 +190,7 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (PROVISIONS_REGEX.test(urlParts.pathname)) {
-      const provisionName = decodeURIComponent(
+      const provisionName = querystring.unescape(
         PROVISIONS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
@@ -276,7 +248,7 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (VIRTUAL_PARAMETERS_REGEX.test(urlParts.pathname)) {
-      const virtualParameterName = decodeURIComponent(
+      const virtualParameterName = querystring.unescape(
         VIRTUAL_PARAMETERS_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
@@ -338,8 +310,8 @@ export function listener(
       }
     } else if (TAGS_REGEX.test(urlParts.pathname)) {
       const r = TAGS_REGEX.exec(urlParts.pathname);
-      const deviceId = decodeURIComponent(r[1]);
-      const tag = decodeURIComponent(r[2]);
+      const deviceId = querystring.unescape(r[1]);
+      const tag = querystring.unescape(r[2]);
       if (request.method === "POST") {
         collections.devices.updateOne(
           { _id: deviceId },
@@ -367,7 +339,7 @@ export function listener(
       }
     } else if (FAULTS_REGEX.test(urlParts.pathname)) {
       if (request.method === "DELETE") {
-        const faultId = decodeURIComponent(
+        const faultId = querystring.unescape(
           FAULTS_REGEX.exec(urlParts.pathname)[1]
         );
         const deviceId = faultId.split(":", 1)[0];
@@ -412,31 +384,15 @@ export function listener(
       }
     } else if (DEVICE_TASKS_REGEX.test(urlParts.pathname)) {
       if (request.method === "POST") {
-        const deviceId = decodeURIComponent(
+        const deviceId = querystring.unescape(
           DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1]
         );
         if (body.length) {
-          let task;
-          try {
-            task = JSON.parse(body.toString());
-          } catch (err) {
-            response.writeHead(400);
-            response.end(`${err.name}: ${err.message}`);
-            return;
-          }
+          const task = JSON.parse(body.toString());
           task.device = deviceId;
-          collections.devices
-            .findOne({ _id: deviceId })
-            .then(async (dev) => {
-              if (!dev) {
-                response.writeHead(404);
-                response.end("No such device");
-                return;
-              }
-
-              const device = flattenDevice(dev);
-              await apiFunctions.insertTasks(task);
-
+          apiFunctions
+            .insertTasks(task)
+            .then(async () => {
               const lastInform = Date.now();
 
               const socketTimeout: number = request.socket["timeout"];
@@ -467,10 +423,7 @@ export function listener(
                 return;
               }
 
-              const status = await apiFunctions.connectionRequest(
-                deviceId,
-                device
-              );
+              const status = await apiFunctions.connectionRequest(deviceId);
 
               if (status) {
                 if (socketTimeout) request.socket.setTimeout(socketTimeout);
@@ -553,28 +506,15 @@ export function listener(
             });
         } else if (urlParts.query.connection_request != null) {
           // No task, send connection request only
-          collections.devices
-            .findOne({ _id: deviceId })
-            .then(async (dev) => {
-              if (!dev) {
-                response.writeHead(404);
-                response.end("No such device");
-                return;
-              }
-              return apiFunctions.connectionRequest(deviceId).then((status) => {
-                if (status) {
-                  response.writeHead(504, status);
-                  response.end(status);
-                  return;
-                }
-                response.writeHead(200);
-                response.end();
-              });
+          apiFunctions
+            .connectionRequest(deviceId)
+            .then(() => {
+              response.writeHead(200);
+              response.end();
             })
             .catch((err) => {
-              setTimeout(() => {
-                throwError(err, response);
-              });
+              response.writeHead(504);
+              response.end(`${err.name}: ${err.message}`);
             });
         } else {
           response.writeHead(400);
@@ -586,7 +526,7 @@ export function listener(
       }
     } else if (TASKS_REGEX.test(urlParts.pathname)) {
       const r = TASKS_REGEX.exec(urlParts.pathname);
-      const taskId = decodeURIComponent(r[1]);
+      const taskId = querystring.unescape(r[1]);
       const action = r[2];
       if (!action || action === "/") {
         if (request.method === "DELETE") {
@@ -672,7 +612,7 @@ export function listener(
         response.end();
       }
     } else if (FILES_REGEX.test(urlParts.pathname)) {
-      const filename = decodeURIComponent(
+      const filename = querystring.unescape(
         FILES_REGEX.exec(urlParts.pathname)[1]
       );
       if (request.method === "PUT") {
@@ -682,9 +622,9 @@ export function listener(
           productClass: request.headers.productclass,
           version: request.headers.version,
         };
-        filesBucket.delete(filename as unknown as ObjectId, () => {
+        filesBucket.delete((filename as unknown) as ObjectId, () => {
           const uploadStream = filesBucket.openUploadStreamWithId(
-            filename as unknown as ObjectId,
+            filename,
             filename,
             {
               metadata: metadata,
@@ -701,7 +641,7 @@ export function listener(
           });
         });
       } else if (request.method === "DELETE") {
-        filesBucket.delete(filename as unknown as ObjectId, (err) => {
+        filesBucket.delete((filename as unknown) as ObjectId, (err) => {
           if (err) {
             if (err.message.startsWith("FileNotFound")) {
               response.writeHead(404);
@@ -719,7 +659,7 @@ export function listener(
         response.end("405 Method Not Allowed");
       }
     } else if (PING_REGEX.test(urlParts.pathname)) {
-      const host = decodeURIComponent(PING_REGEX.exec(urlParts.pathname)[1]);
+      const host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1]);
       ping(host, (err, res, stdout) => {
         if (err) {
           if (!res) {
@@ -745,7 +685,7 @@ export function listener(
         return;
       }
 
-      const deviceId = decodeURIComponent(
+      const deviceId = querystring.unescape(
         DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1]
       );
       apiFunctions
@@ -824,14 +764,7 @@ export function listener(
       const cur = collection.find(q, { projection: projection });
 
       if (urlParts.query.sort) {
-        let s;
-        try {
-          s = JSON.parse(urlParts.query.sort as string);
-        } catch (err) {
-          response.writeHead(400);
-          response.end(`${err.name}: ${err.message}`);
-          return;
-        }
+        const s = JSON.parse(urlParts.query.sort as string);
         const sort = {};
         for (const [k, v] of Object.entries(s)) {
           if (k[k.lastIndexOf(".") + 1] !== "_" && collectionName === "devices")
@@ -842,7 +775,13 @@ export function listener(
         cur.sort(sort);
       }
 
-      cur.count((err, total) => {
+      if (urlParts.query.skip)
+        cur.skip(parseInt(urlParts.query.skip as string));
+
+      if (urlParts.query.limit)
+        cur.limit(parseInt(urlParts.query.limit as string));
+
+      cur.count(false, (err, total) => {
         if (err) return void throwError(err);
 
         response.writeHead(200, {
@@ -855,12 +794,6 @@ export function listener(
           return;
         }
 
-        if (urlParts.query.skip)
-          cur.skip(parseInt(urlParts.query.skip as string));
-
-        if (urlParts.query.limit)
-          cur.limit(parseInt(urlParts.query.limit as string));
-
         response.write("[\n");
         i = 0;
         cur.forEach(

lib/ping.ts

@@ -19,7 +19,6 @@
 
 import { platform } from "os";
 import { exec } from "child_process";
-import { domainToASCII } from "url";
 
 export interface PingResult {
   packetsTransmitted: number;
@@ -31,41 +30,24 @@ export interface PingResult {
   mdev: number;
 }
 
-function isValidHost(host: string): boolean {
-  // Valid chars in IPv4, IPv6, domain names
-  if (/^[a-zA-Z0-9\-.:[\]-]+$/.test(host)) return true;
-
-  // Check if input is an IDN convert to Punycode
-  // Can't merge with above because domainToASCII doesn't accept IP addresses
-  return /^[a-zA-Z0-9\-.:[\]-]+$/.test(domainToASCII(host));
-}
-
 export function ping(
   host: string,
   callback: (err: Error, res?: PingResult, stdout?: string) => void
 ): void {
   let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
-  // Validate input to prevent possible remote code execution
-  // Credit to Alex Hordijk for reporting this vulnerability
-  if (!isValidHost(host)) return callback(new Error("Invalid host"));
-  host = host.replace("[", "").replace("]", "");
   switch (platform()) {
     case "linux":
       cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
-      parseRegExp1 =
-        /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
-      parseRegExp2 =
-        /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
+      parseRegExp1 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+      parseRegExp2 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
       break;
 
     case "freebsd":
       // Send a single packet because on FreeBSD only superuser can send
       // packets that are only 200 ms apart.
       cmd = `ping -t 1 -c 3 ${host}`;
-      parseRegExp1 =
-        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
-      parseRegExp2 =
-        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
+      parseRegExp1 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
+      parseRegExp2 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
       break;
 
     default:

lib/scheduling.ts

@@ -18,7 +18,7 @@
  */
 
 import * as crypto from "crypto";
-import * as later from "@breejs/later";
+import * as later from "later";
 
 function md532(str): number {
   const digest = crypto.createHash("md5").update(str).digest();

lib/session.ts

@@ -59,7 +59,6 @@ import {
   GetParameterValuesResponse,
 } from "./types";
 import { getRequestOrigin } from "./forwarded";
-import * as logger from "./logger";
 
 const VALID_PARAM_TYPES = new Set([
   "xsd:int",
@@ -136,7 +135,7 @@ export function configContextCallback(
     const path = paths.get(Path.parse(name));
     if (path) {
       const attrs = deviceData.attributes.get(path, 1);
-      if (attrs?.value?.[1]) return attrs.value[1][0];
+      if (attrs && attrs.value && attrs.value[1]) return attrs.value[1][0];
     }
   } else if (exp[0] === "FUNC") {
     if (exp[1] === "REMOTE_ADDRESS")
@@ -299,7 +298,7 @@ export async function transferComplete(
   if (!sessionContext.operationsTouched) sessionContext.operationsTouched = {};
   sessionContext.operationsTouched[commandKey] = 1;
 
-  if (rpcReq.faultStruct?.faultCode !== "0") {
+  if (rpcReq.faultStruct && rpcReq.faultStruct.faultCode !== "0") {
     revertDownloadParameters(sessionContext, operation.args.instance);
 
     const fault: Fault = {
@@ -418,7 +417,13 @@ function revertDownloadParameters(
   );
 
   const toClear = device.set(sessionContext.deviceData, p, timestamp, {
-    value: [timestamp, [lastDownload?.value[1]?.[0] || 0, "xsd:dateTime"]],
+    value: [
+      timestamp,
+      [
+        lastDownload && lastDownload.value[1] ? lastDownload.value[1][0] : 0,
+        "xsd:dateTime",
+      ],
+    ],
   });
 
   if (toClear) {
@@ -452,50 +457,22 @@ export async function timeoutOperations(
         (e) => configContextCallback(sessionContext, e)
       ) * 1000;
 
-    if (sessionContext.timestamp < operation.timestamp + DOWNLOAD_TIMEOUT)
-      continue;
+    if (sessionContext.timestamp > operation.timestamp + DOWNLOAD_TIMEOUT) {
+      delete sessionContext.operations[commandKey];
+      if (!sessionContext.operationsTouched)
+        sessionContext.operationsTouched = {};
+      sessionContext.operationsTouched[commandKey] = 1;
 
-    logger.accessWarn({
-      sessionContext: sessionContext,
-      message: "Download operation timed out",
-      commandKey: commandKey,
-    });
+      faults.push({
+        code: "timeout",
+        message: "Download operation timed out",
+        timestamp: operation.timestamp,
+      });
 
-    const SUCCESS_ON_TIMEOUT = +localCache.getConfig(
-      sessionContext.cacheSnapshot,
-      "cwmp.downloadSuccessOnTimeout",
-      {},
-      sessionContext.timestamp,
-      (e) => configContextCallback(sessionContext, e)
-    );
+      operations.push(operation);
 
-    if (SUCCESS_ON_TIMEOUT) {
-      const r = {
-        name: "TransferComplete" as const,
-        commandKey: commandKey,
-        startTime: 0,
-        completeTime: 0,
-      };
-
-      // Call transferComplete code and ignore the response
-      await transferComplete(sessionContext, r);
-      continue;
+      revertDownloadParameters(sessionContext, operation.args.instance);
     }
-
-    delete sessionContext.operations[commandKey];
-    if (!sessionContext.operationsTouched)
-      sessionContext.operationsTouched = {};
-    sessionContext.operationsTouched[commandKey] = 1;
-
-    faults.push({
-      code: "timeout",
-      message: "Download operation timed out",
-      timestamp: operation.timestamp,
-    });
-
-    operations.push(operation);
-
-    revertDownloadParameters(sessionContext, operation.args.instance);
   }
 
   return { faults, operations };
@@ -933,15 +910,10 @@ function runDeclarations(
         if (!unpacked)
           unpacked = device.unpack(sessionContext.deviceData, path);
 
-        for (const u of unpacked) {
+        for (const u of unpacked)
           mergeAttributeValues(u, declaration.attrSet, declaration.defer);
-          // Ensure writable attr is available
-          mergeAttributeTimestamps(u, { writable: 1 });
-        }
       } else {
         mergeAttributeValues(path, declaration.attrSet, declaration.defer);
-        // Ensure writable attr is available
-        mergeAttributeTimestamps(path, { writable: 1 });
       }
     }
 
@@ -965,15 +937,14 @@ function runDeclarations(
         keys = {};
       }
 
-      if (!parent.wildcard && !parent.alias) {
+      if (
+        ((path.wildcard | path.alias) & ((1 << (path.length - 1)) - 1)) ===
+        0
+      ) {
         parent = sessionContext.deviceData.paths.add(parent);
         if (!unpacked)
           unpacked = device.unpack(sessionContext.deviceData, path);
 
-        // Ensure writable attr is available
-        mergeAttributeTimestamps(parent, { writable: 1 });
-        for (const u of unpacked) mergeAttributeTimestamps(u, { writable: 1 });
-
         processInstances(
           sessionContext,
           parent,
@@ -989,19 +960,13 @@ function runDeclarations(
           parent
         );
         for (const par of parentsUnpacked) {
-          const up = device.unpack(
-            sessionContext.deviceData,
-            par.concat(path.slice(-1))
-          );
-
-          // Ensure writable attr is available
-          mergeAttributeTimestamps(par, { writable: 1 });
-          for (const u of up) mergeAttributeTimestamps(u, { writable: 1 });
-
           processInstances(
             sessionContext,
             par,
-            up,
+            device.unpack(
+              sessionContext.deviceData,
+              par.concat(path.slice(-1))
+            ),
             keys,
             minInstances,
             maxInstances,
@@ -1035,7 +1000,7 @@ export async function rpcRequest(
   if (
     !sessionContext.virtualParameters.length &&
     !sessionContext.declarations.length &&
-    !_declarations?.length &&
+    !(_declarations && _declarations.length) &&
     !sessionContext.provisions.length
   )
     return { fault: null, rpcId: null, rpc: null };
@@ -1110,11 +1075,12 @@ export async function rpcRequest(
     return rpcRequest(sessionContext, _declarations);
   }
 
-  if (_declarations?.length) {
+  if (_declarations && _declarations.length) {
     delete sessionContext.syncState;
     if (!sessionContext.declarations[0]) sessionContext.declarations[0] = [];
-    sessionContext.declarations[0] =
-      sessionContext.declarations[0].concat(_declarations);
+    sessionContext.declarations[0] = sessionContext.declarations[0].concat(
+      _declarations
+    );
     return rpcRequest(sessionContext, null);
   }
 
@@ -1445,7 +1411,7 @@ export async function rpcRequest(
       for (const [p, v] of sessionContext.syncState.downloadsValues) {
         const attrs = sessionContext.deviceData.attributes.get(p);
         if (attrs) {
-          if (attrs.writable?.[1] && attrs.value) {
+          if (attrs.writable && attrs.writable[1] && attrs.value) {
             const val = device.sanitizeParameterValue([v, attrs.value[1][1]]);
             if (val[0] !== attrs.value[1][0]) {
               toClear = device.set(
@@ -1686,7 +1652,7 @@ function generateGetRpcRequest(
       syncState.refreshAttributes.value.delete(path);
       // Need to check in case param is deleted or changed to object
       const attrs = sessionContext.deviceData.attributes.get(path);
-      if (attrs?.object?.[1] === 0) {
+      if (attrs && attrs.object && attrs.object[1] === 0) {
         parameterNames.push(path.toString());
         if (parameterNames.length >= GPV_BATCH_SIZE) break;
       }
@@ -1839,7 +1805,7 @@ function generateSetRpcRequest(
   for (const [k, v] of syncState.spv) {
     syncState.spv.delete(k);
     const attrs = sessionContext.deviceData.attributes.get(k);
-    const curVal = attrs.value?.[1];
+    const curVal = attrs.value ? attrs.value[1] : null;
     if (curVal && canWrite(attrs)) {
       const val = v.slice() as [string | number | boolean, string];
       if (!val[1]) val[1] = curVal[1];
@@ -1903,9 +1869,8 @@ function generateSetRpcRequest(
 
   // Downloads
   for (const [p, t] of syncState.downloadsDownload) {
-    if (!(t > 0 && t <= sessionContext.timestamp)) continue;
     const attrs = deviceData.attributes.get(p);
-    if (!(t <= attrs?.value?.[1]?.[0])) {
+    if (!(attrs && attrs.value && t <= attrs.value[1][0])) {
       const fileTypeAttrs = deviceData.attributes.get(
         deviceData.paths.get(p.slice(0, -1).concat(Path.parse("FileType")))
       );
@@ -1922,31 +1887,42 @@ function generateSetRpcRequest(
         name: "Download",
         commandKey: generateRpcId(sessionContext),
         instance: p.segments[1] as string,
-        fileType: fileTypeAttrs?.value?.[1][0] as string,
-        fileName: fileNameAttrs?.value?.[1][0] as string,
-        targetFileName: targetFileNameAttrs?.value?.[1][0] as string,
+        fileType: fileTypeAttrs
+          ? fileTypeAttrs.value
+            ? (fileTypeAttrs.value[1][0] as string)
+            : null
+          : null,
+        fileName: fileNameAttrs
+          ? fileNameAttrs.value
+            ? (fileNameAttrs.value[1][0] as string)
+            : null
+          : null,
+        targetFileName: targetFileNameAttrs
+          ? targetFileNameAttrs.value
+            ? (targetFileNameAttrs.value[1][0] as string)
+            : null
+          : null,
       };
     }
   }
 
   // Reboot
-  if (syncState.reboot > 0 && syncState.reboot <= sessionContext.timestamp) {
+  if (syncState.reboot) {
     const p = sessionContext.deviceData.paths.get(Path.parse("Reboot"));
     const attrs = p ? sessionContext.deviceData.attributes.get(p) : null;
-    if (!(attrs?.value?.[1][0] >= syncState.reboot)) {
+    if (!(attrs && attrs.value && attrs.value[1][0] >= syncState.reboot)) {
       delete syncState.reboot;
       return { name: "Reboot" };
     }
   }
 
   // Factory reset
-  if (
-    syncState.factoryReset > 0 &&
-    syncState.factoryReset <= sessionContext.timestamp
-  ) {
+  if (syncState.factoryReset) {
     const p = sessionContext.deviceData.paths.get(Path.parse("FactoryReset"));
     const attrs = p ? sessionContext.deviceData.attributes.get(p) : null;
-    if (!(attrs?.value?.[1][0] >= syncState.factoryReset)) {
+    if (
+      !(attrs && attrs.value && attrs.value[1][0] >= syncState.factoryReset)
+    ) {
       delete syncState.factoryReset;
       return { name: "FactoryReset" };
     }
@@ -2014,7 +1990,7 @@ function generateSetVirtualParameterProvisions(
   let provisions;
   if (virtualParameterDeclarations) {
     for (const declaration of virtualParameterDeclarations) {
-      if (declaration[2]?.value != null) {
+      if (declaration[2] && declaration[2].value != null) {
         const attrs = sessionContext.deviceData.attributes.get(declaration[0]);
         if (
           attrs &&
@@ -2143,7 +2119,9 @@ function processDeclarations(
 
     if (currentAttributes) {
       leafParam = currentPath;
-      leafIsObject = currentAttributes.object?.[1];
+      leafIsObject = currentAttributes.object
+        ? currentAttributes.object[1]
+        : null;
       // Possible V8 bug causes null === 0
       if (leafIsObject != null && leafIsObject === 0)
         leafTimestamp = Math.max(leafTimestamp, currentAttributes.object[0]);
@@ -2158,13 +2136,13 @@ function processDeclarations(
     ) {
       case "Reboot":
         if (currentPath.length === 1) {
-          if (declareAttributeValues?.value)
+          if (declareAttributeValues && declareAttributeValues.value)
             syncState.reboot = +new Date(declareAttributeValues.value[0]);
         }
         break;
       case "FactoryReset":
         if (currentPath.length === 1) {
-          if (declareAttributeValues?.value)
+          if (declareAttributeValues && declareAttributeValues.value)
             syncState.factoryReset = +new Date(declareAttributeValues.value[0]);
         }
         break;
@@ -2300,7 +2278,7 @@ function processDeclarations(
                     syncState.refreshAttributes.object.add(currentPath);
                   else if (currentAttributes.object[1] === 0)
                     syncState.refreshAttributes.value.add(currentPath);
-                } else if (attrName in syncState.refreshAttributes) {
+                } else {
                   syncState.refreshAttributes[attrName].add(currentPath);
                 }
               }
@@ -2429,16 +2407,9 @@ export async function rpcResponse(
   sessionContext: SessionContext,
   id: string,
   _rpcRes: CpeResponse
-): Promise<Fault> {
-  function invalidResponse(message: string): Fault {
-    return {
-      code: "invalid_response",
-      message: message,
-    };
-  }
-
+): Promise<void> {
   if (id !== generateRpcId(sessionContext))
-    return invalidResponse("Request ID not recognized");
+    throw new Error("Request ID not recognized");
 
   ++sessionContext.rpcCount;
 
@@ -2529,99 +2500,39 @@ export async function rpcResponse(
 
   if (rpcRes.name === "GetParameterValuesResponse") {
     if (rpcReq.name !== "GetParameterValues")
-      return invalidResponse("Response name does not match request name");
+      throw new Error("Response name does not match request name");
 
-    const requested = new Set(rpcReq.parameterNames);
-
-    for (const [path, value, type] of rpcRes.parameterList) {
-      if (!requested.delete(path.toString())) {
-        logger.accessWarn({
-          sessionContext: sessionContext,
-          message: "Unexpected parameter in response",
-          parameter: path.toString(),
-        });
-        continue;
-      }
+    for (const p of rpcRes.parameterList) {
       toClear = device.set(
         sessionContext.deviceData,
-        path,
+        p[0],
         timestamp,
         {
           object: [timestamp, 0],
-          value: [timestamp, [value, type]],
+          value: [timestamp, p.slice(1) as [string | number | boolean, string]],
         },
         toClear
       );
     }
-
-    if (requested.size) {
-      for (const p of requested) {
-        logger.accessWarn({
-          sessionContext: sessionContext,
-          message: "Missing parameter in response",
-          parameter: p,
-        });
-        toClear = device.set(
-          sessionContext.deviceData,
-          Path.parse(p),
-          timestamp,
-          {
-            object: [timestamp, 0],
-            value: [timestamp, ["", "xsd:string"]],
-          },
-          toClear
-        );
-      }
-    }
   } else if (rpcRes.name === "GetParameterAttributesResponse") {
     if (rpcReq.name !== "GetParameterAttributes")
       throw new Error("Response name does not match request name");
 
-    const requested = new Set(rpcReq.parameterNames);
-
-    for (const [path, notification, accessList] of rpcRes.parameterList) {
-      if (!requested.delete(path.toString())) {
-        logger.accessWarn({
-          sessionContext: sessionContext,
-          message: "Unexpected parameter in response",
-          parameter: path.toString(),
-        });
-        continue;
-      }
+    for (const p of rpcRes.parameterList) {
       toClear = device.set(
         sessionContext.deviceData,
-        path,
+        p[0],
         timestamp,
         {
-          notification: [timestamp, notification],
-          accessList: [timestamp, accessList],
+          notification: [timestamp, p[1] as number],
+          accessList: [timestamp, p[2] as string[]],
         },
         toClear
       );
     }
-
-    if (requested.size) {
-      for (const p of requested) {
-        logger.accessWarn({
-          sessionContext: sessionContext,
-          message: "Missing parameter in response",
-          parameter: p,
-        });
-        toClear = device.set(
-          sessionContext.deviceData,
-          Path.parse(p),
-          timestamp,
-          {
-            notification: [timestamp, 0],
-            accessList: [timestamp, []],
-          },
-          toClear
-        );
-      }
-    }
   } else if (rpcRes.name === "GetParameterNamesResponse") {
     if (rpcReq.name !== "GetParameterNames")
-      return invalidResponse("Response name does not match request name");
+      throw new Error("Response name does not match request name");
 
     let root: Path;
     if (rpcReq.parameterPath.endsWith("."))
@@ -2678,14 +2589,6 @@ export async function rpcResponse(
     const wildcardParams: Path[] = [root.concat(wildcardPath)];
 
     for (const [path, object, writable] of rpcRes.parameterList) {
-      if (!path.toString().startsWith(rpcReq.parameterPath)) {
-        logger.accessWarn({
-          sessionContext: sessionContext,
-          message: "Unexpected parameter in response",
-          parameter: path.toString(),
-        });
-        continue;
-      }
       if (object && !rpcReq.nextLevel)
         wildcardParams.push(path.concat(wildcardPath));
 
@@ -2712,7 +2615,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "SetParameterValuesResponse") {
     if (rpcReq.name !== "SetParameterValues")
-      return invalidResponse("Response name does not match request name");
+      throw new Error("Response name does not match request name");
 
     for (const p of rpcReq.parameterList) {
       toClear = device.set(
@@ -2731,7 +2634,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "SetParameterAttributesResponse") {
     if (rpcReq.name !== "SetParameterAttributes")
-      return invalidResponse("Response name does not match request name");
+      throw new Error("Response name does not match request name");
 
     for (const p of rpcReq.parameterList) {
       let attrs;
@@ -2761,7 +2664,7 @@ export async function rpcResponse(
     }
   } else if (rpcRes.name === "AddObjectResponse") {
     if (rpcReq.name !== "AddObject")
-      return invalidResponse("Response name does not match request name");
+      throw new Error("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2772,7 +2675,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "DeleteObjectResponse") {
     if (rpcReq.name !== "DeleteObject")
-      return invalidResponse("Response name does not match request name");
+      throw new Error("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2783,7 +2686,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "RebootResponse") {
     if (rpcReq.name !== "Reboot")
-      return invalidResponse("Response name does not match request name");
+      throw new Error("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2794,7 +2697,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "FactoryResetResponse") {
     if (rpcReq.name !== "FactoryReset")
-      return invalidResponse("Response name does not match request name");
+      throw new Error("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2805,7 +2708,7 @@ export async function rpcResponse(
     );
   } else if (rpcRes.name === "DownloadResponse") {
     if (rpcReq.name !== "Download")
-      return invalidResponse("Response name does not match request name");
+      throw new Error("Response name does not match request name");
 
     toClear = device.set(
       sessionContext.deviceData,
@@ -2891,15 +2794,13 @@ export async function rpcResponse(
       sessionContext.operationsTouched[rpcReq.commandKey] = 1;
     }
   } else {
-    return invalidResponse("Response name not recognized");
+    throw new Error("Response name not recognized");
   }
 
   if (toClear) {
     for (const c of toClear)
       device.clear(sessionContext.deviceData, c[0], c[1], c[2], c[3]);
   }
-
-  return null;
 }
 
 export async function rpcFault(
@@ -2932,9 +2833,13 @@ export async function rpcFault(
         (p) => [Path.parse(p.replace(/\.$/, "")), timestamp] as Clear
       );
     } else if (rpcReq.name === "SetParameterValues") {
-      toClear = (
-        rpcReq.parameterList as [string, string | number | boolean, string][]
-      ).map((p) => [Path.parse(p[0].replace(/\.$/, "")), timestamp] as Clear);
+      toClear = (rpcReq.parameterList as [
+        string,
+        string | number | boolean,
+        string
+      ][]).map(
+        (p) => [Path.parse(p[0].replace(/\.$/, "")), timestamp] as Clear
+      );
     } else if (rpcReq.name === "AddObject") {
       toClear = [[Path.parse(rpcReq.objectName.replace(/\.$/, "")), timestamp]];
     } else if (rpcReq.name === "DeleteObject") {
@@ -2971,10 +2876,10 @@ export async function deserialize(
   const sessionContext = JSON.parse(sessionContextString) as SessionContext;
 
   for (const decs of sessionContext.declarations)
-    for (const d of decs) d.path = Path.parse(d.path as unknown as string);
+    for (const d of decs) d.path = Path.parse((d.path as unknown) as string);
 
   const deviceData = initDeviceData();
-  for (const r of sessionContext.deviceData as unknown as any[]) {
+  for (const r of (sessionContext.deviceData as unknown) as any[]) {
     const path = deviceData.paths.add(Path.parse(r[0]));
 
     if (r[1]) deviceData.trackers.set(path, r[1]);

lib/soap.ts

@@ -109,8 +109,8 @@ function event(xml: Element): string[] {
 
 function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
   return xml.children
+    .filter((e) => e.localName === "ParameterInfoStruct")
     .map<[Path, boolean, boolean]>((e) => {
-      if (e.localName !== "ParameterInfoStruct") return null;
       let param: string, value: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -134,20 +134,10 @@ function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
         parsed = false;
       }
 
-      try {
-        if (param && !param.endsWith("."))
-          return [Path.parse(param), false, parsed];
-        else return [Path.parse(param.slice(0, -1)), true, parsed];
-      } catch (err) {
-        warnings.push({
-          message: "Missing or invalid XML node",
-          element: "Name",
-          parameter: param,
-        });
-        return null;
-      }
-    })
-    .filter((e) => e != null);
+      if (param && !param.endsWith("."))
+        return [Path.parse(param), false, parsed];
+      else return [Path.parse(param.slice(0, -1)), true, parsed];
+    });
 }
 
 const getValueType = memoize((str: string) => {
@@ -161,8 +151,8 @@ function parameterValueList(
   xml: Element
 ): [Path, string | number | boolean, string][] {
   return xml.children
+    .filter((e) => e.localName === "ParameterValueStruct")
     .map<[Path, string | number | boolean, string]>((e) => {
-      if (e.localName !== "ParameterValueStruct") return null;
       let valueElement: Element, param: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -218,24 +208,15 @@ function parameterValueList(
           parsed = value;
         }
       }
-      try {
-        return [Path.parse(param), parsed, valueType];
-      } catch (err) {
-        warnings.push({
-          message: "Missing or invalid XML node",
-          element: "Name",
-          parameter: param,
-        });
-        return null;
-      }
-    })
-    .filter((e) => e != null);
+
+      return [Path.parse(param), parsed, valueType];
+    });
 }
 
 function parameterAttributeList(xml: Element): [Path, number, string[]][] {
   return xml.children
+    .filter((e) => e.localName === "ParameterAttributeStruct")
     .map<[Path, number, string[]]>((e) => {
-      if (e.localName !== "ParameterAttributeStruct") return null;
       let notificationElement: Element,
         accessListElement: Element,
         param: string;
@@ -267,18 +248,8 @@ function parameterAttributeList(xml: Element): [Path, number, string[]][] {
         .filter((c) => c.localName === "string")
         .map((c) => decodeEntities(c.text));
 
-      try {
-        return [Path.parse(param), notification, accessList];
-      } catch (err) {
-        warnings.push({
-          message: "Missing or invalid XML node",
-          element: "Name",
-          parameter: param,
-        });
-        return null;
-      }
-    })
-    .filter((e) => e != null);
+      return [Path.parse(param), notification, accessList];
+    });
 }
 
 function GetParameterNames(methodRequest): string {
@@ -730,18 +701,6 @@ function RequestDownloadResponse(): string {
   return "<cwmp:RequestDownloadResponse></cwmp:RequestDownloadResponse>";
 }
 
-function AcsFault(f: CpeFault): string {
-  return `<soap-env:Body:Fault><faultcode>${encodeEntities(
-    f.faultCode
-  )}</faultcode><faultstring>${encodeEntities(
-    f.faultString
-  )}</faultstring><detail><cwmp:Fault><FaultCode>${encodeEntities(
-    f.detail.faultCode
-  )}</FaultCode><FaultString>${encodeEntities(
-    f.detail.faultString
-  )}</FaultString></cwmp:Fault></detail></soap-env:Body:Fault>`;
-}
-
 function faultStruct(xml: Element): FaultStruct {
   let faultCode: string,
     faultString: string,
@@ -839,18 +798,18 @@ function fault(xml: Element): CpeFault {
 
 export function request(
   body: string,
+  cwmpVersion: string,
   warn: Record<string, unknown>[]
 ): SoapMessage {
   warnings = warn;
 
   const rpc = {
     id: null,
-    cwmpVersion: null,
+    cwmpVersion: cwmpVersion,
     sessionTimeout: null,
     cpeRequest: null,
     cpeFault: null,
     cpeResponse: null,
-    unknownMethod: null,
   };
 
   if (!body.length) return rpc;
@@ -889,7 +848,7 @@ export function request(
 
   const methodElement = bodyElement.children[0];
 
-  if (methodElement.localName === "Inform") {
+  if (!rpc.cwmpVersion && methodElement.localName !== "Fault") {
     let namespace, namespaceHref;
     for (const e of [methodElement, bodyElement, envelope]) {
       namespace = namespace || e.namespace;
@@ -972,8 +931,7 @@ export function request(
       rpc.cpeFault = fault(methodElement);
       break;
     default:
-      rpc.unknownMethod = methodElement.localName;
-      break;
+      throw new Error(`8000 Method not supported ${methodElement.localName}`);
   }
 
   return rpc;
@@ -1001,7 +959,6 @@ export function response(rpc: {
   id: string;
   acsRequest?: AcsRequest;
   acsResponse?: AcsResponse;
-  acsFault?: CpeFault;
   cwmpVersion?: string;
 }): { code: number; headers: Record<string, string>; data: string } {
   const headers = {
@@ -1070,8 +1027,6 @@ export function response(rpc: {
           `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`
         );
     }
-  } else if (rpc.acsFault) {
-    body = AcsFault(rpc.acsFault);
   }
 
   headers["Content-Type"] = 'text/xml; charset="utf-8"';

lib/types.ts

@@ -477,7 +477,7 @@ export interface VirtualParameters {
 }
 
 export interface Files {
-  [name: string]: { length: number };
+  [name: string]: { length: number; md5: string; contentType: string };
 }
 
 export interface Users {
@@ -526,7 +526,6 @@ export interface SoapMessage {
   cpeRequest?: CpeRequest;
   cpeFault?: CpeFault;
   cpeResponse?: CpeResponse;
-  unknownMethod?: string;
 }
 
 export interface ScriptResult {

lib/ui.ts

@@ -59,10 +59,8 @@ const getAuthorizer = memoize(
   }
 );
 
-koa.on("error", async (err, ctx) => {
+koa.on("error", async (err) => {
   setTimeout(() => {
-    // Ignored errors resulting from aborted requests
-    if (ctx?.req.aborted) return;
     throw err;
   });
 });
@@ -95,7 +93,7 @@ koa.use(
 koa.use(async (ctx, next) => {
   let roles: string[] = [];
 
-  if (ctx.state.user?.username) {
+  if (ctx.state.user && ctx.state.user.username) {
     let user;
     if (ctx.state.user.authMethod === "local") {
       user = localCache.getUsers(ctx.state.configSnapshot)[
@@ -221,8 +219,7 @@ router.post("/init", async (ctx) => {
 });
 
 router.get("/", async (ctx) => {
-  const permissionSets: PermissionSet[] =
-    ctx.state.authorizer.getPermissionSets();
+  const permissionSets: PermissionSet[] = ctx.state.authorizer.getPermissionSets();
 
   let wizard = "";
   if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
@@ -248,7 +245,7 @@ router.get("/", async (ctx) => {
         )};
         window.permissionSets = ${JSON.stringify(permissionSets)};
       </script>
-      <script type="module" src="app.js"></script>${wizard} 
+      <script src="app.js"></script>${wizard} 
     </body>
   </html>
   `;
@@ -262,11 +259,6 @@ koa.use(
     deflate: {
       flush: constants.Z_SYNC_FLUSH,
     },
-    br: {
-      params: {
-        [constants.BROTLI_PARAM_QUALITY]: 5,
-      },
-    },
   })
 );
 

lib/ui/api-functions.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { ObjectId } from "mongodb";
+import { ObjectID } from "mongodb";
 import * as db from "./db";
 import { del } from "../cache";
 import { getUsers } from "../local-cache";
@@ -38,7 +38,7 @@ async function deleteFault(id): Promise<void> {
   await Promise.all([
     db.deleteFault(id),
     channel.startsWith("task_")
-      ? db.deleteTask(new ObjectId(channel.slice(5)))
+      ? db.deleteTask(new ObjectID(channel.slice(5)))
       : null,
   ]);
 
@@ -108,7 +108,7 @@ export async function postTasks(
   const lastInform = Date.now();
   const notInSession = await awaitSessionEnd(deviceId, 30000);
   if (!notInSession) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
     return {
       connectionRequest: "Session took too long to complete",
       tasks: statuses,
@@ -120,7 +120,7 @@ export async function postTasks(
   const status = await connectionRequest(deviceId, device);
 
   if (status) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
     return {
       connectionRequest: status,
       tasks: statuses,
@@ -129,7 +129,7 @@ export async function postTasks(
 
   const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
   if (!sessionStarted) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
     return {
       connectionRequest: "No contact from CPE",
       tasks: statuses,
@@ -138,7 +138,7 @@ export async function postTasks(
 
   const sessionEnded = await awaitSessionEnd(deviceId, 120000);
   if (!sessionEnded) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
     return {
       connectionRequest: "Session took too long to complete",
       tasks: statuses,
@@ -163,7 +163,7 @@ export async function postTasks(
     }
   }
 
-  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
 
   return { connectionRequest: "OK", tasks: statuses };
 }
@@ -200,7 +200,7 @@ export function authLocal(
   return new Promise((resolve, reject) => {
     const users = getUsers(snapshot);
     const user = users[username];
-    if (!user?.password) return void resolve(null);
+    if (!user || !user.password) return void resolve(null);
     hashPassword(password, user.salt)
       .then((hash) => {
         if (hash === user.password) resolve(true);

lib/ui/api.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { PassThrough } from "stream";
+import * as stream from "stream";
 import Router from "koa-router";
 import * as db from "./db";
 import * as apiFunctions from "./api-functions";
@@ -30,8 +30,8 @@ import { generateSalt, hashPassword } from "../auth";
 import { del } from "../cache";
 import Authorizer from "../common/authorizer";
 import { ping } from "../ping";
+import * as url from "url";
 import { decodeTag } from "../common";
-import { stringify as yamlStringify } from "../common/yaml";
 
 const router = new Router();
 export default router;
@@ -70,10 +70,6 @@ const resources = {
   tasks: 0,
 };
 
-function singleParam(p: string | string[]): string {
-  return Array.isArray(p) ? p[p.length - 1] : p;
-}
-
 router.get(`/devices/:id.csv`, async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
@@ -103,35 +99,26 @@ router.get(`/devices/:id.csv`, async (ctx) => {
       .replace(/[:.]/g, "")}.csv`
   );
 
-  ctx.body = new PassThrough();
+  ctx.body = new stream.PassThrough();
   ctx.body.write(
     "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp,Notification,Notification timestamp,Access list,Access list timestamp\n"
   );
 
   for (const k of Object.keys(res[0]).sort()) {
     const p = res[0][k];
-    let value = "";
-    let type = "";
-    if (p.value) {
-      value = p.value[0];
-      type = p.value[1];
-      if (type === "xsd:dateTime" && typeof value === "number")
-        value = new Date(value).toJSON();
-    }
-
     const row = [
       k,
       p.object,
-      new Date(p.objectTimestamp).toJSON(),
+      p.objectTimestamp,
       p.writable,
-      new Date(p.writableTimestamp).toJSON(),
-      `"${value.toString().replace(/"/g, '""')}"`,
-      type,
-      new Date(p.valueTimestamp).toJSON(),
+      p.writableTimestamp,
+      p.value != null ? `"${p.value[0].toString().replace(/"/g, '""')}"` : "",
+      p.value != null ? p.value[1] : "",
+      p.valueTimestamp,
       p.notification,
-      new Date(p.notificationTimestamp).toJSON(),
+      p.notificationTimestamp,
       p.accessList ? p.accessList.join(", ") : "",
-      new Date(p.accessListTimestamp).toJSON(),
+      p.accessListTimestamp,
     ];
     ctx.body.write(row.map((r) => (r != null ? r : "")).join(",") + "\n");
   }
@@ -144,7 +131,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     const authorizer: Authorizer = ctx.state.authorizer;
     let filter: Expression = authorizer.getFilter(resource, 1);
     if (ctx.request.query.filter)
-      filter = and(filter, parse(singleParam(ctx.request.query.filter)));
+      filter = and(filter, parse(ctx.request.query.filter));
 
     const log = {
       message: `Count ${resource}`,
@@ -181,13 +168,13 @@ for (const [resource, flags] of Object.entries(resources)) {
     const options: QueryOptions = {};
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
-      filter = and(filter, parse(singleParam(ctx.request.query.filter)));
+      filter = and(filter, parse(ctx.request.query.filter));
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
-      options.sort = JSON.parse(singleParam(ctx.request.query.sort));
+      options.sort = JSON.parse(ctx.request.query.sort);
     if (ctx.request.query.projection) {
-      options.projection = singleParam(ctx.request.query.projection)
+      options.projection = ctx.request.query.projection
         .split(",")
         .reduce((obj, k) => Object.assign(obj, { [k]: 1 }), {});
     }
@@ -216,7 +203,7 @@ for (const [resource, flags] of Object.entries(resources)) {
       ]);
     }
 
-    ctx.body = new PassThrough();
+    ctx.body = new stream.PassThrough();
     ctx.type = "application/json";
 
     let c = 0;
@@ -235,11 +222,11 @@ for (const [resource, flags] of Object.entries(resources)) {
     const options: QueryOptions = { projection: {} };
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
-      filter = and(filter, parse(singleParam(ctx.request.query.filter)));
+      filter = and(filter, parse(ctx.request.query.filter));
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
-      options.sort = JSON.parse(singleParam(ctx.request.query.sort));
+      options.sort = JSON.parse(ctx.request.query.sort);
 
     const log = {
       message: `Query ${resource} (CSV)`,
@@ -256,7 +243,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     }
 
     const columns: Record<string, Expression> = JSON.parse(
-      singleParam(ctx.request.query.columns)
+      ctx.request.query.columns
     );
     const now = Date.now();
 
@@ -276,7 +263,7 @@ for (const [resource, flags] of Object.entries(resources)) {
       ]);
     }
 
-    ctx.body = new PassThrough();
+    ctx.body = new stream.PassThrough();
     ctx.type = "text/csv";
     ctx.attachment(
       `${resource}-${new Date(now).toISOString().replace(/[:.]/g, "")}.csv`
@@ -298,23 +285,11 @@ for (const [resource, flags] of Object.entries(resources)) {
 
                   return tags.join(", ");
                 }
-                if (e === exp) {
-                  const p = obj[e[1]];
-                  if (
-                    p &&
-                    p.value &&
-                    p.value[1] === "xsd:dateTime" &&
-                    typeof p.value[0] === "number"
-                  )
-                    return new Date(p.value[0]).toJSON();
-                }
-              } else if (resource === "faults") {
-                if (e[1] === "detail") return yamlStringify(obj["detail"]);
               }
             } else if (e[0] === "FUNC") {
               if (e[1] === "DATE_STRING") {
                 if (e[2] && !Array.isArray(e[2]))
-                  return new Date(e[2]).toJSON();
+                  return new Date(e[2]).toISOString();
               }
             }
           }
@@ -460,36 +435,6 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 }
 
-router.get("/blob/files/:id", async (ctx) => {
-  const authorizer: Authorizer = ctx.state.authorizer;
-  const resource = "files";
-  const id = ctx.params.id;
-
-  const log = {
-    message: `Download ${resource}`,
-    context: ctx,
-    id: id,
-  };
-
-  const filter = and(authorizer.getFilter(resource, 2), [
-    "=",
-    ["PARAM", RESOURCE_IDS[resource]],
-    ctx.params.id,
-  ]);
-
-  if (!authorizer.hasAccess(resource, 2)) {
-    logUnauthorizedWarning(log);
-    return void (ctx.status = 403);
-  }
-
-  const count = await db.count(resource, filter);
-  if (!count) return void (ctx.status = 404);
-
-  logger.accessInfo(log);
-  ctx.body = db.downloadFile(id);
-  ctx.attachment(id);
-});
-
 router.put("/files/:id", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const resource = "files";
@@ -508,11 +453,10 @@ router.put("/files/:id", async (ctx) => {
   }
 
   const metadata = {
-    fileType: singleParam(ctx.request.headers["metadata-filetype"]) || "",
-    oui: singleParam(ctx.request.headers["metadata-oui"]) || "",
-    productClass:
-      singleParam(ctx.request.headers["metadata-productclass"]) || "",
-    version: singleParam(ctx.request.headers["metadata-version"]) || "",
+    fileType: ctx.request.headers["metadata.filetype"] || "",
+    oui: ctx.request.headers["metadata.oui"] || "",
+    productClass: ctx.request.headers["metadata.productclass"] || "",
+    version: ctx.request.headers["metadata.version"] || "",
   };
 
   const validate = authorizer.getValidator(resource, metadata);
@@ -573,12 +517,12 @@ router.post("/devices/:id/tasks", async (ctx) => {
       if (!Array.isArray(exp)) return exp;
       if (exp[0] === "PARAM") {
         const p = device[exp[1]];
-        if (p?.value) return p.value[0];
+        if (p && p.value) return p.value[0];
       } else if (exp[0] === "FUNC") {
         if (exp[1] === "REMOTE_ADDRESS") {
           for (const root of ["InternetGatewayDevice", "Device"]) {
             const p = device[`${root}.ManagementServer.ConnectionRequestURL`];
-            if (p?.value) return new URL(p.value[0]).hostname;
+            if (p && p.value) return url.parse(p.value[0]).host;
           }
           return null;
         }
@@ -652,13 +596,13 @@ router.post("/devices/:id/tags", async (ctx) => {
 });
 
 router.get("/ping/:host", async (ctx) => {
-  return new Promise<void>((resolve) => {
+  return new Promise((resolve) => {
     ping(ctx.params.host, (err, parsed) => {
       if (parsed) {
         ctx.body = parsed;
       } else {
         ctx.status = 500;
-        ctx.body = err ? `${err.name}: ${err.message}` : "Unknown error";
+        ctx.body = `${err.name}: ${err.message}`;
       }
       resolve();
     });

lib/ui/db.ts

@@ -17,13 +17,13 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Db, GridFSBucket, ObjectId } from "mongodb";
+import { Db, GridFSBucket, ObjectID } from "mongodb";
 import { Script } from "vm";
 import { onConnect } from "../db";
 import * as mongodbFunctions from "../mongodb-functions";
 import * as expression from "../common/expression";
 import { QueryOptions, Expression } from "../types";
-import { Readable, Writable } from "stream";
+import { Readable } from "stream";
 import { minimize } from "../common/boolean-expression";
 
 const RESOURCE_COLLECTION = {
@@ -36,23 +36,12 @@ onConnect(async (_db) => {
   db = _db;
 });
 
-export function query(
-  resource: string,
-  filter: Expression,
-  options?: QueryOptions
-): Promise<any[]>;
-export function query(
-  resource: string,
-  filter: Expression,
-  options: QueryOptions,
-  callback: (doc: any) => void
-): Promise<void>;
 export function query(
   resource: string,
   filter: Expression,
   options?: QueryOptions,
   callback?: (doc: any) => void
-): Promise<void | any[]> {
+): Promise<any[]> {
   options = options || {};
   let q;
   filter = expression.evaluate(filter, null, Date.now());
@@ -190,8 +179,8 @@ export async function updateDeviceTags(
   const collection = db.collection("devices");
   const object = {};
 
-  if (add?.length) object["$addToSet"] = { _tags: { $each: add } };
-  if (pull?.length) object["$pullAll"] = { _tags: pull };
+  if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
+  if (pull && pull.length) object["$pullAll"] = { _tags: pull };
 
   await collection.updateOne({ _id: deviceId }, object);
 }
@@ -208,7 +197,7 @@ function putResource(resource, id, object): Promise<void> {
 
 function deleteResource(
   resource: string,
-  id: string | ObjectId
+  id: string | ObjectID
 ): Promise<void> {
   return new Promise((resolve, reject) => {
     const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
@@ -237,17 +226,9 @@ export function putProvision(
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
-    new Script(`"use strict";(function(){\n${object.script}\n})();`, {
-      filename: id,
-      lineOffset: -1,
-    });
-  } catch (err) {
-    if (err.stack?.startsWith(`${id}:`)) {
-      return Promise.reject(
-        new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`)
-      );
-    }
-    return Promise.reject(err);
+    new Script(`"use strict";(function(){\n${object.script}\n})();`);
+  } catch (error) {
+    return Promise.reject(error);
   }
   return putResource("provisions", id, object);
 }
@@ -262,17 +243,9 @@ export function putVirtualParameter(
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
-    new Script(`"use strict";(function(){\n${object.script}\n})();`, {
-      filename: id,
-      lineOffset: -1,
-    });
-  } catch (err) {
-    if (err.stack?.startsWith(`${id}:`)) {
-      return Promise.reject(
-        new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`)
-      );
-    }
-    return Promise.reject(err);
+    new Script(`"use strict";(function(){\n${object.script}\n})();`);
+  } catch (error) {
+    return Promise.reject(error);
   }
   return putResource("virtualParameters", id, object);
 }
@@ -326,11 +299,6 @@ export function deleteUser(id: string): Promise<void> {
   return deleteResource("users", id);
 }
 
-export function downloadFile(filename: string): Readable {
-  const bucket = new GridFSBucket(db);
-  return bucket.openDownloadStreamByName(filename);
-}
-
 export function putFile(
   filename: string,
   metadata: Record<string, string>,
@@ -338,17 +306,13 @@ export function putFile(
 ): Promise<void> {
   return new Promise((resolve, reject) => {
     const bucket = new GridFSBucket(db);
-    const uploadStream = bucket.openUploadStreamWithId(
-      filename as unknown as ObjectId,
-      filename,
-      {
-        metadata: metadata,
-      }
-    );
+    const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
+      metadata: metadata,
+    });
     uploadStream.on("error", reject);
     contentStream.on("error", reject);
     uploadStream.on("finish", resolve);
-    contentStream.pipe(uploadStream as Writable);
+    contentStream.pipe(uploadStream);
   });
 }
 
@@ -366,6 +330,6 @@ export function deleteFault(id: string): Promise<void> {
   return deleteResource("faults", id);
 }
 
-export function deleteTask(id: ObjectId): Promise<void> {
+export function deleteTask(id: ObjectID): Promise<void> {
   return deleteResource("tasks", id);
 }

lib/versioned-map.ts

@@ -26,10 +26,10 @@ interface Revisions<V> {
 }
 
 export default class VersionedMap<K, V> {
-  private declare _sizeDiff: number[];
-  private declare _revision: number;
-  private declare map: Map<K, (V | symbol)[]>;
-  public declare dirty: number;
+  private _sizeDiff: number[];
+  private _revision: number;
+  private map: Map<K, (V | symbol)[]>;
+  public dirty: number;
 
   public constructor() {
     this._sizeDiff = [0];

lib/xmpp-client.ts

@@ -0,0 +1,481 @@
+/**
+ * Copyright 2013-2020  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as net from "net";
+import * as tls from "tls";
+import { EventEmitter } from "events";
+import { createHash, createHmac, randomBytes } from "crypto";
+import { parseXml, Element, parseAttrs } from "./xml-parser";
+
+function encodeBase64(str: string): string {
+  return Buffer.from(str).toString("base64");
+}
+
+function decodeBase64(str: string): string {
+  return Buffer.from(str, "base64").toString();
+}
+
+function detectStreamTag(data: string): number {
+  const i1 = data.indexOf("<stream:stream");
+  if (i1 < 0) throw new Error("Cannot detect opening stream tag");
+  const i2 = data.indexOf(">", i1);
+  if (i2 < 0) throw new Error("Cannot detect opening stream tag");
+  return i2 + 1;
+}
+
+function xmppStream<T>(
+  socket: net.Socket,
+  callback: Generator<void, T, Element>
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const onError = (err: Error): void => {
+      socket.removeListener("error", onError);
+      reject(err);
+    };
+    socket.on("error", onError);
+
+    const onData = (chunk: Buffer): void => {
+      try {
+        const str = chunk.toString("utf8");
+        const xml = parseXml(str);
+        const { value, done } = callback.next(xml.children[0]);
+        if (done) {
+          socket.removeListener("error", onError);
+          socket.removeListener("data", onData);
+          resolve(value as T);
+        }
+      } catch (err) {
+        socket.removeListener("error", onError);
+        socket.removeListener("data", onData);
+        reject(err);
+      }
+    };
+
+    socket.once("data", (chunk: Buffer) => {
+      try {
+        const str = chunk.toString("utf8");
+        const i = detectStreamTag(str);
+        const streamTagStr = str.slice(0, i);
+        const xml = parseXml(streamTagStr + "</stream:stream>");
+        callback.next(xml.children[0]);
+        chunk = chunk.slice(Buffer.byteLength(streamTagStr));
+        if (chunk.length) onData(chunk);
+        socket.on("data", onData);
+      } catch (err) {
+        socket.removeListener("error", onError);
+        socket.removeListener("data", onData);
+        reject(err);
+      }
+    });
+
+    try {
+      callback.next();
+    } catch (err) {
+      socket.removeListener("error", onError);
+      socket.removeListener("data", onData);
+      reject(err);
+    }
+  });
+}
+
+const INT_1 = Buffer.from([0, 0, 0, 1]);
+const saltedPasswordCache = {
+  password: "",
+  iterationCount: 0,
+  saltBase64: "",
+  salted: Buffer.allocUnsafe(0),
+};
+
+function saltPassword(
+  password: string,
+  saltBase64: string,
+  iteractionCount: number
+): Buffer {
+  if (
+    password === saltedPasswordCache.password &&
+    saltBase64 === saltedPasswordCache.saltBase64 &&
+    iteractionCount === saltedPasswordCache.iterationCount
+  )
+    return saltedPasswordCache.salted;
+
+  const hi = createHmac("sha1", password)
+    .update(Buffer.concat([Buffer.from(saltBase64, "base64"), INT_1]))
+    .digest();
+  let hi2: Buffer = hi;
+  for (let i = 1; i < iteractionCount; ++i) {
+    hi2 = createHmac("sha1", password).update(hi2).digest();
+    for (const [j, b] of hi2.entries()) hi[j] ^= b;
+  }
+
+  saltedPasswordCache.saltBase64 = saltBase64;
+  saltedPasswordCache.password = password;
+  saltedPasswordCache.iterationCount = iteractionCount;
+  saltedPasswordCache.salted = hi;
+
+  return hi;
+}
+
+function* loginPlain(
+  socket: net.Socket,
+  username: string,
+  password: string
+): Generator<void, void, Element> {
+  socket.write(
+    `<auth xmlns="urn:ietf:params:xml:ns:xmpp-sasl" mechanism="PLAIN">${encodeBase64(
+      `\x00${username}\x00${password}`
+    )}</auth>`
+  );
+  const res1 = yield;
+
+  if (
+    res1.name === "failure" &&
+    res1.children.some((c) => c.name === "not-authorized")
+  )
+    throw new Error("Not authorized");
+
+  if (res1.name !== "success")
+    throw new Error(`Unexpected response ${res1.name}`);
+}
+
+function* loginScram(
+  socket: net.Socket,
+  username: string,
+  password: string
+): Generator<void, void, Element> {
+  const cnonce = randomBytes(8).toString("base64");
+  const gs2Header = "n,,";
+  const clientFirstMessageBare = `n=${username},r=${cnonce}`;
+  const clientFirstMessage = gs2Header + clientFirstMessageBare;
+  socket.write(
+    `<auth xmlns="urn:ietf:params:xml:ns:xmpp-sasl" mechanism="SCRAM-SHA-1">${encodeBase64(
+      clientFirstMessage
+    )}</auth>`
+  );
+  const res1 = yield;
+  if (res1.name !== "challenge")
+    throw new Error(`Unexpected element ${res1.name}`);
+  const serverFirstMessage = decodeBase64(res1.text);
+
+  let iterationCount: number;
+  let saltBase64: string;
+  let nonce: string;
+  for (const s of serverFirstMessage.split(",")) {
+    if (s.startsWith("i=")) iterationCount = parseInt(s.slice(2));
+    else if (s.startsWith("s=")) saltBase64 = s.slice(2);
+    else if (s.startsWith("r=")) nonce = s.slice(2);
+  }
+  if (iterationCount == null || isNaN(iterationCount))
+    throw new Error("Invalid iteration count");
+  if (saltBase64 == null) throw new Error("Missing salt");
+  if (nonce == null) throw new Error("Missing nonce");
+
+  const saltedPassword = saltPassword(password, saltBase64, iterationCount);
+  const clientKey = createHmac("sha1", saltedPassword)
+    .update("Client Key")
+    .digest();
+  const storedKey = createHash("sha1").update(clientKey).digest();
+
+  const clientFinalMessageWithoutProof = `c=${encodeBase64(
+    gs2Header
+  )},r=${nonce}`;
+  const authMessage = `${clientFirstMessageBare},${serverFirstMessage},${clientFinalMessageWithoutProof}`;
+  const clientSignature = createHmac("sha1", storedKey)
+    .update(authMessage)
+    .digest();
+
+  const clientProof = Buffer.from(clientKey);
+  for (const [i, b] of clientSignature.entries()) clientProof[i] ^= b;
+
+  const clientFinalMessage = `${clientFinalMessageWithoutProof},p=${clientProof.toString(
+    "base64"
+  )}`;
+  socket.write(
+    `<response xmlns="urn:ietf:params:xml:ns:xmpp-sasl">${encodeBase64(
+      clientFinalMessage
+    )}</response>`
+  );
+  const res2 = yield;
+
+  if (
+    res2.name === "failure" &&
+    res2.children.some((c) => c.name === "not-authorized")
+  )
+    throw new Error("Not authorized");
+
+  if (res2.name !== "success")
+    throw new Error(`Unexpected response ${res2.name}`);
+
+  const serverKey = createHmac("sha1", saltedPassword)
+    .update("Server Key")
+    .digest();
+  const serverSignature = createHmac("sha1", serverKey)
+    .update(authMessage)
+    .digest("base64");
+
+  if (!decodeBase64(res2.text).endsWith(serverSignature))
+    throw new Error("Invalid server signature");
+}
+
+function* starttls(socket: net.Socket): Generator<void, void, Element> {
+  socket.write("<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>");
+  const res1 = yield;
+  if (res1.name !== "proceed") throw new Error("Failed to initiate STARTTLS");
+}
+
+function* bind(
+  socket: net.Socket,
+  resource: string
+): Generator<void, void, Element> {
+  const id = randomBytes(8).toString("base64");
+  socket.write(
+    `<iq id='${id}' type='set'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'><resource>${resource}</resource></bind></iq>`
+  );
+  const res1 = yield;
+  if (res1.name !== "iq") throw new Error(`Unexpected element ${res1.name}`);
+  const attrs1 = parseAttrs(res1.attrs);
+  const idAttr = attrs1.find((a) => a.name === "id");
+  if (!idAttr || idAttr.value !== id) throw new Error("Invalid ID");
+  const typeAttr = attrs1.find((a) => a.name === "type");
+  if (!typeAttr) throw new Error("Missing type attribute");
+  if (typeAttr.value !== "result") throw new Error("Cannot bind to resource");
+}
+
+const STATUS_RESTART_STREAM = 1;
+const STATUS_STARTTLS = 2;
+
+function* init(
+  socket: net.Socket,
+  host: string,
+  username: string,
+  password: string,
+  resource: string
+): Generator<void, number, Element> {
+  socket.write(
+    `<?xml version='1.0'?><stream:stream from='${username}@${host}' to='${host}' version='1.0' xml:lang='en' xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams'>`
+  );
+  const open = yield;
+  if (open.name !== "stream:stream")
+    throw new Error(`Unexpected element ${open.name}`);
+  const features = yield;
+  if (features.name !== "stream:features")
+    throw new Error(`Unexpected element ${features.name}`);
+  for (const feature of features.children) {
+    if (feature.name === "starttls") {
+      if (feature.children.some((c) => c.name === "required")) {
+        yield* starttls(socket);
+        return STATUS_STARTTLS;
+      }
+    } else if (feature.name === "mechanisms") {
+      const mechanisms: Set<string> = new Set(
+        feature.children.map((c) => c.text)
+      );
+      if (mechanisms.has("PLAIN")) {
+        yield* loginPlain(socket, username, password);
+        return STATUS_RESTART_STREAM;
+      } else if (mechanisms.has("SCRAM-SHA-1")) {
+        yield* loginScram(socket, username, password);
+        return STATUS_RESTART_STREAM;
+      } else {
+        throw new Error("No supported SASL method");
+      }
+    } else if (feature.name === "bind") {
+      yield* bind(socket, resource);
+      return 0;
+    }
+  }
+  return 0;
+}
+
+function upgradeTls(socket: net.Socket, host: string): Promise<tls.TLSSocket> {
+  return new Promise((resolve, reject) => {
+    socket.on("error", reject);
+    const newSocket = tls.connect({ socket, host }, () => {
+      socket.removeListener("error", reject);
+      resolve(newSocket);
+    });
+  });
+}
+
+interface XmppClientOptions {
+  host: string;
+  port?: number;
+  username?: string;
+  password?: string;
+  resource?: string;
+  timeout?: number;
+}
+
+export default class XmppClient extends EventEmitter {
+  private _socket: net.Socket;
+  private _host: string;
+  private _username: string;
+  private _resource: string;
+  private _iqStanzaCallbacks: Map<
+    string,
+    (err: Error, r?: { rawRes: string; res: Element }) => void
+  >;
+
+  private constructor() {
+    super();
+    this._socket = null;
+    this._host = null;
+    this._username = null;
+    this._resource = null;
+    this._iqStanzaCallbacks = new Map();
+  }
+
+  static connect(opts: XmppClientOptions): Promise<XmppClient> {
+    return new Promise((resolve, reject) => {
+      let socket = new net.Socket();
+      socket.on("error", reject);
+
+      socket.connect(opts.port || 5222, opts.host, async () => {
+        socket.removeListener("error", reject);
+        try {
+          let status = 1;
+          while (status) {
+            if (status === STATUS_STARTTLS)
+              socket = await upgradeTls(socket, opts.host);
+            status = await xmppStream(
+              socket,
+              init(
+                socket,
+                opts.host,
+                opts.username,
+                opts.password,
+                opts.resource
+              )
+            );
+          }
+
+          const client = new XmppClient();
+          client._socket = socket;
+          client._host = opts.host;
+          client._username = opts.username;
+          client._resource = opts.resource;
+          socket.on("data", client._onData.bind(client));
+          socket.on("error", client._onError.bind(client));
+          if (opts.timeout)
+            socket.setTimeout(opts.timeout, client.close.bind(client));
+          resolve(client);
+        } catch (err) {
+          socket.destroy();
+          reject(err);
+        }
+      });
+    });
+  }
+
+  close(): void {
+    this._socket.end("</stream:stream>");
+    this._socket.removeAllListeners("data");
+    this._socket.removeAllListeners("error");
+    this.emit("close");
+  }
+
+  ref(): void {
+    this._socket.ref();
+  }
+
+  unref(): void {
+    this._socket.unref();
+  }
+
+  get host(): string {
+    return this._host;
+  }
+
+  get username(): string {
+    return this._username;
+  }
+
+  get resource(): string {
+    return this._resource;
+  }
+
+  private _onData(chunk: Buffer): void {
+    try {
+      let close = false;
+      let str = chunk.toString("utf8");
+      if (str.endsWith("</stream:stream>")) {
+        str = str.slice(0, -16);
+        close = true;
+      }
+      const xml = parseXml(str);
+      const idx = xml.children.map((c) => c.bodyIndex);
+      for (const [i, c] of xml.children.entries()) {
+        const s = str.slice(idx[i], idx[i + 1]);
+        if (c.name === "iq") {
+          const attrs = parseAttrs(c.attrs);
+          const id = attrs.find((a) => a.name === "id");
+          if (id) {
+            const cb = this._iqStanzaCallbacks.get(id.value);
+            if (cb) cb(null, { rawRes: s, res: c });
+          }
+        }
+        this.emit("stanza", c, s);
+      }
+      if (close) {
+        this._socket.removeAllListeners("data");
+        this._socket.removeAllListeners("error");
+        this.emit("close");
+      }
+    } catch (err) {
+      this._socket.removeAllListeners("data");
+      this._socket.removeAllListeners("error");
+      this.emit("error", err);
+    }
+  }
+
+  private _onError(err: Error): void {
+    this._socket.end();
+    this.emit("error", err);
+    for (const cb of this._iqStanzaCallbacks.values()) cb(err);
+  }
+
+  send(msg: string): void {
+    this._socket.write(msg);
+  }
+
+  sendIqStanza(
+    from: string,
+    to: string,
+    type: string,
+    body: string,
+    timeout = 3000
+  ): Promise<{ rawReq: string; rawRes: string; res: Element }> {
+    return new Promise((resolve, reject) => {
+      const id = randomBytes(8).toString("base64");
+      const rawReq = `<iq from="${from}" to="${to}" id="${id}" type="${type}">${body}</iq>`;
+      this.send(rawReq);
+      const t = setTimeout(() => {
+        this._iqStanzaCallbacks.delete(id);
+        reject(
+          new Error("Did not receive IQ stanza response in a timely manner")
+        );
+      }, timeout);
+      this._iqStanzaCallbacks.set(id, (err, r) => {
+        this._iqStanzaCallbacks.delete(id);
+        clearTimeout(t);
+        if (err) reject(err);
+        else resolve(Object.assign(r, { rawReq }));
+      });
+    });
+  }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.8",
+  "version": "1.2.3",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -14,69 +14,73 @@
   ],
   "author": "Zaid Abdulla <zaid@genieacs.com>",
   "license": "AGPL-3.0",
-  "private": true,
+  "preferGlobal": true,
   "directories": {
-    "bin": "bin"
+    "bin": "bin",
+    "tools": "tools"
   },
   "dependencies": {
-    "@breejs/later": "^4.1.0",
-    "espresso-iisojs": "^1.0.4",
-    "iconv-lite": "^0.6.3",
-    "ipaddr.js": "^2.0.1",
+    "espresso-iisojs": "^1.0.3",
+    "iconv-lite": "^0.6.2",
+    "ipaddr.js": "^2.0.0",
     "jsonwebtoken": "^8.5.1",
-    "koa": "^2.13.4",
+    "koa": "^2.13.0",
     "koa-bodyparser": "^4.3.0",
-    "koa-compress": "^5.1.0",
-    "koa-jwt": "^4.0.3",
-    "koa-router": "^10.1.1",
+    "koa-compress": "^5.0.1",
+    "koa-jwt": "^4.0.0",
+    "koa-router": "^9.4.0",
     "koa-static": "^5.0.0",
-    "mongodb": "^4.1.3",
-    "parsimmon": "^1.18.0",
-    "seedrandom": "^3.0.5"
+    "later": "^1.2.0",
+    "mongodb": "^3.6.2",
+    "parsimmon": "^1.16.0",
+    "seedrandom": "^3.0.5",
+    "yaml": "^1.10.0"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^21.0.1",
+    "@rollup/plugin-commonjs": "^16.0.0",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^13.0.6",
-    "@rollup/plugin-typescript": "^8.3.0",
-    "@types/codemirror": "^5.60.5",
-    "@types/jsonwebtoken": "^8.5.5",
-    "@types/koa": "^2.13.4",
-    "@types/koa-compress": "^4.0.3",
-    "@types/koa-router": "^7.4.4",
-    "@types/koa-static": "^4.0.2",
-    "@types/mithril": "^2.0.8",
-    "@types/node": "^16.11.6",
-    "@types/parsimmon": "^1.10.6",
-    "@types/seedrandom": "^3.0.1",
-    "@typescript-eslint/eslint-plugin": "^5.2.0",
-    "@typescript-eslint/parser": "^5.2.0",
-    "ava": "^3.15.0",
-    "codemirror": "^5.63.3",
-    "cssnano": "^5.0.8",
-    "eslint": "^8.1.0",
-    "eslint-config-prettier": "^8.3.0",
+    "@rollup/plugin-node-resolve": "^10.0.0",
+    "@rollup/plugin-typescript": "^6.0.0",
+    "@types/jsonwebtoken": "^8.5.0",
+    "@types/koa": "^2.11.6",
+    "@types/koa-compress": "^4.0.1",
+    "@types/koa-router": "^7.4.1",
+    "@types/koa-static": "^4.0.1",
+    "@types/mithril": "^2.0.5",
+    "@types/mongodb": "^3.5.31",
+    "@types/node": "^14.14.5",
+    "@types/parsimmon": "^1.10.3",
+    "@types/seedrandom": "^2.4.28",
+    "@types/svgo": "^1.3.3",
+    "@typescript-eslint/eslint-plugin": "^4.6.0",
+    "@typescript-eslint/parser": "^4.6.0",
+    "ava": "^3.13.0",
+    "codemirror": "^5.58.2",
+    "cssnano": "^4.1.10",
+    "eslint": "^7.12.1",
+    "eslint-config-prettier": "^6.14.0",
     "esm": "^3.2.25",
+    "jsbi": "^3.1.4",
     "mithril": "^2.0.4",
-    "postcss": "^8.3.11",
-    "postcss-import": "^14.0.2",
+    "postcss": "^8.1.4",
+    "postcss-import": "^13.0.0",
     "postcss-preset-env": "^6.7.0",
-    "prettier": "^2.4.1",
-    "rollup": "^2.58.3",
+    "prettier": "^2.1.2",
+    "rollup": "^2.32.1",
     "rollup-plugin-terser": "^7.0.2",
-    "sql.js": "^1.6.2",
-    "svgo": "^2.7.0",
-    "ts-node": "^10.4.0",
-    "typescript": "^4.4.4",
-    "yaml": "^1.10.2"
+    "source-map-loader": "^1.1.1",
+    "svgo": "^1.3.2",
+    "ts-node": "^9.0.0",
+    "typescript": "^4.0.5",
+    "webpack": "^5.2.0"
   },
   "engines": {
     "node": ">=12.13.0"
   },
   "scripts": {
     "test": "ava --verbose --color",
-    "lint-eslint": "eslint --fix \"bin/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\"",
-    "lint-prettier": "prettier --prose-wrap always --write .",
+    "lint-eslint": "eslint --fix \"bin/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/*.ts\"",
+    "lint-prettier": "prettier --parser typescript --write \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
     "lint-tsc": "tsc --noEmit",
     "lint": "npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
     "build": "node -r esm -r ts-node/register/transpile-only build/build"

test/boolean-expression.ts

@@ -1,118 +0,0 @@
-import ava, { TestInterface } from "ava";
-import { minimize, unionDiff } from "../lib/common/boolean-expression";
-import { parse, stringify } from "../lib/common/expression-parser";
-import initSqlJs from "sql.js";
-
-const test = ava as TestInterface<{
-  query: (filter: string) => Promise<Set<number>>;
-}>;
-
-const STRING_VALUES = [null, "", "a", "ab", "ab10", "ab-10"];
-const DECIMAL_VALUES = [null, 0, -10, 10];
-
-test.before(async (t) => {
-  const sql = await initSqlJs();
-  const db = new sql.Database();
-
-  t.context.query = async (filter: string): Promise<Set<number>> => {
-    const res = db.exec(`SELECT id FROM test WHERE ${filter}`);
-    if (!res.length) return new Set();
-    return new Set(res[0].values.flat());
-  };
-
-  db.run(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY, string STRING, decimal DECIMAL(4,2))"
-  );
-
-  const stmt = db.prepare("INSERT INTO test (string, decimal) VALUES (?, ?)");
-  const count = STRING_VALUES.length * DECIMAL_VALUES.length;
-  for (let i = 0; i < count; ++i) {
-    const str = i % STRING_VALUES.length;
-    const dec = Math.trunc(i / STRING_VALUES.length) % DECIMAL_VALUES.length;
-    stmt.run([STRING_VALUES[str], DECIMAL_VALUES[dec]]);
-  }
-  stmt.free();
-});
-
-function setsEqual(set1: Set<number>, set2: Set<number>): boolean {
-  if (set1.size !== set2.size) return false;
-  for (const s of set1) if (!set2.has(s)) return false;
-  return true;
-}
-
-function getPermutations(...arrs: any[][]): any[][] {
-  const count = arrs.reduce((total, arr) => total * arr.length, 1);
-  const res = [];
-  for (let i = 0; i < count; ++i) {
-    let j = i;
-    const row = [];
-    for (const arr of arrs) {
-      const v = arr[j % arr.length];
-      j = Math.trunc(j / arr.length);
-      row.push(v);
-    }
-    res.push(row);
-  }
-  return res;
-}
-
-test("minimize", async (t) => {
-  const cases: string[] = [];
-
-  cases.push("null");
-  cases.push("false");
-  cases.push("true");
-  cases.push("string");
-  cases.push("(string + decimal) IS NULL");
-  cases.push("(string + decimal) = NULL");
-  cases.push("COALESCE(string, decimal) = 0");
-
-  for (const [s1, s2, s3, op1, op2] of getPermutations(
-    STRING_VALUES.filter((s) => s),
-    STRING_VALUES.filter((s) => s),
-    STRING_VALUES.filter((s) => s),
-    [">", "=", "<"],
-    ["<>", ">="]
-  )) {
-    cases.push(
-      `string ${op1} "${s1}" OR string ='${s2}' OR NOT string ${op2} '${s3}'`
-    );
-  }
-
-  for (const [s1, s2] of getPermutations(
-    STRING_VALUES.filter((s) => s),
-    STRING_VALUES.filter((s) => s)
-  ))
-    cases.push(`string > "${s1}" AND string < '${s2}'`);
-
-  for (const c of cases) {
-    const res1 = await t.context.query(c);
-    const min = stringify(minimize(parse(c), true));
-    const res2 = await t.context.query(min);
-    t.true(setsEqual(res1, res2));
-  }
-});
-
-test("unionDiff", async (t) => {
-  const cases = [
-    "true",
-    "decimal > 0",
-    "decimal > 10",
-    "UPPER(string || decimal) LIKE 'AB10'",
-    "COALESCE(string, decimal) = 0",
-  ];
-
-  for (const [c1, c2] of getPermutations(cases, cases)) {
-    const res1 = await t.context.query(c1);
-    const res2 = await t.context.query(c2);
-    const [union, diff] = unionDiff(parse(c1), parse(c2));
-    const res3 = await t.context.query(stringify(union));
-    const res4 = await t.context.query(stringify(diff));
-
-    const unionSet = new Set([...res1, ...res2]);
-    const diffSet = new Set(Array.from(res2).filter((r) => !res1.has(r)));
-
-    t.true(setsEqual(res3, unionSet));
-    t.true(setsEqual(res4, diffSet));
-  }
-});

test/mongodb-functions.ts

@@ -49,16 +49,8 @@ ava("convertOldPrecondition", (t) => {
   ];
 
   t.plan(tests.length + 2 * shouldFailTests.length);
-  for (const test of tests) {
-    t.is(
-      stringify(
-        mongodbFunctions.convertOldPrecondition(
-          test[0] as Record<string, unknown>
-        )
-      ),
-      test[1]
-    );
-  }
+  for (const test of tests)
+    t.is(stringify(mongodbFunctions.convertOldPrecondition(test[0])), test[1]);
 
   for (const test of shouldFailTests) {
     const func = (): void => {

test/path.ts

@@ -1,25 +1,6 @@
 import ava from "ava";
 import Path from "../lib/common/path";
 
-ava("parse", (t) => {
-  t.throws(() => Path.parse("."));
-  t.throws(() => Path.parse("a "));
-  t.throws(() => Path.parse(".a"));
-  t.throws(() => Path.parse("a."));
-  t.throws(() => Path.parse("a..b"));
-  t.throws(() => Path.parse("b*"));
-  t.throws(() => Path.parse("*b"));
-  t.throws(() => Path.parse("a.b c.d"));
-  t.throws(() => Path.parse("a["));
-  t.throws(() => Path.parse("a[b"));
-  t.throws(() => Path.parse("a[b:"));
-  t.throws(() => Path.parse('a[b:"waef]'));
-  t.notThrows(() => Path.parse("[]"));
-  t.notThrows(() => Path.parse("[a:]"));
-  t.notThrows(() => Path.parse("*"));
-  t.notThrows(() => Path.parse(""));
-});
-
 ava("toString", (t) => {
   const path1 = Path.parse('abc.[ abc : 123 , 123 : abc , 123: " abc "].123');
   const path2 = Path.parse('abc.[123:  " abc ",abc:123,123:abc].123');

test/yaml-tests.json

@@ -1,861 +0,0 @@
-[
-  [
-    {
-      "name": "Mark McGwire",
-      "hr": 65,
-      "avg": 0.278
-    },
-    {
-      "name": "Sammy Sosa",
-      "hr": 63,
-      "avg": 0.288
-    }
-  ],
-  {
-    "top1": {
-      "key1": "scalar1"
-    },
-    "top2": {
-      "key2": "scalar2"
-    },
-    "top3": {
-      "scalar1": "scalar3"
-    },
-    "top4": {
-      "scalar2": "scalar4"
-    },
-    "top5": "scalar5",
-    "top6": {
-      "key6": "scalar6"
-    }
-  },
-  "text",
-  ["a", "b", 42, "d"],
-  {
-    "a!\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~": "safe",
-    "?foo": "safe question mark",
-    ":foo": "safe colon",
-    "-foo": "safe dash",
-    "this is#not": "a comment"
-  },
-  {
-    "": "a"
-  },
-  "foo",
-  {
-    "key": "value",
-    "foo": "key"
-  },
-  [1, -2, 33],
-  {
-    "plain": "a b\nc"
-  },
-  [["s1_i1", "s1_i2"], "s2"],
-  {
-    "First occurrence": "Foo",
-    "Second occurrence": "Foo",
-    "Override anchor": "Bar",
-    "Reuse anchor": "Bar"
-  },
-  "k:#foo &a !t s",
-  ["a"],
-  {
-    "escaped slash": "a/b"
-  },
-  {
-    "plain": "This unquoted scalar spans many lines.",
-    "quoted": "So does this quoted scalar.\n"
-  },
-  "here's to \"quotes\"",
-  {
-    "foo": "bar"
-  },
-  "ab cd\nef\n\ngh\n",
-  ["detected\n", "\n\n# detected\n", " explicit\n", "detected\n"],
-  "foo: bar\": baz",
-  "plain\\value\\with\\backslashes",
-  {
-    "plain": "text lines",
-    "quoted": "text lines",
-    "block": "text\n \tlines\n"
-  },
-  "a",
-  {
-    "foo": "you",
-    "bar": "far"
-  },
-  {
-    "sequence": ["entry", ["nested"]],
-    "mapping": {
-      "foo": "bar"
-    }
-  },
-  {
-    "literal": "some\ntext\n",
-    "folded": "some text\n"
-  },
-  [
-    {
-      "one": "two",
-      "three": "four"
-    },
-    {
-      "five": "six",
-      "seven": "eight"
-    }
-  ],
-  {
-    "Folding": "Empty line\nas a line feed",
-    "Chomping": "Clipped empty lines\n"
-  },
-  [
-    ["one", "two"],
-    ["three", "four"]
-  ],
-  {
-    "foo": "bar"
-  },
-  {
-    "key": "value"
-  },
-  {
-    "explicit key": null,
-    "block key\n": ["one", "two"]
-  },
-  ["foo"],
-  [
-    {
-      "foo": "bar"
-    },
-    ["baz", "baz"]
-  ],
-  "ab\n\n \n",
-  {
-    "foo: bar\\": "baz'"
-  },
-  {
-    "Not indented": {
-      "By one space": "By four\n  spaces\n",
-      "Flow style": ["By two", "Also by two", "Still by two"]
-    }
-  },
-  "\\//||\\/||\n// ||  ||__\n",
-  {
-    "foo": [
-      "a",
-      {
-        "key": "value"
-      }
-    ]
-  },
-  {
-    "a": null,
-    "b": null
-  },
-  "foo",
-  {
-    "a": "b",
-    "": "a"
-  },
-  {
-    "foo\nbar:baz\tx \\$%^&*()x": 23,
-    "x\\ny:z\\tx $%^&*()x": 24
-  },
-  "Sammy Sosa completed another fine season with great stats.\n\n  63 Home Runs\n  0.288 Batting Average\n\nWhat a year!\n",
-  " foo\nbar\nbaz ",
-  { "matches %": 20 },
-  [
-    "flow in block",
-    "Block scalar\n",
-    {
-      "foo": "bar"
-    }
-  ],
-  {
-    "a": "b",
-    "c": 42,
-    "e": "f",
-    "g": "h",
-    "23": false
-  },
-  "ab",
-  " 1st non-empty\n2nd non-empty 3rd non-empty ",
-  {
-    "top1": {
-      "key1": "one"
-    },
-    "top2": {
-      "key2": "two"
-    },
-    "top3": {
-      "key3": "three"
-    },
-    "top4": {
-      "key4": "four"
-    },
-    "top5": {
-      "key5": "five"
-    },
-    "top6": "six",
-    "top7": "seven"
-  },
-  {
-    "hr": ["Mark McGwire", "Sammy Sosa"],
-    "rbi": ["Sammy Sosa", "Ken Griffey"]
-  },
-  "\nfolded line\nnext line\n  * bullet\n\n  * list\n  * lines\n\nlast line\n",
-  ["word1", "word2"],
-  {
-    "a": null,
-    "b": null,
-    "c": null
-  },
-  {
-    "nested sequences": [[[[]]], [[{}]]],
-    "key1": [],
-    "key2": {}
-  },
-  "---word1 word2",
-  {
-    "implicit block key": [
-      {
-        "implicit flow key": "value"
-      }
-    ]
-  },
-  {
-    "key ends with two colons::": "value"
-  },
-  [
-    {
-      "single line": null,
-      "a": "b"
-    },
-    {
-      "multi line": null,
-      "a": "b"
-    }
-  ],
-  "a",
-  {
-    "key": ["item1", "item2"]
-  },
-  [
-    "double quoted",
-    "single quoted",
-    "plain text",
-    ["nested"],
-    {
-      "single": "pair"
-    }
-  ],
-  ["unicode anchor"],
-  [
-    {
-      "key": "value",
-      "key2": "value2"
-    },
-    {
-      "key3": "value3"
-    }
-  ],
-  "trimmed\n\n\nas space",
-  "Mark McGwire's year was crippled by a knee injury.\n",
-  [
-    {
-      "single line": null,
-      "a": "b"
-    },
-    {
-      "multi line": null,
-      "a": "b"
-    }
-  ],
-  {
-    "a": {
-      "b": {
-        "c": "d"
-      },
-      "e": {
-        "f": "g"
-      }
-    },
-    "h": "i"
-  },
-  {
-    "foo": {
-      "bar": "baz"
-    }
-  },
-  [
-    {
-      "single line": "value"
-    },
-    {
-      "multi line": "value"
-    }
-  ],
-  {
-    "single": "text",
-    "double": "text"
-  },
-  " 1st non-empty\n2nd non-empty 3rd non-empty ",
-  [
-    {
-      "item": "Super Hoop",
-      "quantity": 1
-    },
-    {
-      "item": "Basketball",
-      "quantity": 4
-    },
-    {
-      "item": "Big Shoes",
-      "quantity": 1
-    }
-  ],
-  "a b c d\ne",
-  {
-    "a": ["b", ["c", "d"]]
-  },
-  {
-    "strip": "text",
-    "clip": "text\n",
-    "keep": "text\n"
-  },
-  {
-    "a": "b c",
-    "d": "e f"
-  },
-  ["single multiline - sequence entry"],
-  {
-    "1": [2, 3],
-    "4": 5
-  },
-  [
-    {
-      "bla\"keks": "foo"
-    },
-    {
-      "bla]keks": "foo"
-    }
-  ],
-  "folded text\n",
-  "foo",
-  {
-    "key": {
-      "a": "b"
-    }
-  },
-  {
-    "adjacent": "value",
-    "readable": "value",
-    "empty": null
-  },
-  [
-    {
-      "a": "b"
-    },
-    {
-      "c": "d"
-    },
-    {
-      "e": "f"
-    },
-    {
-      "g": "h"
-    }
-  ],
-  {
-    "tab": "\tstring"
-  },
-  [
-    {
-      "foo bar": "baz"
-    }
-  ],
-  {
-    "anchored": "value",
-    "alias": "value"
-  },
-  ["explicit indent and chomp", "chomp and explicit indent"],
-  {
-    "a": ["b", "c"]
-  },
-  {
-    "foo": "bar"
-  },
-  [
-    "::vector",
-    ": - ()",
-    "Up, up, and away!",
-    -123,
-    "http://example.com/foo#bar",
-    [
-      "::vector",
-      ": - ()",
-      "Up, up and away!",
-      -123,
-      "http://example.com/foo#bar"
-    ]
-  ],
-  {
-    "a": "b",
-    "seq": ["a"],
-    "c": "d"
-  },
-  ["foo", "bar", 42],
-  "line1 # no comment line3\n",
-  "\n\nliteral\n \n\ntext\n",
-  {
-    "a": "b"
-  },
-  {
-    "k": ["a", "b"]
-  },
-  "a b c d\ne",
-  "---word1 word2",
-  ["a", 2, 4, "d"],
-  {
-    "a": [
-      "b",
-      "c",
-      {
-        "d": ["e", "f"]
-      }
-    ]
-  },
-  {
-    "a": " more indented\nregular\n",
-    "b": "\n\n more indented\nregular\n"
-  },
-  {
-    "strip": "# text",
-    "clip": "# text\n",
-    "keep": "# text\n\n"
-  },
-  {
-    "safe": "a!\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~ !\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~",
-    "safe question mark": "?foo",
-    "safe colon": ":foo",
-    "safe dash": "-foo"
-  },
-  "line1 line2 line3\n",
-  ["Mark McGwire", "Sammy Sosa", "Ken Griffey"],
-  ["a"],
-  ["a", ["b", "c"]],
-  {
-    "unicode": "Sosa did fine.☺",
-    "control": "\b1998\t1999\t2000\n",
-    "hex esc": "\r\n is \r\n",
-    "single": "\"Howdy!\" he cried.",
-    "quoted": " # Not a 'comment'.",
-    "tie-fighter": "|\\-*-/|"
-  },
-  [["-", "-"]],
-  "folded text\n",
-  {
-    "a": 13,
-    "1.5": "d"
-  },
-  {
-    "foo": 1,
-    "bar": 2,
-    "text": "a\n  \nb\n\nc\n\nd\n"
-  },
-  {
-    "wanted": "love ♥ and peace ☮"
-  },
-  {
-    "name": "Mark McGwire",
-    "accomplishment": "Mark set a major league home run record in 1998.\n",
-    "stats": "65 Home Runs\n0.278 Batting Average\n"
-  },
-  {
-    "foo": "bar",
-    "baz": "foo"
-  },
-  "1st non-empty\n2nd non-empty 3rd non-empty",
-  {
-    "quoted": "Quoted \t",
-    "block": "void main() {\n\tprintf(\"Hello, world!\\n\");\n}\n"
-  },
-  {
-    "foo": "blue",
-    "bar": "arrr",
-    "baz": "jazz"
-  },
-  [
-    {
-      "Mark McGwire": 65
-    },
-    {
-      "Sammy Sosa": 63
-    },
-    {
-      "Ken Griffy": 58
-    }
-  ],
-  {
-    "1": 2,
-    "3": 4
-  },
-  {
-    "hr": ["Mark McGwire", "Sammy Sosa"],
-    "rbi": ["Sammy Sosa", "Ken Griffey"]
-  },
-  "k:#foo &a !t s",
-  {
-    "block sequence": [
-      "one",
-      {
-        "two": "three"
-      }
-    ]
-  },
-  {
-    "First occurrence": "Value",
-    "Second occurrence": "Value"
-  },
-  {
-    "a true": "null d",
-    "e 42": null
-  },
-  {
-    "foo": "bar"
-  },
-  ["foo", "bar", 42],
-  "trimmed\n\n\nas space",
-  "scalar",
-  {
-    "strip": "",
-    "clip": "",
-    "keep": "\n"
-  },
-  {
-    "foo": {
-      "bar": 1
-    },
-    "baz": 2
-  },
-  {
-    "a": 47,
-    "c": "d"
-  },
-  {
-    "implicit block key": [
-      {
-        "implicit flow key": "value"
-      }
-    ]
-  },
-  ["a", "b", "c", "c", ""],
-  [
-    ["a", "b", "c"],
-    {
-      "a": "b",
-      "c": "d",
-      "e": "f"
-    },
-    []
-  ],
-  {
-    "implicit block key": [
-      {
-        "implicit flow key": "value"
-      }
-    ]
-  },
-  {
-    "a": "ab\n\ncd\nef\n"
-  },
-  {
-    "literal": "value\n",
-    "folded": "value\n"
-  },
-  {
-    "a": [
-      "b",
-      "c",
-      {
-        "d": ["e", "f"]
-      }
-    ]
-  },
-  "literal\n\ttext\n",
-  "foo \n\n\t bar\n\nbaz\n",
-  [
-    {
-      "a": "b"
-    }
-  ],
-  "ab",
-  ["plain", "double quoted", "single quoted", "block\n", "plain again"],
-  {
-    "a": " ",
-    "b": " ",
-    "c": " ",
-    "d": " ",
-    "e": "\n",
-    "f": "\n",
-    "g": "\n\n",
-    "h": "\n\n"
-  },
-  {
-    "key": "value with\ntabs"
-  },
-  {
-    "": null
-  },
-  [
-    {
-      "single line": "value"
-    },
-    {
-      "multi line": "value"
-    }
-  ],
-  "folded to a space,\nto a line feed, or \t \tnon-content",
-  ["literal\n", " folded\n", "keep\n\n", " strip"],
-  {
-    "key": "value"
-  },
-  {
-    "american": ["Boston Red Sox", "Detroit Tigers", "New York Yankees"],
-    "national": ["New York Mets", "Chicago Cubs", "Atlanta Braves"]
-  },
-  " 1st non-empty\n2nd non-empty 3rd non-empty ",
-  {},
-  [
-    ["a", "b"],
-    {
-      "a": "b"
-    },
-    "a",
-    "b",
-    "c"
-  ],
-  "folded to a space,\nto a line feed, or \t \tnon-content",
-  [
-    {
-      "foo": "bar"
-    }
-  ],
-  ["detected\n", "\n\n# detected\n", " explicit\n", "\t\ndetected\n"],
-  {
-    "top1": [
-      "item1",
-      {
-        "key2": "value2"
-      },
-      "item3"
-    ],
-    "top2": "value2"
-  },
-  {
-    "foo": [42],
-    "bar": [44]
-  },
-  {
-    "23": "d",
-    "a": 4.2
-  },
-  "Document",
-  {
-    "Date": "2001-11-23 15:03:17 -5",
-    "User": "ed",
-    "Fatal": "Unknown variable \"bar\"",
-    "Stack": [
-      {
-        "file": "TopClass.py",
-        "line": 23,
-        "code": "x = MoreObject(\"345\\n\")\n"
-      },
-      { "file": "MoreClass.py", "line": 58, "code": "foo = bar" }
-    ]
-  },
-  {
-    "plain key": "in-line value",
-    "": null,
-    "quoted key": ["entry"]
-  },
-  ["12", 12, "12"],
-  {
-    "aaa": "bbb"
-  },
-  [":,"],
-  {
-    "sequence": ["one", "two"],
-    "mapping": {
-      "sky": "blue",
-      "sea": "green"
-    }
-  },
-  {
-    "seq": ["a", "b"]
-  },
-  "here's to \"quotes\"",
-  {
-    "hr": 65,
-    "avg": 0.278,
-    "rbi": 147
-  },
-  "\n\nliteral\n \n\ntext\n",
-  " 1st non-empty\n2nd non-empty 3rd non-empty ",
-  "literal\n\ttext\n",
-  {
-    "block mapping": {
-      "key": "value"
-    }
-  },
-  " foo\nbar\nbaz ",
-  "ab cd\nef\n\ngh\n",
-  "foo",
-  {
-    "top1": {
-      "key1": "one"
-    },
-    "top2": {
-      "key2": "two"
-    },
-    "top3": {
-      "key3": "three"
-    },
-    "top4": {
-      "key4": "four"
-    },
-    "top5": {
-      "key5": "five"
-    },
-    "top6": "six",
-    "top7": "seven"
-  },
-  [
-    {
-      "url": "http://example.org"
-    }
-  ],
-  {
-    "sequence": ["one", "two"],
-    "mapping": {
-      "sky": "blue",
-      "sea": "green"
-    }
-  },
-  {
-    "invoice": 34843,
-    "date": "2001-01-23",
-    "bill-to": {
-      "given": "Chris",
-      "family": "Dumars",
-      "address": {
-        "lines": "458 Walkman Dr.\nSuite #292\n",
-        "city": "Royal Oak",
-        "state": "MI",
-        "postal": 48046
-      }
-    },
-    "ship-to": {
-      "given": "Chris",
-      "family": "Dumars",
-      "address": {
-        "lines": "458 Walkman Dr.\nSuite #292\n",
-        "city": "Royal Oak",
-        "state": "MI",
-        "postal": 48046
-      }
-    },
-    "product": [
-      {
-        "sku": "BL394D",
-        "quantity": 4,
-        "description": "Basketball",
-        "price": 450
-      },
-      {
-        "sku": "BL4438H",
-        "quantity": 1,
-        "description": "Super Hoop",
-        "price": 2392
-      }
-    ],
-    "tax": 251.42,
-    "total": 4443.52,
-    "comments": "Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338."
-  },
-  ["a", "b", "a", "b"],
-  [
-    null,
-    "block node\n",
-    ["one", "two"],
-    {
-      "one": "two"
-    }
-  ],
-  {
-    "a": "scalar a",
-    "b": "scalar a"
-  },
-  {
-    "foo": "",
-    "": "bar"
-  },
-  {
-    "key": "value"
-  },
-  "scalar %YAML 1.2",
-  {
-    "Folding": "Empty line\nas a line feed",
-    "Chomping": "Clipped empty lines\n"
-  },
-  {
-    "key": "value"
-  },
-  [
-    ["name", "hr", "avg"],
-    ["Mark McGwire", 65, 0.278],
-    ["Sammy Sosa", 63, 0.288]
-  ],
-  {
-    "literal": "value\n",
-    "folded": "value\n"
-  },
-  {
-    "Mark McGwire": {
-      "hr": 65,
-      "avg": 0.278
-    },
-    "Sammy Sosa": {
-      "hr": 63,
-      "avg": 0.288
-    }
-  },
-  {
-    "a": "b",
-    "c": "d"
-  },
-  {
-    "key": [[["value"]]]
-  },
-  {
-    "a": 1,
-    "b": null,
-    "c": 3
-  },
-  {
-    "event": "outgoing HTTP response",
-    "timestamp": "2021-01-21T18:44:57.116Z",
-    "remoteAddress": "127.0.0.1",
-    "deviceId": "202BC1-BM632w-000000",
-    "connection": "2021-01-21T18:44:57.105Z",
-    "statusCode": 200,
-    "headers": {
-      "content-length": 528,
-      "server": "GenieACS/1.2.3+20210121000910",
-      "soapserver": "GenieACS/1.2.3+20210121000910",
-      "content-type": "text/xml; charset=\"utf-8\"",
-      "set-cookie": "session=ea31d97ccf6832fb"
-    },
-    "body": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<soap-env:Envelope xmlns:soap-enc=\"http://schemas.xmlsoap.org/soap/encoding/\" xmlns:soap-env=\"http://schemas.xmlsoap.org/soap/envelope/\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:cwmp=\"urn:dslforum-org:cwmp-1-0\"><soap-env:Header><cwmp:ID soap-env:mustUnderstand=\"1\">wdsx50vq</cwmp:ID></soap-env:Header><soap-env:Body><cwmp:InformResponse><MaxEnvelopes>1</MaxEnvelopes></cwmp:InformResponse></soap-env:Body></soap-env:Envelope>"
-  }
-]

test/yaml.ts

@@ -1,14 +0,0 @@
-import ava from "ava";
-import * as yaml from "yaml";
-import { stringify } from "../lib/common/yaml";
-
-import testCases from "./yaml-tests.json";
-
-ava("stringify", (t) => {
-  for (const testCase of testCases) {
-    let str = stringify(testCase);
-    if (str.startsWith(">2")) str = ">3" + str.slice(2);
-    if (str.startsWith("|2")) str = "|3" + str.slice(2);
-    t.deepEqual(yaml.parse(yaml.stringify(testCase)), yaml.parse(str));
-  }
-});

tools/dump-data-model.ts

@@ -0,0 +1,82 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import { connect, devicesCollection } from "../lib/db";
+
+const outputFilename = path.resolve(`data_model_${process.argv[2]}.json`);
+process.chdir(path.resolve(path.dirname(fs.realpathSync(__filename)), ".."));
+
+const dataModel = {};
+
+function recursive(object, p = ""): void {
+  for (const key in object) {
+    if (key[0] === "_" || !object[key]) continue;
+
+    const isObject = object[key]["_object"];
+    const isValue = "_value" in object[key];
+
+    if ((isObject && isValue) || (!isObject && !isValue))
+      throw new Error("Unexpected object");
+
+    if (isValue) {
+      let value = object[key]["_value"];
+      if (typeof value === "boolean") value = +value;
+
+      const values = [object[key]["_writable"] || false, value.toString()];
+      if (object[key]["_type"]) values.push(object[key]["_type"]);
+
+      dataModel[`${p}${key}`] = values;
+    }
+
+    if (isObject) {
+      const values = [object[key]["_writable"] || false];
+      dataModel[`${p}${key}.`] = values;
+      recursive(object[key], `${p}${key}.`);
+    }
+  }
+}
+
+connect()
+  .then(() => {
+    devicesCollection.findOne(
+      {
+        _id: process.argv[2],
+      },
+      {},
+      (err, device) => {
+        if (err) throw err;
+
+        if (!device) throw new Error("Device not found");
+
+        recursive(device);
+
+        fs.writeFile(outputFilename, JSON.stringify(dataModel, null, 2), () => {
+          if (err) throw err;
+
+          process.stdout.write(`${outputFilename} saved\n`);
+          process.exit(0);
+        });
+      }
+    );
+  })
+  .catch((err) => {
+    throw err;
+  });

tsconfig.json

@@ -1,9 +1,11 @@
 {
   "compilerOptions": {
-    "lib": ["ES2019", "dom"],
-    "module": "ES2020",
+    "lib": [
+      "esnext", "dom"
+    ],
+    "module": "esnext",
     "moduleResolution": "node",
-    "target": "ES2019",
+    "target": "esnext",
     "resolveJsonModule": true,
     "allowSyntheticDefaultImports": true,
     "alwaysStrict": true,
@@ -12,10 +14,11 @@
     "allowJs": true
   },
   "include": [
-    "./bin/*.ts",
-    "./lib/**/*",
-    "./ui/**/*",
-    "./test/**/*",
-    "./build/**/*"
+      "./bin/*.ts",
+      "./lib/**/*",
+      "./ui/**/*",
+      "./test/**/*",
+      "./build/**/*",
+      "./tools/*.ts"
   ]
 }

ui/app.ts

@@ -64,37 +64,50 @@ const adminPages = [
 
 let state;
 
+let lastRenderTimestamp = 0;
+let pageVisitTimestamp = 0;
+let fulfillTimeout = null;
+
+function fulfill(): void {
+  clearTimeout(fulfillTimeout);
+  fulfillTimeout = setTimeout(() => {
+    store.fulfill(lastRenderTimestamp, pageVisitTimestamp, (updated) => {
+      if (updated) m.redraw();
+    });
+  }, 100);
+}
+
 function pagify(pageName, page): RouteResolver {
   const component: RouteResolver = {
     render: () => {
-      const lastRenderTimestamp = Date.now();
+      lastRenderTimestamp = Date.now();
       let p;
-      if (state?.error) p = m(errorPage.component, state);
+      if (state && state.error) p = m(errorPage.component, state);
       else p = m(contextifyComponent(page.component), state);
+      fulfill();
       const attrs = {};
       attrs["page"] = pageName;
-      attrs["oncreate"] = attrs["onupdate"] = () => {
-        store.fulfill(lastRenderTimestamp);
-      };
       return m(layout, attrs, p);
     },
     onmatch: null,
   };
 
   component.onmatch = (args, requestedPath) => {
-    store.setTimestamp(Date.now());
+    pageVisitTimestamp = Date.now();
     if (!page.init) {
       state = null;
+      fulfill();
       return null;
     }
 
-    return new Promise<void>((resolve) => {
+    return new Promise((resolve) => {
       page
         .init(args)
         .then((st) => {
           if (!st) return void m.route.set("/");
           state = st;
           resolve();
+          fulfill();
         })
         .catch((err) => {
           if (!window.username && err.message.indexOf("authorized") >= 0) {

ui/autocomplete-compnent.ts

@@ -23,13 +23,13 @@ type AutocompleteCallback = (
 ) => void;
 
 export default class Autocomplete {
-  private declare callback: AutocompleteCallback;
-  private declare element: HTMLInputElement;
-  private declare hideTimeout: NodeJS.Timeout;
-  private declare visible: boolean;
-  private declare default: string;
-  private declare selection: number;
-  private declare container: HTMLElement;
+  private callback: AutocompleteCallback;
+  private element: HTMLInputElement;
+  private hideTimeout: NodeJS.Timeout;
+  private visible: boolean;
+  private default: string;
+  private selection: number;
+  private container: HTMLElement;
 
   public constructor(className: string, callback: AutocompleteCallback) {
     this.callback = callback;
@@ -51,8 +51,11 @@ export default class Autocomplete {
 
     el.addEventListener("focus", () => {
       this.element = el;
+      const domRect = el.getBoundingClientRect();
+      this.container.style.left = `${domRect.left + window.pageXOffset}px`;
+      this.container.style.width = `${domRect.width}px`;
+      this.container.style.top = `${domRect.bottom + window.pageYOffset}px`;
       this.update();
-      this.reposition();
     });
 
     el.addEventListener("blur", () => {
@@ -68,8 +71,7 @@ export default class Autocomplete {
       } else if (e.key === "Enter") {
         if (this.default != null) {
           el.value = this.default;
-          e.stopImmediatePropagation();
-          el.dispatchEvent(new InputEvent("input"));
+          e.preventDefault();
           this.update();
         }
       } else if (e.key === "ArrowDown") {
@@ -91,19 +93,6 @@ export default class Autocomplete {
     });
   }
 
-  public reposition(): void {
-    if (!this.element) return;
-    const domRect = this.element.getBoundingClientRect();
-    if (!domRect.width) {
-      // Element has been removed
-      if (this.visible) this.hide();
-      return;
-    }
-    this.container.style.left = `${domRect.left + window.pageXOffset}px`;
-    this.container.style.width = `${domRect.width}px`;
-    this.container.style.top = `${domRect.bottom + window.pageYOffset}px`;
-  }
-
   private hide(): void {
     this.container.style.opacity = "0";
     this.visible = false;
@@ -170,7 +159,6 @@ export default class Autocomplete {
         e.addEventListener("mousedown", (ev) => {
           ev.preventDefault();
           el.value = suggestion.value;
-          el.dispatchEvent(new InputEvent("input"));
           if (this.element === el) this.update();
         });
         this.container.appendChild(e);

ui/change-password-component.ts

@@ -17,23 +17,17 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { VnodeDOM, ClosureComponent } from "mithril";
+import { VnodeDOM, ClosureComponent, Component } from "mithril";
 import * as notifications from "./notifications";
 import { changePassword } from "./store";
 import { m } from "./components";
 
-interface Attrs {
-  noAuth?: boolean;
-  username?: string;
-  onPasswordChange: () => void;
-}
-
-const component: ClosureComponent<Attrs> = () => {
+const component: ClosureComponent = (): Component => {
   return {
     view: (vnode) => {
-      const onPasswordChange = vnode.attrs.onPasswordChange;
-      const enforceAuth = !vnode.attrs.noAuth;
-      const username = vnode.attrs.username;
+      const onPasswordChange = vnode.attrs["onPasswordChange"];
+      const enforceAuth = !vnode.attrs["noAuth"];
+      const username = vnode.attrs["username"];
 
       if (username) vnode.state["username"] = username;
 

ui/code-editor-component.ts

@@ -17,54 +17,41 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Editor } from "codemirror";
-import { ClosureComponent } from "mithril";
+import { ClosureComponent, Component } from "mithril";
 import { m } from "./components";
 import { codeMirror } from "./dynamic-loader";
 
-interface Attrs {
-  id: string;
-  value: string;
-  mode: string;
-  readOnly?: boolean;
-  focus?: boolean;
-  onSubmit?: (dom: Element) => void;
-  onChange?: (value: string) => void;
-  onReady?: (editor: Editor) => void;
-}
-
-const component: ClosureComponent<Attrs> = () => {
+const component: ClosureComponent = (): Component => {
   return {
     view: (vnode) => {
       return m("textarea", {
-        name: vnode.attrs.id,
-        value: vnode.attrs.value,
+        name: vnode.attrs["id"],
+        value: vnode.attrs["value"],
         oncreate: (_vnode) => {
-          const editor = codeMirror.fromTextArea(
-            _vnode.dom as HTMLTextAreaElement,
-            {
-              mode: vnode.attrs.mode,
-              lineNumbers: true,
-              readOnly: vnode.attrs.readOnly,
-              extraKeys: {
-                "Ctrl-Enter": () => {
-                  if (vnode.attrs.onSubmit) vnode.attrs.onSubmit(_vnode.dom);
-                },
-                "Cmd-Enter": () => {
-                  if (vnode.attrs.onSubmit) vnode.attrs.onSubmit(_vnode.dom);
-                },
+          const editor = codeMirror.fromTextArea(_vnode.dom, {
+            mode: vnode.attrs["mode"],
+            lineNumbers: true,
+            readOnly: vnode.attrs["readOnly"],
+            extraKeys: {
+              "Ctrl-Enter": () => {
+                if (vnode.attrs["onSubmit"])
+                  vnode.attrs["onSubmit"](_vnode.dom);
               },
-            }
-          );
+              "Cmd-Enter": () => {
+                if (vnode.attrs["onSubmit"])
+                  vnode.attrs["onSubmit"](_vnode.dom);
+              },
+            },
+          });
 
-          if (vnode.attrs.onChange) {
+          if (vnode.attrs["onChange"]) {
             editor.on("change", (e) => {
-              vnode.attrs.onChange(e.getValue());
+              vnode.attrs["onChange"](e.getValue());
             });
           }
 
-          if (vnode.attrs.focus) editor.focus();
-          if (vnode.attrs.onReady) vnode.attrs.onReady(editor);
+          if (vnode.attrs["focus"]) editor.focus();
+          if (vnode.attrs["onReady"]) vnode.attrs["onReady"](editor);
         },
       });
     },

ui/components.ts

@@ -22,7 +22,7 @@ import m, {
   Attributes,
   Children,
   ComponentTypes,
-  CommonAttributes,
+  Lifecycle,
   ClosureComponent,
   Vnode,
 } from "mithril";
@@ -39,7 +39,6 @@ import tags from "./components/tags";
 import ping from "./components/ping";
 import deviceLink from "./components/device-link";
 import longTextComponent from "./long-text-component";
-import loading from "./components/loading";
 
 const comps = {
   parameter,
@@ -55,7 +54,6 @@ const comps = {
   ping,
   "device-link": deviceLink,
   "long-text": longTextComponent,
-  loading: loading,
 };
 
 const contextifiedComponents = new WeakMap<ComponentTypes, ComponentTypes>();
@@ -81,7 +79,7 @@ interface MC extends Static {
     <Attrs, State>(
       ctx: Attributes,
       component: ComponentTypes<Attrs, State>,
-      attributes: Attrs & CommonAttributes<Attrs, State>,
+      attributes: Attrs & Lifecycle<Attrs, State> & { key?: string | number },
       ...args: Children[]
     ): Vnode<Attrs, State>;
   };
@@ -116,7 +114,7 @@ function applyContext(vnode, parentContext): void {
       vnodeContext.set(vnode, vc);
       vnode.attrs = Object.assign({}, vc, vnode.attrs);
     }
-    if (vnode.children?.length)
+    if (vnode.children && vnode.children.length)
       for (const c of vnode.children) applyContext(c, vc);
   }
 }
@@ -133,26 +131,15 @@ export function contextifyComponent(component: ComponentTypes): ComponentTypes {
         applyContext(res, context);
         return res;
       };
-    } else if (!component.prototype?.view) {
+    } else if (!component.prototype || !component.prototype.view) {
       c = (initialNode) => {
         const state = (component as ClosureComponent)(initialNode);
         const view = state.view;
         state.view = function (vnode) {
           const context = vnodeContext.get(vnode) || {};
-          try {
-            const res = Reflect.apply(view, this, [vnode]);
-            applyContext(res, context);
-            return res;
-          } catch (err) {
-            return m(
-              "p.error",
-              {
-                title: "Click to print stack trace to console",
-                onclick: () => console.error(err),
-              },
-              "Error!"
-            );
-          }
+          const res = Reflect.apply(view, this, [vnode]);
+          applyContext(res, context);
+          return res;
         };
         return state;
       };

ui/components/all-parameters.ts

@@ -23,7 +23,6 @@ import * as taskQueue from "../task-queue";
 import { parse } from "../../lib/common/expression-parser";
 import memoize from "../../lib/common/memoize";
 import { getIcon } from "../icons";
-import { evaluateExpression } from "../store";
 
 const memoizedParse = memoize(parse);
 
@@ -56,8 +55,6 @@ const component: ClosureComponent = (): Component => {
     view: (vnode) => {
       const device = vnode.attrs["device"];
 
-      const limit = evaluateExpression(vnode.attrs["limit"], device) || 100;
-
       const search = m("input", {
         type: "text",
         placeholder: "Search parameters",
@@ -86,10 +83,10 @@ const component: ClosureComponent = (): Component => {
         let c = 0;
         for (const k of keys) {
           const p = device[k];
-          const str = p.value?.[0] ? `${k} ${p.value[0]}` : k;
+          const str = p.value && p.value[0] ? `${k} ${p.value[0]}` : k;
           if (re && !re.test(str)) continue;
           ++c;
-          if (count < limit) filteredKeys.push(k);
+          if (count < 100) filteredKeys.push(k);
         }
         count += c;
       }
@@ -172,29 +169,25 @@ const component: ClosureComponent = (): Component => {
       });
 
       return m(
-        "loading",
-        { queries: [vnode.attrs["deviceQuery"]] },
+        ".all-parameters",
         m(
-          ".all-parameters",
+          "a.download-csv",
+          {
+            href: `api/devices/${encodeURIComponent(
+              device["DeviceID.ID"].value[0]
+            )}.csv`,
+            download: "",
+            style: "float: right;",
+          },
+          "Download"
+        ),
+        search,
+        m(
+          ".parameter-list",
+          m("table", m("tbody", rows)),
           m(
-            "a.download-csv",
-            {
-              href: `api/devices/${encodeURIComponent(
-                device["DeviceID.ID"].value[0]
-              )}.csv`,
-              download: "",
-              style: "float: right;",
-            },
-            "Download"
-          ),
-          search,
-          m(
-            ".parameter-list",
-            m("table", m("tbody", rows)),
-            m(
-              "m",
-              `Displaying ${filteredKeys.length} out of ${count} parameters.`
-            )
+            "m",
+            `Displaying ${filteredKeys.length} out of ${count} parameters.`
           )
         )
       );

ui/components/container.ts

@@ -17,76 +17,29 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import { Attributes, ClosureComponent, Component } from "mithril";
-import { map } from "../../lib/common/expression-parser";
-import memoize from "../../lib/common/memoize";
-import { Expression } from "../../lib/types";
+import { ClosureComponent, Component } from "mithril";
 import { m } from "../components";
-import { evaluateExpression, getTimestamp } from "../store";
-
-const evaluateAttributes = memoize(
-  (
-    attrs: Record<string, Expression>,
-    obj: Record<string, unknown>,
-    now: number // eslint-disable-line @typescript-eslint/no-unused-vars
-  ): Attributes => {
-    const res: Attributes = {};
-    for (const [k, v] of Object.entries(attrs)) {
-      const vv = map(v, (e) => {
-        if (
-          Array.isArray(e) &&
-          e[0] === "FUNC" &&
-          e[1] === "ENCODEURICOMPONENT"
-        ) {
-          const a = evaluateExpression(e[2], obj);
-          if (a == null) return null;
-          return encodeURIComponent(a as string);
-        }
-        return e;
-      });
-      res[k] = evaluateExpression(vv, obj);
-    }
-    return res;
-  }
-);
+import * as store from "../store";
 
 const component: ClosureComponent = (): Component => {
   return {
     view: (vnode) => {
-      const device = vnode.attrs["device"];
-      if ("filter" in vnode.attrs) {
-        if (!evaluateExpression(vnode.attrs["filter"], device || {}))
+      if (vnode.attrs["filter"]) {
+        if (
+          !store.evaluateExpression(
+            vnode.attrs["filter"],
+            vnode.attrs["device"]
+          )
+        )
           return null;
       }
 
       const children = Object.values(vnode.attrs["components"]).map((c) => {
-        if (Array.isArray(c)) c = evaluateExpression(c, device || {});
         if (typeof c !== "object") return `${c}`;
-        const type = evaluateExpression(c["type"], device || {});
-        if (!type) return null;
-        return m(type as string, c);
+        return m(c["type"], c);
       });
-
-      let el = vnode.attrs["element"];
-
-      if (el == null) return children;
-
-      let attrs: Attributes;
-      if (Array.isArray(el)) {
-        el = evaluateExpression(el, device || {});
-      } else if (typeof el === "object") {
-        if (el["attributes"] != null) {
-          attrs = evaluateAttributes(
-            el["attributes"],
-            device || {},
-            getTimestamp()
-          );
-        }
-
-        el = evaluateExpression(el["tag"], device || {});
-      }
-
-      return m(el, attrs, children);
+      if (vnode.attrs["element"]) return m(vnode.attrs["element"], children);
+      else return children;
     },
   };
 };

ui/components/device-faults.ts

@@ -22,7 +22,6 @@ import { m } from "../components";
 import * as store from "../store";
 import * as notifications from "../notifications";
 import { getIcon } from "../icons";
-import { stringify } from "../../lib/common/yaml";
 
 const component: ClosureComponent = (): Component => {
   return {
@@ -39,7 +38,6 @@ const component: ClosureComponent = (): Component => {
         "Channel",
         "Code",
         "Message",
-        "Detail",
         "Retries",
         "Timestamp",
       ].map((l) => m("th", l));
@@ -50,9 +48,7 @@ const component: ClosureComponent = (): Component => {
         rows.push([
           m("td", f["channel"]),
           m("td", f["code"]),
-          m("td", m("long-text", { text: f["message"] })),
-          m("td", m("long-text", { text: stringify(f["detail"]) })),
-
+          m("td", f["message"]),
           m("td", f["retries"]),
           m("td", new Date(f["timestamp"]).toLocaleString()),
           m(
@@ -67,12 +63,12 @@ const component: ClosureComponent = (): Component => {
                     .deleteResource("faults", f["_id"])
                     .then(() => {
                       notifications.push("success", "Fault deleted");
-                      store.setTimestamp(Date.now());
+                      store.fulfill(Date.now(), Date.now());
                       m.redraw();
                     })
                     .catch((err) => {
                       notifications.push("error", err.message);
-                      store.setTimestamp(Date.now());
+                      store.fulfill(Date.now(), Date.now());
                     });
                 },
               },
@@ -95,11 +91,7 @@ const component: ClosureComponent = (): Component => {
         );
       }
 
-      return m(
-        "loading",
-        { queries: [faults] },
-        m("table.table", thead, tbody)
-      );
+      return m("table.table", thead, tbody);
     },
   };
 };

ui/components/device-link.ts

@@ -19,22 +19,18 @@
 
 import { ClosureComponent, Component } from "mithril";
 import { m } from "../components";
-import { evaluateExpression } from "../store";
 
 const component: ClosureComponent = (): Component => {
   return {
     view: (vnode) => {
       let deviceId;
-      const device = vnode.attrs["device"];
-      if (device) deviceId = device["DeviceID.ID"].value[0];
+      if (vnode.attrs["device"])
+        deviceId = vnode.attrs["device"]["DeviceID.ID"].value[0];
 
       const children = Object.values(vnode.attrs["components"]).map((c) => {
-        if (Array.isArray(c)) c = evaluateExpression(c, device || {});
         if (typeof c !== "object") return `${c}`;
-        const type = evaluateExpression(c["type"], device || {});
-        if (!type) return null;
         const attrs = Object.assign({}, vnode.attrs, c);
-        return m(type as string, attrs);
+        return m(attrs["type"], attrs);
       });
       if (deviceId) {
         return m(

ui/components/loading.ts

@@ -1,80 +0,0 @@
-/**
- * Copyright 2013-2021  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { VnodeDOM, ClosureComponent, Component } from "mithril";
-
-const component: ClosureComponent = (): Component => {
-  let overlay: HTMLElement;
-  let dom: Element;
-  let loading = false;
-
-  function apply(vnode: VnodeDOM): void {
-    if (!loading) {
-      if (overlay) overlay.parentElement.remove();
-      if (dom) dom.classList.remove("loading");
-      overlay = null;
-      dom = null;
-      return;
-    }
-
-    if (dom && dom !== vnode.dom) dom.classList.remove("loading");
-
-    dom = vnode.dom;
-    dom.classList.add("loading");
-
-    if (!overlay) {
-      const wrapper = document.createElement("div");
-      wrapper.style.position = "relative";
-      wrapper.style.pointerEvents = "none";
-      overlay = document.createElement("div");
-      overlay.classList.add("loading-overlay");
-      overlay.style.position = "absolute";
-      wrapper.appendChild(overlay);
-    }
-
-    const wrapper = overlay.parentElement;
-    if (wrapper.parentElement !== dom.parentElement)
-      dom.parentNode.appendChild(wrapper);
-
-    const wrapperRect = wrapper.getBoundingClientRect();
-    const domRect = dom.getBoundingClientRect();
-    overlay.style.width = `${dom.scrollWidth}px`;
-    overlay.style.height = `${dom.scrollHeight}px`;
-    overlay.style.left = `${domRect.left - wrapperRect.left}px`;
-    overlay.style.top = `${domRect.top - wrapperRect.top}px`;
-  }
-
-  return {
-    view: (vnode) => {
-      const queries = vnode.attrs["queries"];
-      loading = queries.some((q) => q.fulfilling);
-      return vnode.children;
-    },
-    oncreate: apply,
-    onupdate: apply,
-    onremove: () => {
-      if (overlay) overlay.parentElement.remove();
-      if (dom) dom.classList.remove("loading");
-      overlay = null;
-      dom = null;
-    },
-  };
-};
-
-export default component;

ui/components/overview-dot.ts

@@ -20,7 +20,7 @@
 import { ClosureComponent, Component } from "mithril";
 import { m } from "../components";
 import config from "../config";
-import { evaluateExpression } from "../store";
+import * as store from "../store";
 
 const CHARTS = config.ui.overview.charts;
 
@@ -28,12 +28,11 @@ const component: ClosureComponent = (): Component => {
   return {
     view: (vnode) => {
       const device = vnode.attrs["device"];
-      const chartName = evaluateExpression(vnode.attrs["chart"], device || {});
-      const chart = CHARTS[chartName as string] as Record<string, unknown>;
+      const chart = CHARTS[vnode.attrs["chart"]] as Record<string, unknown>;
       if (!chart) return null;
       for (const slice of Object.values(chart.slices)) {
         const filter = slice["filter"];
-        if (evaluateExpression(filter, device || {})) {
+        if (store.evaluateExpression(filter, device)) {
           const dot = m(
             "svg",
             {

ui/components/parameter-list.ts

@@ -19,7 +19,6 @@
 
 import { ClosureComponent, Component, VnodeDOM } from "mithril";
 import { m } from "../components";
-import { evaluateExpression } from "../store";
 
 const component: ClosureComponent = (): Component => {
   return {
@@ -27,24 +26,18 @@ const component: ClosureComponent = (): Component => {
       const device = vnode.attrs["device"];
 
       const rows = Object.values(vnode.attrs["parameters"]).map((parameter) => {
-        const type = evaluateExpression(parameter["type"], device);
         const p = m.context(
           {
             device: device,
             parameter: parameter["parameter"],
           },
-          (type as string) || "parameter",
+          parameter["type"] || "parameter",
           parameter
         );
 
         return m(
           "tr",
           {
-            oncreate: (vn) => {
-              (vn.dom as HTMLElement).style.display = (p as VnodeDOM).dom
-                ? ""
-                : "none";
-            },
             onupdate: (vn) => {
               (vn.dom as HTMLElement).style.display = (p as VnodeDOM).dom
                 ? ""
@@ -56,11 +49,7 @@ const component: ClosureComponent = (): Component => {
         );
       });
 
-      return m(
-        "loading",
-        { queries: [vnode.attrs["deviceQuery"]] },
-        m("table.parameter-list", rows)
-      );
+      return m("table.parameter-list", rows);
     },
   };
 };

ui/components/parameter-table.ts

@@ -20,7 +20,7 @@
 import { ClosureComponent, Component } from "mithril";
 import { m } from "../components";
 import * as taskQueue from "../task-queue";
-import { evaluateExpression } from "../store";
+import * as store from "../store";
 import * as expressionParser from "../../lib/common/expression-parser";
 import { getIcon } from "../icons";
 
@@ -35,7 +35,7 @@ const component: ClosureComponent = (): Component => {
     },
     view: (vnode) => {
       const device = vnode.attrs["device"];
-      const object = evaluateExpression(vnode.state["object"], device);
+      const object = store.evaluateExpression(vnode.state["object"], device);
       const parameters = vnode.state["parameters"];
 
       if (typeof object !== "string" || !device[object]) return null;
@@ -56,7 +56,8 @@ const component: ClosureComponent = (): Component => {
 
       const rows = [];
       for (const i of instances) {
-        let filter = "filter" in vnode.attrs ? vnode.attrs["filter"] : true;
+        let filter =
+          vnode.attrs["filter"] != null ? vnode.attrs["filter"] : true;
 
         filter = expressionParser.map(filter, (e) => {
           if (Array.isArray(e) && e[0] === "PARAM")
@@ -64,7 +65,7 @@ const component: ClosureComponent = (): Component => {
           return e;
         });
 
-        if (!evaluateExpression(filter, device)) continue;
+        if (!store.evaluateExpression(filter, device)) continue;
 
         const row = parameters.map((p) => {
           const param = expressionParser.map(p.parameter, (e) => {
@@ -146,18 +147,9 @@ const component: ClosureComponent = (): Component => {
       }
 
       let label;
+      if (vnode.attrs["label"]) label = m("h2", vnode.attrs["label"]);
 
-      const l = evaluateExpression(vnode.attrs["label"], device);
-      if (l != null) label = m("h2", l);
-
-      return [
-        label,
-        m(
-          "loading",
-          { queries: [vnode.attrs["deviceQuery"]] },
-          m("table.table", thead, m("tbody", rows))
-        ),
-      ];
+      return [label, m("table.table", thead, m("tbody", rows))];
     },
   };
 };

ui/components/parameter.ts

@@ -38,7 +38,7 @@ const evaluateParam = memoize((exp, obj, now: number) => {
       ) {
         let v = null;
         const p = obj[e[i][1]];
-        if (p?.value) {
+        if (p && p.value) {
           v = p.value[0];
           timestamp = Math.min(timestamp, p.valueTimestamp);
         }
@@ -58,12 +58,10 @@ const evaluateParam = memoize((exp, obj, now: number) => {
     value = exp;
   } else if (exp[0] === "PARAM") {
     const p = obj[exp[1]];
-    if (p?.value) {
+    if (p && p.value) {
       timestamp = p.valueTimestamp;
       value = p.value[0];
       parameter = exp[1];
-      if (p.value[1] === "xsd:dateTime" && typeof value === "number")
-        value = new Date(value).toLocaleString();
     }
   }
 
@@ -77,14 +75,14 @@ const component: ClosureComponent = (): Component => {
 
       const { value, timestamp, parameter } = evaluateParam(
         vnode.attrs["parameter"],
-        device,
-        store.getTimestamp() + store.getClockSkew()
+        vnode.attrs["device"],
+        store.getTimestamp()
       );
 
       if (value == null) return null;
 
       let edit;
-      if (device[parameter]?.writable) {
+      if (device[parameter] && device[parameter].writable) {
         edit = m(
           "button",
           {
@@ -117,7 +115,7 @@ const component: ClosureComponent = (): Component => {
             e.redraw = false;
             // Don't update any child element
             if (e.target === (el as VnodeDOM).dom) {
-              const now = Date.now() + store.getClockSkew();
+              const now = Date.now();
               const localeString = new Date(timestamp).toLocaleString();
               e.target.title = `${localeString} (${timeAgo(now - timestamp)})`;
             }

ui/components/ping.ts

@@ -63,11 +63,9 @@ const component: ClosureComponent = (vn): Component => {
         param = device["Device.ManagementServer.ConnectionRequestURL"];
 
       let h;
-      try {
+      if (param && param.value) {
         const url = new URL(param.value[0]);
         h = url.hostname;
-      } catch (err) {
-        // Ignore
       }
 
       if (host !== h) {
@@ -79,7 +77,7 @@ const component: ClosureComponent = (vn): Component => {
         }
       }
 
-      return m("div", host ? `Pinging ${host}:` : "");
+      return m("div", `Pinging ${host}:`);
     },
   };
 };

ui/components/summon-button.ts

@@ -78,7 +78,7 @@ const component: ClosureComponent = (): Component => {
               )
               .then(() => {
                 e.target.disabled = false;
-                store.setTimestamp(Date.now());
+                store.fulfill(0, Date.now());
               })
               .catch((err) => {
                 e.target.disabled = false;

ui/components/tags.ts

@@ -29,8 +29,7 @@ const component: ClosureComponent = (): Component => {
     view: (vnode) => {
       const device = vnode.attrs["device"];
       let writable = true;
-      if ("writable" in vnode.attrs)
-        writable = !!store.evaluateExpression(vnode.attrs["writable"], device);
+      if (vnode.attrs["writable"] != null) writable = vnode.attrs["writable"];
 
       const tags = [];
       for (const p of Object.keys(device))
@@ -65,7 +64,7 @@ const component: ClosureComponent = (): Component => {
                         "success",
                         `${deviceId}: Tags updated`
                       );
-                      store.setTimestamp(Date.now());
+                      store.fulfill(0, Date.now());
                     })
                     .catch((err) => {
                       e.target.disabled = false;
@@ -100,7 +99,7 @@ const component: ClosureComponent = (): Component => {
                   .then(() => {
                     e.target.disabled = false;
                     notifications.push("success", `${deviceId}: Tags updated`);
-                    store.setTimestamp(Date.now());
+                    store.fulfill(0, Date.now());
                   })
                   .catch((err) => {
                     e.target.disabled = false;

ui/config-functions.ts

@@ -45,7 +45,7 @@ export function flattenConfig(config: Record<string, unknown>): any {
 // Order keys such that nested objects come last
 function orderKeys(config: any): number {
   let res = 1;
-  if (config == null || typeof config !== "object") return res;
+  if (typeof config !== "object") return res;
   if (Array.isArray(config)) {
     for (const c of config) res += orderKeys(c);
     return res;
@@ -79,7 +79,7 @@ export function structureConfig(config: Config[]): any {
     let ref = _config;
     while (keys.length > 1) {
       const k = keys.shift();
-      if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
+      if (typeof ref[k] !== "object") ref[k] = {};
       ref = ref[k];
     }
     ref[keys[0]] = c.value;
@@ -137,7 +137,7 @@ export function diffConfig(
   };
 
   for (const [k, v] of Object.entries(target))
-    if (v && current[k] !== v) diff.add.push({ _id: k, value: v });
+    if (!current[k] || current[k] !== v) diff.add.push({ _id: k, value: v });
 
   for (const k of Object.keys(current)) if (!target[k]) diff.remove.push(k);
 

ui/config-page.ts

@@ -60,12 +60,12 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
         .resourceExists("config", id)
         .then((exists) => {
           if (exists && isNew) {
-            store.setTimestamp(Date.now());
+            store.fulfill(0, Date.now());
             return void resolve({ _id: "Config already exists" });
           }
 
           if (!exists && !isNew) {
-            store.setTimestamp(Date.now());
+            store.fulfill(0, Date.now());
             return void resolve({ _id: "Config does not exist" });
           }
 
@@ -76,8 +76,8 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
                 "success",
                 `Config ${exists ? "updated" : "created"}`
               );
-              store.setTimestamp(Date.now());
-              resolve(null);
+              store.fulfill(0, Date.now());
+              resolve();
             })
             .catch(reject);
         })
@@ -87,11 +87,11 @@ function putActionHandler(action, _object, isNew?): Promise<ValidationErrors> {
         .deleteResource("config", object["_id"])
         .then(() => {
           notifications.push("success", "Config deleted");
-          store.setTimestamp(Date.now());
-          resolve(null);
+          store.fulfill(0, Date.now());
+          resolve();
         })
         .catch((err) => {
-          store.setTimestamp(Date.now());
+          store.fulfill(0, Date.now());
           reject(err);
         });
     } else {
@@ -154,7 +154,7 @@ function renderTable(confsResponse, searchString): Children {
               {
                 base: conf,
                 actionHandler: (action, object) => {
-                  return new Promise<void>((resolve) => {
+                  return new Promise((resolve) => {
                     putActionHandler(action, object, false)
                       .then((errors) => {
                         const ErrorList = errors ? Object.values(errors) : [];
@@ -255,7 +255,7 @@ export const component: ClosureComponent = (): Component => {
                 Object.assign(
                   {
                     actionHandler: (action, object) => {
-                      return new Promise<void>((resolve) => {
+                      return new Promise((resolve) => {
                         putActionHandler(action, object, true)
                           .then((errors) => {
                             const errorList = errors
@@ -267,7 +267,7 @@ export const component: ClosureComponent = (): Component => {
                             } else {
                               overlay.close(cb);
                             }
-                            resolve(null);
+                            resolve();
                           })
                           .catch((err) => {
                             notifications.push("error", err.message);
@@ -322,7 +322,7 @@ export const component: ClosureComponent = (): Component => {
                     uiConfigComponent,
                     Object.assign(
                       {
-                        onUpdate: (errs: Record<string, string>) => {
+                        onUpdate: (errs: string[]) => {
                           const errors = errs ? Object.values(errs) : [];
                           if (errors.length) {
                             for (const err of errors)
@@ -337,11 +337,11 @@ export const component: ClosureComponent = (): Component => {
                             );
                             overlay.close(cb);
                           }
-                          store.setTimestamp(Date.now());
+                          store.fulfill(0, Date.now());
                         },
                         onError: (err) => {
                           notifications.push("error", err.message);
-                          store.setTimestamp(Date.now());
+                          store.fulfill(0, Date.now());
                           overlay.close(cb);
                         },
                       },
@@ -366,18 +366,14 @@ export const component: ClosureComponent = (): Component => {
       return [
         m("h1", "Listing config"),
         m(
-          "loading",
-          { queries: [confs] },
+          ".all-parameters",
+          search,
           m(
-            ".all-parameters",
-            search,
-            m(
-              ".parameter-list",
-              { style: "height: 400px" },
-              renderTable(confs, vnode.state["searchString"])
-            ),
-            m(".actions-bar", [newConfig].concat(subs))
-          )
+            ".parameter-list",
+            { style: "height: 400px" },
+            renderTable(confs, vnode.state["searchString"])
+          ),
+          m(".actions-bar", [newConfig].concat(subs))
         ),
       ];
     },

ui/css/app.css

@@ -46,7 +46,6 @@ button,
 select,
 option,
 textarea {
-  outline-color: var(--color4);
   font-family: sans-serif;
   color: var(--color5);
 }
@@ -286,21 +285,9 @@ body {
   }
 
   & > .logo {
-    display: inline-block;
-    position: relative;
-    & > img {
-      margin: 10px;
-      height: 56px;
-    }
-
-    & > .version {
-      position: absolute;
-      bottom: 5px;
-      right: 10px;
-      font-family: monospace;
-      font-size: 10px;
-      color: #666;
-    }
+    margin: 10px;
+    display: inline;
+    height: 56px;
   }
 
   & > .user-menu {
@@ -342,18 +329,12 @@ body {
   flex-grow: 1;
   margin: 10px;
   min-height: 250px;
-
-  /* Workaround for Chrome ignoring bottom margin when flex container overflows */
-  &::after {
-    content: "";
-    display: block;
-    padding-bottom: 1px;
-  }
 }
 
 table.table {
   border-spacing: 0;
   text-align: left;
+  display: block;
   overflow-x: auto;
 
   & table.table {
@@ -759,41 +740,3 @@ span.parameter-value {
     opacity: 0.3;
   }
 }
-
-.loading {
-  opacity: 0.6;
-}
-
-@keyframes spin {
-  0% {
-    transform: rotate(0deg);
-  }
-  100% {
-    transform: rotate(360deg);
-  }
-}
-
-.loading-overlay {
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  &::after {
-    background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxMDAgMTAwIiBwcmVzZXJ2ZUFzcGVjdFJhdGlvPSJ4TWlkWU1pZCI+PHBhdGggZD0iTTEwIDUwYTQwIDQwIDAgMDA4MCAwIDQwIDQyIDAgMDEtODAgMCIgZmlsbD0iI2IxMmQ1YyIgLz48L3N2Zz4=);
-    content: "";
-    width: 48px;
-    height: 48px;
-    animation: spin 1s linear infinite;
-    background-repeat: no-repeat;
-    background-size: 100% 100%;
-  }
-}
-
-.progress {
-  background-color: #eee;
-  overflow: hidden;
-  height: 3px;
-  & > .progress-bar {
-    height: 100%;
-    background-color: #b12d5c;
-  }
-}

ui/datalist.ts

@@ -1,62 +0,0 @@
-/**
- * Copyright 2013-2021  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import m, { ClosureComponent, Component, Vnode } from "mithril";
-
-const elements: Map<string, Vnode> = new Map();
-
-// Source: https://werxltd.com/wp/2010/05/13/javascript-implementation-of-javas-string-hashcode-method/
-function hash(str: string): number {
-  let res = 0;
-  for (let i = 0; i < str.length; ++i) {
-    const c = str.charCodeAt(i);
-    res = (res << 5) - res + c;
-    res |= 0;
-  }
-  return res;
-}
-
-export function getDatalistId(options: string[]): string {
-  const id = "datalist" + options.reduce((acc, cur) => acc ^ hash(cur), 0);
-  if (!elements.has(id)) {
-    const n = m(
-      "datalist",
-      { id },
-      options.map((o) => m("option", { value: o }))
-    );
-    elements.set(id, n);
-  }
-  return id;
-}
-
-const component: ClosureComponent = (): Component => {
-  return {
-    view: () => {
-      return [...elements.values()];
-    },
-    onupdate: () => {
-      for (const id of elements.keys()) {
-        const used = document.querySelector(`[list='${id}']`);
-        if (!used) elements.delete(id);
-      }
-    },
-  };
-};
-
-export default component;

ui/device-page.ts

@@ -42,25 +42,13 @@ export const component: ClosureComponent = (): Component => {
     view: (vnode) => {
       document.title = `${vnode.attrs["deviceId"]} - Devices - GenieACS`;
 
-      const dev = store.fetch("devices", vnode.attrs["deviceFilter"]);
-      if (!dev.value.length) {
-        if (!dev.fulfilling)
-          return m("p.error", `No such device ${vnode.attrs["deviceId"]}`);
-        return m(
-          "loading",
-          { queries: [dev] },
-          m("div", { style: "height: 100px;" })
-        );
-      }
-
+      const dev = store.fetch("devices", vnode.attrs["deviceFilter"]).value;
+      if (!dev.length) return "Loading";
       const conf = config.ui.device;
       const cmps = [];
 
-      for (const c of Object.values(conf)) {
-        cmps.push(
-          m.context({ device: dev.value[0], deviceQuery: dev }, c["type"], c)
-        );
-      }
+      for (const c of Object.values(conf))
+        cmps.push(m.context({ device: dev[0] }, c["type"], c));
 
       return [m("h1", vnode.attrs["deviceId"]), cmps];
     },

ui/devices-page.ts

@@ -154,14 +154,14 @@ function renderActions(selected: Set<string>): Children {
               .deleteResource("devices", id)
               .then(() => {
                 notifications.push("success", `${id}: Deleted`);
-                if (--counter === 0) store.setTimestamp(Date.now());
+                if (--counter === 0) store.fulfill(0, Date.now());
               })
               .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
-                if (--counter === 0) store.setTimestamp(Date.now());
+                if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
-          if (--counter === 0) store.setTimestamp(Date.now());
+          if (--counter === 0) store.fulfill(0, Date.now());
         },
       },
       "Delete"
@@ -186,14 +186,14 @@ function renderActions(selected: Set<string>): Children {
               .updateTags(id, { [tag]: true })
               .then(() => {
                 notifications.push("success", `${id}: Tags updated`);
-                if (--counter === 0) store.setTimestamp(Date.now());
+                if (--counter === 0) store.fulfill(0, Date.now());
               })
               .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
-                if (--counter === 0) store.setTimestamp(Date.now());
+                if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
-          if (--counter === 0) store.setTimestamp(Date.now());
+          if (--counter === 0) store.fulfill(0, Date.now());
         },
       },
       "Tag"
@@ -220,14 +220,14 @@ function renderActions(selected: Set<string>): Children {
               .updateTags(id, { [tag]: false })
               .then(() => {
                 notifications.push("success", `${id}: Tags updated`);
-                if (--counter === 0) store.setTimestamp(Date.now());
+                if (--counter === 0) store.fulfill(0, Date.now());
               })
               .catch((err) => {
                 notifications.push("error", `${id}: ${err.message}`);
-                if (--counter === 0) store.setTimestamp(Date.now());
+                if (--counter === 0) store.fulfill(0, Date.now());
               });
           }
-          if (--counter === 0) store.setTimestamp(Date.now());
+          if (--counter === 0) store.fulfill(0, Date.now());
         },
       },
       "Untag"
@@ -268,18 +268,17 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortedAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = Object.assign({}, sort);
         for (const [index, direction] of Object.entries(sortedAttrs)) {
           const param = memoizedGetSortable(attributes[index].parameter);
           if (param) {
             // Changing the priority of columns
             delete _sort[param];
-            if (direction) _sort = Object.assign({ [param]: direction }, _sort);
+            _sort[param] = direction;
           }
         }
 
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
+        const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/devices", ops);
       }
@@ -329,16 +328,15 @@ export const component: ClosureComponent = (): Component => {
       if (window.authorizer.hasAccess("devices", 3))
         attrs["actionsCallback"] = renderActions;
 
-      const filterAttrs = {
-        resource: "devices",
-        filter: vnode.attrs["filter"],
-        onChange: onFilterChanged,
-      };
+      const filterAttrs = {};
+      filterAttrs["resource"] = "devices";
+      filterAttrs["filter"] = vnode.attrs["filter"];
+      filterAttrs["onChange"] = onFilterChanged;
 
       return [
         m("h1", "Listing devices"),
         m(filterComponent, filterAttrs),
-        m("loading", { queries: [devs, count] }, m(indexTableComponent, attrs)),
+        m(indexTableComponent, attrs),
       ];
     },
   };

ui/drawer-component.ts

@@ -76,13 +76,13 @@ function renderStagingSpv(task: StageTask, queueFunc, cancelFunc): Children {
       ]
     );
   } else {
-    const type = task.parameterValues[0][2];
-    let value = task.parameterValues[0][1];
-    if (type === "xsd:dateTime" && typeof value === "number")
-      value = new Date(value).toJSON() || value;
     input = m("input", {
-      type: ["xsd:int", "xsd:unsignedInt"].includes(type) ? "number" : "text",
-      value: value,
+      type: ["xsd:dateTime", "xsd:int", "xsd:unsignedInt"].includes(
+        task.parameterValues[0][2]
+      )
+        ? "number"
+        : "text",
+      value: task.parameterValues[0][1],
       oninput: (e) => {
         e.redraw = false;
         task.parameterValues[0][1] = input.dom.value;
@@ -121,15 +121,12 @@ function renderStagingDownload(task: StageTask): Children {
   if (productClass) productClass = decodeURIComponent(productClass);
 
   const typesList = [
-    ...new Set([
-      "",
-      "1 Firmware Upgrade Image",
-      "2 Web Content",
-      "3 Vendor Configuration File",
-      "4 Tone File",
-      "5 Ringer File",
-      ...files.value.map((f) => f["metadata.fileType"]).filter((f) => f),
-    ]),
+    "",
+    "1 Firmware Upgrade Image",
+    "2 Web Content",
+    "3 Vendor Configuration File",
+    "4 Tone File",
+    "5 Ringer File",
   ].map((t) =>
     m(
       "option",
@@ -358,7 +355,7 @@ function renderNotifications(notifs): Child[] {
           },
           onbeforeremove: (vnode) => {
             (vnode.dom as HTMLDivElement).style.opacity = "0";
-            return new Promise<void>((resolve) => {
+            return new Promise((resolve) => {
               setTimeout(() => {
                 resolve();
               }, 500);
@@ -468,7 +465,7 @@ const component: ClosureComponent = (): Component => {
                   }
                 )
                   .then(() => {
-                    store.setTimestamp(Date.now());
+                    store.fulfill(0, Date.now());
                   })
                   .catch((err) => {
                     notifications.push("error", err.message);

ui/dynamic-loader.ts

@@ -19,8 +19,8 @@
 
 import * as notifications from "./notifications";
 
-export let codeMirror: typeof import("codemirror");
-export let yaml: typeof import("yaml");
+export let codeMirror;
+export let yaml;
 
 let note;
 
@@ -43,9 +43,11 @@ export function loadCodeMirror(): Promise<void> {
 
   return new Promise((resolve, reject) => {
     const promises = [
-      import("codemirror"),
-      import("codemirror/mode/javascript/javascript"),
-      import("codemirror/mode/yaml/yaml"),
+      import(/* webpackChunkName: "codemirror" */ "codemirror"),
+      import(
+        /* webpackChunkName: "codemirror" */ "codemirror/mode/javascript/javascript"
+      ),
+      import(/* webpackChunkName: "codemirror" */ "codemirror/mode/yaml/yaml"),
     ];
     Promise.all(promises)
       .then((modules) => {
@@ -63,7 +65,7 @@ export function loadYaml(): Promise<void> {
   if (yaml) return Promise.resolve();
 
   return new Promise((resolve, reject) => {
-    import("yaml")
+    import(/* webpackChunkName: "yaml" */ "yaml")
       .then((module) => {
         yaml = module;
         resolve();

ui/faults-page.ts

@@ -27,7 +27,7 @@ import * as notifications from "./notifications";
 import memoize from "../lib/common/memoize";
 import * as smartQuery from "./smart-query";
 import { map, parse, stringify } from "../lib/common/expression-parser";
-import { stringify as yamlStringify } from "../lib/common/yaml";
+import { loadYaml, yaml } from "./dynamic-loader";
 
 const PAGE_SIZE = config.ui.pageSize || 10;
 
@@ -76,7 +76,13 @@ export function init(
 
   const sort = args.hasOwnProperty("sort") ? "" + args["sort"] : "";
   const filter = args.hasOwnProperty("filter") ? "" + args["filter"] : "";
-  return Promise.resolve({ filter, sort });
+  return new Promise((resolve, reject) => {
+    loadYaml()
+      .then(() => {
+        resolve({ filter, sort });
+      })
+      .catch(reject);
+  });
 }
 
 export const component: ClosureComponent = (): Component => {
@@ -107,16 +113,14 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = Object.assign({}, sort);
         for (const [index, direction] of Object.entries(sortAttrs)) {
           // Changing the priority of columns
           delete _sort[attributes[index].id];
-          if (direction)
-            _sort = Object.assign({ [attributes[index].id]: direction }, _sort);
+          _sort[attributes[index].id] = direction;
         }
 
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
+        const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/faults", ops);
       }
@@ -143,11 +147,8 @@ export const component: ClosureComponent = (): Component => {
           return m("a", { href: deviceHref }, fault["device"]);
         }
 
-        if (attr.id === "message")
-          return m("long-text", { text: fault["message"] });
-
         if (attr.id === "detail")
-          return m("long-text", { text: yamlStringify(fault["detail"]) });
+          return m("long-text", { text: yaml.stringify(fault["detail"]) });
 
         if (attr.id === "timestamp")
           return new Date(fault["timestamp"]).toLocaleString();
@@ -189,11 +190,11 @@ export const component: ClosureComponent = (): Component => {
                       "success",
                       `${res.length} faults deleted`
                     );
-                    store.setTimestamp(Date.now());
+                    store.fulfill(0, Date.now());
                   })
                   .catch((err) => {
                     notifications.push("error", err.message);
-                    store.setTimestamp(Date.now());
+                    store.fulfill(0, Date.now());
                   });
               },
             },
@@ -202,20 +203,15 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {
-        resource: "faults",
-        filter: vnode.attrs["filter"],
-        onChange: onFilterChanged,
-      };
+      const filterAttrs = {};
+      filterAttrs["resource"] = "faults";
+      filterAttrs["filter"] = vnode.attrs["filter"];
+      filterAttrs["onChange"] = onFilterChanged;
 
       return [
         m("h1", "Listing faults"),
         m(filterComponent, filterAttrs),
-        m(
-          "loading",
-          { queries: [faults, count] },
-          m(indexTableComponent, attrs)
-        ),
+        m(indexTableComponent, attrs),
       ];
     },
   };

ui/files-page.ts

@@ -45,6 +45,7 @@ const attributes: {
   {
     id: "metadata.fileType",
     label: "Type",
+    type: "combo",
     options: [
       "1 Firmware Upgrade Image",
       "2 Web Content",
@@ -73,32 +74,58 @@ const unpackSmartQuery = memoize((query) => {
   });
 });
 
-function upload(
-  file: File,
-  headers: Record<string, string>,
-  abortSignal?: AbortSignal,
-  progressListener?: (e: ProgressEvent) => void
-): Promise<void> {
-  headers = Object.assign(
-    {
-      "Content-Type": "application/octet-stream",
-    },
-    headers
-  );
-  return store.xhrRequest({
-    method: "PUT",
-    headers: headers,
-    url: `api/files/${encodeURIComponent(file.name)}`,
-    serialize: (body) => body, // Identity function to prevent JSON.parse on blob data
-    body: file,
-    config: (xhr) => {
-      if (progressListener)
-        xhr.upload.addEventListener("progress", progressListener);
-      if (abortSignal) {
-        if (abortSignal.aborted) xhr.abort();
-        abortSignal.addEventListener("abort", () => xhr.abort());
-      }
-    },
+interface ValidationErrors {
+  [prop: string]: string;
+}
+
+function putActionHandler(action, _object): Promise<ValidationErrors> {
+  return new Promise((resolve, reject) => {
+    const object = Object.assign({}, _object);
+    if (action === "save") {
+      const file = object["file"] ? object["file"][0] : null;
+      delete object["file"];
+
+      if (!file) return void resolve({ file: "File not selected" });
+
+      const id = file.name;
+
+      store
+        .resourceExists("files", id)
+        .then((exists) => {
+          if (exists) {
+            store.fulfill(0, Date.now());
+            return void resolve({ file: "File already exists" });
+          }
+          const headers = Object.assign(
+            {
+              "Content-Type": "application/octet-stream",
+              Accept: "application/octet-stream",
+            },
+            object
+          );
+
+          store
+            .xhrRequest({
+              method: "PUT",
+              headers: headers,
+              url: `api/files/${encodeURIComponent(id)}`,
+              serialize: (body) => body, // Identity function to prevent JSON.parse on blob data
+              body: file,
+            })
+            .then(() => {
+              notifications.push(
+                "success",
+                `File ${exists ? "updated" : "created"}`
+              );
+              store.fulfill(0, Date.now());
+              resolve();
+            })
+            .catch(reject);
+        })
+        .catch(reject);
+    } else {
+      reject(new Error("Undefined action"));
+    }
   });
 }
 
@@ -151,16 +178,14 @@ export const component: ClosureComponent = (): Component => {
         sortAttributes[i] = sort[attributes[i].id] || 0;
 
       function onSortChange(sortAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = Object.assign({}, sort);
         for (const [index, direction] of Object.entries(sortAttrs)) {
           // Changing the priority of columns
           delete _sort[attributes[index].id];
-          if (direction)
-            _sort = Object.assign({ [attributes[index].id]: direction }, _sort);
+          _sort[attributes[index].id] = direction;
         }
 
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
+        const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/files", ops);
       }
@@ -187,9 +212,6 @@ export const component: ClosureComponent = (): Component => {
       attrs["sortAttributes"] = sortAttributes;
       attrs["onSortChange"] = onSortChange;
       attrs["downloadUrl"] = downloadUrl;
-      attrs["recordActionsCallback"] = (file) => {
-        return [m("a", { href: "api/blob/files/" + file["_id"] }, "Download")];
-      };
 
       if (window.authorizer.hasAccess("files", 3)) {
         attrs["actionsCallback"] = (selected: Set<string>): Children => {
@@ -200,83 +222,42 @@ export const component: ClosureComponent = (): Component => {
                 title: "Create new file",
                 onclick: () => {
                   let cb: () => Children = null;
-                  const abortController = new AbortController();
-                  let progress = -1;
                   const comp = m(
                     putFormComponent,
                     Object.assign(
                       {
-                        actionHandler: async (action, obj) => {
-                          if (action !== "save")
-                            throw new Error("Undefined action");
-                          const file = obj["file"]?.[0];
-
-                          // nginx strips out headers with dot, so replace with dash
-                          const headers = {
-                            "metadata-fileType": obj["metadata.fileType"] || "",
-                            "metadata-oui": obj["metadata.oui"] || "",
-                            "metadata-productclass":
-                              obj["metadata.productClass"] || "",
-                            "metadata-version": obj["metadata.version"] || "",
-                          };
-
-                          if (!file) {
-                            notifications.push("error", "File not selected");
-                            return;
-                          }
-
-                          if (await store.resourceExists("files", file.name)) {
-                            store.setTimestamp(Date.now());
-                            notifications.push("error", "File already exists");
-                            return;
-                          }
-
-                          const progressListener = (e: ProgressEvent): void => {
-                            progress = e.loaded / e.total;
-                            m.redraw();
-                          };
-
-                          progress = 0;
-                          try {
-                            await upload(
-                              file,
-                              headers,
-                              abortController.signal,
-                              progressListener
-                            );
-                            store.setTimestamp(Date.now());
-                            notifications.push("success", "File created");
-                            overlay.close(cb);
-                          } catch (err) {
-                            notifications.push("error", err.message);
-                          }
-                          progress = -1;
+                        actionHandler: (action, object) => {
+                          return new Promise((resolve) => {
+                            putActionHandler(action, object)
+                              .then((errors) => {
+                                const errorList = errors
+                                  ? Object.values(errors)
+                                  : [];
+                                if (errorList.length) {
+                                  for (const err of errorList)
+                                    notifications.push("error", err);
+                                } else {
+                                  overlay.close(cb);
+                                }
+                                resolve();
+                              })
+                              .catch((err) => {
+                                notifications.push("error", err.message);
+                                resolve();
+                              });
+                          });
                         },
                       },
                       formData
                     )
                   );
-                  cb = () => {
-                    if (progress < 0) return [null, comp];
-                    return [
-                      m(
-                        "div.progress",
-                        m("div.progress-bar", {
-                          style: `width: ${Math.trunc(progress * 100)}%`,
-                        })
-                      ),
-                      comp,
-                    ];
-                  };
-                  overlay.open(cb, () => {
-                    if (
-                      comp.state["current"]["modified"] &&
-                      !confirm("You have unsaved changes. Close anyway?")
-                    )
-                      return false;
-                    abortController.abort();
-                    return true;
-                  });
+                  cb = () => comp;
+                  overlay.open(
+                    cb,
+                    () =>
+                      !comp.state["current"]["modified"] ||
+                      confirm("You have unsaved changes. Close anyway?")
+                  );
                 },
               },
               "New"
@@ -304,11 +285,11 @@ export const component: ClosureComponent = (): Component => {
                         "success",
                         `${res.length} files deleted`
                       );
-                      store.setTimestamp(Date.now());
+                      store.fulfill(0, Date.now());
                     })
                     .catch((err) => {
                       notifications.push("error", err.message);
-                      store.setTimestamp(Date.now());
+                      store.fulfill(0, Date.now());
                     });
                 },
               },
@@ -318,20 +299,15 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {
-        resource: "files",
-        filter: vnode.attrs["filter"],
-        onChange: onFilterChanged,
-      };
+      const filterAttrs = {};
+      filterAttrs["resource"] = "files";
+      filterAttrs["filter"] = vnode.attrs["filter"];
+      filterAttrs["onChange"] = onFilterChanged;
 
       return [
         m("h1", "Listing files"),
         m(filterComponent, filterAttrs),
-        m(
-          "loading",
-          { queries: [files, count] },
-          m(indexTableComponent, attrs)
-        ),
+        m(indexTableComponent, attrs),
       ];
     },
   };

ui/filter-component.ts

@@ -17,7 +17,7 @@
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import m, { ClosureComponent } from "mithril";
+import m, { ClosureComponent, Component } from "mithril";
 import { parse, stringify, map } from "../lib/common/expression-parser";
 import memoize from "../lib/common/memoize";
 import Autocomplete from "./autocomplete-compnent";
@@ -82,9 +82,9 @@ function stringifyFilter(f: Expression): string {
   return stringify(f);
 }
 
-function splitFilter(filter: string): string[] {
+const splitFilter = memoize((filter) => {
   if (!filter) return [""];
-  const list: string[] = [];
+  const list = [];
   const f = parse(filter);
   if (Array.isArray(f) && f[0] === "AND")
     for (const ff of f.slice(1)) list.push(stringifyFilter(ff));
@@ -92,82 +92,69 @@ function splitFilter(filter: string): string[] {
 
   list.push("");
   return list;
-}
-
-interface Attrs {
-  resource: string;
-  filter: string;
-  onChange: (filter: string) => void;
-}
-
-const component: ClosureComponent<Attrs> = (initialVnode) => {
-  let filterList = splitFilter(initialVnode.attrs.filter);
-  let filterInvalid = 0;
-  let filterTouched = false;
-  let attrs: Attrs = initialVnode.attrs;
-
-  function onChange(): void {
-    filterTouched = false;
-    filterInvalid = 0;
-    filterList = filterList.filter((f) => f);
-    const list = filterList.map((f, idx) => {
-      try {
-        return parseFilter(attrs.resource, f);
-      } catch (err) {
-        filterInvalid |= 1 << idx;
-      }
-      return null;
-    });
-    filterList.push("");
-
-    if (filterInvalid) {
-      m.redraw();
-      return;
-    }
-    if (list.length === 0) attrs.onChange("");
-    else if (list.length > 1) attrs.onChange(stringify(["AND", ...list]));
-    else attrs.onChange(stringify(list[0]));
-  }
+});
 
+const component: ClosureComponent = (): Component => {
   return {
-    onupdate: (vnode) => {
-      getAutocomplete(vnode.attrs.resource).reposition();
-    },
     view: (vnode) => {
-      if (attrs.filter !== vnode.attrs.filter) {
-        filterInvalid = 0;
-        filterList = splitFilter(vnode.attrs.filter);
+      if (
+        !vnode.state["filterList"] ||
+        vnode.attrs["filter"] !== vnode.state["filter"]
+      ) {
+        vnode.state["filterInvalid"] = 0;
+        vnode.state["filter"] = vnode.attrs["filter"];
+        vnode.state["filterList"] = splitFilter(vnode.attrs["filter"]);
       }
 
-      attrs = vnode.attrs;
+      function onChange(): void {
+        vnode.state["filterInvalid"] = 0;
+        vnode.state["filterList"] = vnode.state["filterList"].filter((f) => f);
+        let filter = vnode.state["filterList"].map((f, idx) => {
+          try {
+            return parseFilter(vnode.attrs["resource"], f);
+          } catch (err) {
+            vnode.state["filterInvalid"] |= 1 << idx;
+          }
+          return null;
+        });
+        vnode.state["filterList"].push("");
 
-      return m("div.filter", [
-        m("b", "Filter"),
-        ...filterList.map((fltr, idx) => {
-          return m("input", {
-            type: "text",
-            class: `${(filterInvalid >> idx) & 1 ? "error" : ""}`,
-            value: fltr,
-            oninput: (e) => {
-              e.redraw = false;
-              filterList[idx] = e.target.value;
-              filterTouched = true;
-            },
-            oncreate: (vn) => {
-              const el = vn.dom as HTMLInputElement;
-              getAutocomplete(vnode.attrs.resource).attach(el);
+        if (!vnode.state["filterInvalid"]) {
+          delete vnode.state["filter"];
+          if (filter.length === 0) {
+            vnode.attrs["onChange"]("");
+          } else {
+            if (filter.length > 1) filter = ["AND"].concat(filter);
+            else filter = filter[0];
+            vnode.attrs["onChange"](stringify(filter));
+          }
+        }
+      }
 
-              el.addEventListener("blur", () => {
-                if (filterTouched) onChange();
-              });
-
-              el.addEventListener("keydown", (e) => {
-                if (e.key === "Enter" && filterTouched) onChange();
-              });
-            },
-          });
-        }),
-      ]);
+      return m(
+        "div.filter",
+        [m("b", "Filter")].concat(
+          vnode.state["filterList"].map((fltr, idx) => {
+            return m("input", {
+              type: "text",
+              class: `${
+                (vnode.state["filterInvalid"] >> idx) & 1 ? "error" : ""
+              }`,
+              value: fltr,
+              onchange: (e) => {
+                vnode.state["filterList"] = vnode.state["filterList"].slice();
+                vnode.state["filterList"][idx] = e.target.value.trim();
+                onChange();
+              },
+              oncreate: (vn) => {
+                getAutocomplete(vnode.attrs["resource"]).attach(
+                  vn.dom as HTMLInputElement
+                );
+              },
+            });
+          })
+        )
+      );
     },
   };
 };

ui/index-table-component.ts

@@ -79,6 +79,8 @@ function renderTable(
       continue;
     }
 
+    let direction = 1;
+
     let symbol = getIcon("unsorted");
     if (sortAttributes[i] > 0) symbol = getIcon("sorted-asc");
     else if (sortAttributes[i] < 0) symbol = getIcon("sorted-dsc");
@@ -87,7 +89,7 @@ function renderTable(
       "button",
       {
         onclick: () => {
-          const direction = ((sortAttributes[i] + 2) % 3) - 1;
+          if (sortAttributes[i] > 0) direction *= -1;
           return onSortChange({ [i]: direction });
         },
       },

ui/layout.ts

@@ -23,8 +23,6 @@ import drawerComponent from "./drawer-component";
 import userMenu from "./user-menu";
 import adminMenu from "./admin-menu";
 import * as overlay from "./overlay";
-import { version as VERSION } from "../package.json";
-import datalist from "./datalist";
 
 const adminPages = [
   "presets",
@@ -53,11 +51,7 @@ const component: ClosureComponent = (): Component => {
 
       return [
         m("#header", [
-          m(
-            "div.logo",
-            m("img", { src: "logo.svg" }),
-            m("span.version", "v" + VERSION)
-          ),
+          m("img.logo", { src: "logo.svg" }),
           m(userMenu),
           m(menu, attrs),
           m(drawerComponent),
@@ -70,7 +64,6 @@ const component: ClosureComponent = (): Component => {
           ])
         ),
         overlay.render(),
-        m(datalist),
       ];
     },
   };

ui/overlay.ts

@@ -60,7 +60,7 @@ export function render(): Children {
         },
         onbeforeremove: (vnode) => {
           (vnode.dom as HTMLDivElement).style.opacity = "0";
-          return new Promise<void>((resolve) => {
+          return new Promise((resolve) => {
             setTimeout(() => {
               resolve();
             }, 500);

ui/permissions-page.ts

@@ -117,7 +117,7 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
         .resourceExists("permissions", id)
         .then((exists) => {
           if (exists) {
-            store.setTimestamp(Date.now());
+            store.fulfill(0, Date.now());
             return void resolve({ _id: "Permission already exists" });
           }
 
@@ -125,8 +125,8 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
             .putResource("permissions", id, object)
             .then(() => {
               notifications.push("success", "Permission created");
-              store.setTimestamp(Date.now());
-              resolve(null);
+              store.fulfill(0, Date.now());
+              resolve();
             })
             .catch(reject);
         })
@@ -136,11 +136,11 @@ function putActionHandler(action, _object): Promise<ValidationErrors> {
         .deleteResource("permissions", object["_id"])
         .then(() => {
           notifications.push("success", "Permission deleted");
-          store.setTimestamp(Date.now());
-          resolve(null);
+          store.fulfill(0, Date.now());
+          resolve();
         })
         .catch((err) => {
-          store.setTimestamp(Date.now());
+          store.fulfill(0, Date.now());
           reject(err);
         });
     } else {
@@ -205,16 +205,14 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = Object.assign({}, sort);
         for (const [index, direction] of Object.entries(sortAttrs)) {
           // Changing the priority of columns
           delete _sort[attributes[index].id];
-          if (direction)
-            _sort = Object.assign({ [attributes[index].id]: direction }, _sort);
+          _sort[attributes[index].id] = direction;
         }
 
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
+        const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/permissions", ops);
       }
@@ -293,7 +291,7 @@ export const component: ClosureComponent = (): Component => {
                     Object.assign(
                       {
                         actionHandler: (action, object) => {
-                          return new Promise<void>((resolve) => {
+                          return new Promise((resolve) => {
                             putActionHandler(action, object)
                               .then((errors) => {
                                 const errorList = errors
@@ -353,11 +351,11 @@ export const component: ClosureComponent = (): Component => {
                         "success",
                         `${res.length} permissions deleted`
                       );
-                      store.setTimestamp(Date.now());
+                      store.fulfill(0, Date.now());
                     })
                     .catch((err) => {
                       notifications.push("error", err.message);
-                      store.setTimestamp(Date.now());
+                      store.fulfill(0, Date.now());
                     });
                 },
               },
@@ -367,20 +365,15 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {
-        resource: "permissions",
-        filter: vnode.attrs["filter"],
-        onChange: onFilterChanged,
-      };
+      const filterAttrs = {};
+      filterAttrs["resource"] = "permissions";
+      filterAttrs["filter"] = vnode.attrs["filter"];
+      filterAttrs["onChange"] = onFilterChanged;
 
       return [
         m("h1", "Listing permissions"),
         m(filterComponent, filterAttrs),
-        m(
-          "loading",
-          { queries: [permissions, count] },
-          m(indexTableComponent, attrs)
-        ),
+        m(indexTableComponent, attrs),
       ];
     },
   };

ui/pie-chart-component.ts

@@ -116,27 +116,21 @@ function drawChart(chartData): Children {
 
   legend.push(m("span.legend-total", `Total: ${total}`));
 
-  return m(
-    "loading",
-    {
-      queries: Object.values(chartData.slices).map((s) => s["count"]),
-    },
-    m("div", { class: "pie-chart" }, [
-      m(
-        "svg",
-        {
-          // Adding 2 as padding; strokes must not be more than 2
-          viewBox: "-102 -102 204 204",
-          width: "204px",
-          height: "204px",
-          xmlns: "http://www.w3.org/2000/svg",
-          "xmlns:xlink": "http://www.w3.org/1999/xlink",
-        },
-        paths.concat(links)
-      ),
-      m(".legend", legend),
-    ])
-  );
+  return m("div", { class: "pie-chart" }, [
+    m(
+      "svg",
+      {
+        // Adding 2 as padding; strokes must not be more than 2
+        viewBox: "-102 -102 204 204",
+        width: "204px",
+        height: "204px",
+        xmlns: "http://www.w3.org/2000/svg",
+        "xmlns:xlink": "http://www.w3.org/1999/xlink",
+      },
+      paths.concat(links)
+    ),
+    m(".legend", legend),
+  ]);
 }
 
 const component: ClosureComponent = (): Component => {

ui/presets-page.ts

@@ -86,12 +86,12 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
         .resourceExists("presets", id)
         .then((exists) => {
           if (exists && isNew) {
-            store.setTimestamp(Date.now());
+            store.fulfill(0, Date.now());
             return void resolve({ _id: "Preset already exists" });
           }
 
           if (!exists && !isNew) {
-            store.setTimestamp(Date.now());
+            store.fulfill(0, Date.now());
             return void resolve({ _id: "Preset does not exist" });
           }
 
@@ -102,8 +102,8 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
                 "success",
                 `Preset ${exists ? "updated" : "created"}`
               );
-              store.setTimestamp(Date.now());
-              resolve(null);
+              store.fulfill(0, Date.now());
+              resolve();
             })
             .catch(reject);
         })
@@ -113,12 +113,12 @@ function putActionHandler(action, _object, isNew): Promise<ValidationErrors> {
         .deleteResource("presets", object["_id"])
         .then(() => {
           notifications.push("success", "Preset deleted");
-          store.setTimestamp(Date.now());
-          resolve(null);
+          store.fulfill(0, Date.now());
+          resolve();
         })
         .catch((err) => {
           reject(err);
-          store.setTimestamp(Date.now());
+          store.fulfill(0, Date.now());
         });
     } else {
       reject(new Error("Undefined action"));
@@ -190,16 +190,14 @@ export const component: ClosureComponent = (): Component => {
       }
 
       function onSortChange(sortAttrs): void {
-        let _sort = Object.assign({}, sort);
+        const _sort = Object.assign({}, sort);
         for (const [index, direction] of Object.entries(sortAttrs)) {
           // Changing the priority of columns
           delete _sort[attributes[index].id];
-          if (direction)
-            _sort = Object.assign({ [attributes[index].id]: direction }, _sort);
+          _sort[attributes[index].id] = direction;
         }
 
-        const ops = {};
-        if (Object.keys(_sort).length) ops["sort"] = JSON.stringify(_sort);
+        const ops = { sort: JSON.stringify(_sort) };
         if (vnode.attrs["filter"]) ops["filter"] = vnode.attrs["filter"];
         m.route.set("/admin/presets", ops);
       }
@@ -296,7 +294,7 @@ export const component: ClosureComponent = (): Component => {
                     {
                       base: preset,
                       actionHandler: (action, object) => {
-                        return new Promise<void>((resolve) => {
+                        return new Promise((resolve) => {
                           putActionHandler(action, object, false)
                             .then((errors) => {
                               const errorList = errors
@@ -325,7 +323,7 @@ export const component: ClosureComponent = (): Component => {
                     return m(
                       "div",
                       { style: "margin:20px" },
-                      "This UI only supports presets with a single 'provision' configuration. If this preset was originally created from the old UI (genieacs-gui), you must edit it there."
+                      "This UI only supports presets with a single 'provision' configuraiton. If this preset was originally created from the old UI (genieacs-gui), you must edit it there."
                     );
                   }
                   return comp;
@@ -333,7 +331,7 @@ export const component: ClosureComponent = (): Component => {
                 overlay.open(
                   cb,
                   () =>
-                    !comp.state?.["current"]["modified"] ||
+                    !comp.state["current"]["modified"] ||
                     confirm("You have unsaved changes. Close anyway?")
                 );
               },
@@ -357,7 +355,7 @@ export const component: ClosureComponent = (): Component => {
                     Object.assign(
                       {
                         actionHandler: (action, object) => {
-                          return new Promise<void>((resolve) => {
+                          return new Promise((resolve) => {
                             putActionHandler(action, object, true)
                               .then((errors) => {
                                 const errorList = errors
@@ -415,11 +413,11 @@ export const component: ClosureComponent = (): Component => {
                         "success",
                         `${res.length} presets deleted`
                       );
-                      store.setTimestamp(Date.now());
+                      store.fulfill(0, Date.now());
                     })
                     .catch((err) => {
                       notifications.push("error", err.message);
-                      store.setTimestamp(Date.now());
+                      store.fulfill(0, Date.now());
                     });
                 },
               },
@@ -429,20 +427,15 @@ export const component: ClosureComponent = (): Component => {
         };
       }
 
-      const filterAttrs = {
-        resource: "presets",
-        filter: vnode.attrs["filter"],
-        onChange: onFilterChanged,
-      };
+      const filterAttrs = {};
+      filterAttrs["resource"] = "presets";
+      filterAttrs["filter"] = vnode.attrs["filter"];
+      filterAttrs["onChange"] = onFilterChanged;
 
       return [
         m("h1", "Listing presets"),
         m(filterComponent, filterAttrs),
-        m(
-          "loading",
-          { queries: [presets, count] },
-          m(indexTableComponent, attrs)
-        ),
+        m(indexTableComponent, attrs),
       ];
     },
   };