Bump esbuild from 0.21.4 to 0.25.0

Bumps [esbuild](https://github.com/evanw/esbuild) from 0.21.4 to 0.25.0.
- [Release notes](https://github.com/evanw/esbuild/releases)
- [Changelog](https://github.com/evanw/esbuild/blob/main/CHANGELOG-2024.md)
- [Commits](https://github.com/evanw/esbuild/compare/v0.21.4...v0.25.0)

---
updated-dependencies:
- dependency-name: esbuild
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <support@github.com>

.eslintignore

@@ -1,2 +0,0 @@
-!/bin/.*.ts
-!/tools/.*.ts

.eslintrc.json

@@ -1,6 +1,6 @@
 {
   "env": {
-    "es6": true,
+    "es2022": true,
     "node": true
   },
   "parser": "@typescript-eslint/parser",
@@ -8,21 +8,29 @@
     "project": "./tsconfig.json"
   },
   "plugins": ["@typescript-eslint"],
-  "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended", "prettier", "prettier/@typescript-eslint"],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
   "rules": {
     "no-shadow": ["error", { "allow": ["err"] }],
     "handle-callback-err": "error",
     "prefer-arrow-callback": "error",
-    "curly": ["error", "multi-or-nest", "consistent"],
     "no-buffer-constructor": "error",
-    "prefer-const": ["error", {"destructuring": "all"}],
+    "prefer-const": ["error", { "destructuring": "all" }],
-    "eqeqeq": ["error", "always", {"null": "ignore"}],
+    "eqeqeq": ["error", "always", { "null": "ignore" }],
-    "no-return-await": "error",
     "@typescript-eslint/no-explicit-any": 0,
-    "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
+    "@typescript-eslint/no-use-before-define": [
-    "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
+      "error",
+      { "functions": false }
+    ],
+    "@typescript-eslint/explicit-function-return-type": [
+      "error",
+      { "allowExpressions": true }
+    ],
     "@typescript-eslint/no-floating-promises": "error",
-    "require-atomic-updates": 0,
+    "@typescript-eslint/no-misused-promises": "error",
     "no-prototype-builtins": 0
   }
 }

.gitignore

@@ -1,6 +1,4 @@
 *~
 node_modules
 dist
-bin/.*.ts
-tools/.*.ts
 docs/_build

.npmignore

@@ -1,25 +0,0 @@
-*~
-.npmignore
-
-config/config.json
-config/cwmp.key
-config/cwmp.crt
-config/cwmp.ca-bundle
-config/nbi.key
-config/nbi.crt
-config/nbi.ca-bundle
-config/fs.key
-config/fs.crt
-config/fs.ca-bundle
-config/ext
-
-debug/*
-!debug/.gitkeep
-
-dist
-.eslintrc.json
-.eslintignore
-ava.config.js
-tsconfig.json
-bin/.*
-tools/.*

.prettierignore

@@ -0,0 +1,2 @@
+dist
+npm-shrinkwrap.json

CHANGELOG.md

@@ -1,20 +1,196 @@
 # Change Log
 
+## 1.2.13 (2024-06-06)
+
+- Increase connection timeout for UI and NBI from 30 to 120 seconds to avoid
+  timeouts when running unindexed queries in large deployments.
+
+- Fix race condition causing 503 error when deleting multiple faults at once.
+
+- Fix some UI config options not being evaluated as dynamic expressions.
+
+- Fix an issue where certain edge-case query expressions were not being
+  correctly converted to MongoDB queries, resulting in inaccurate search
+  results.
+
+## 1.2.12 (2024-03-28)
+
+- Fix broken XMPP support in the previous release.
+
+- Fix regression causing CSV downloads to be buffered in memory before being
+  streamed to the client.
+
+## 1.2.11 (2024-03-21)
+
+- Resolved an issue from the previous release that caused incompatibility with
+  Node.js versions 12 through 15.
+
+## 1.2.10 (2024-03-18)
+
+- Add support for XMPP connection requests. Use the environment variables
+  `XMPP_JID` and `XMPP_PASSWORD` to configure the XMPP connection for the ACS.
+
+- The environment variables `CWMP_SSL_CERT` and `CWMP_SSL_KEY`, as well as their
+  counterparts for UI, NBI, and FS, now accept PEM-encoded certificates and keys
+  in addition to file paths.
+
+- The UI no longer requires users to refresh the page after modifying presets,
+  provisions, or virtual parameters. Refreshing is now only necessary for
+  changes to users, permissions, or UI configurations.
+
+- Improved conversion of GenieACS expressions into MongoDB queries for more
+  optimized queries and better index utilization.
+
+- Refactor UI pagination and sorting to fix issues from the previous approach,
+  especially with sorting by rapidly changing parameters such as 'last inform'
+  time.
+
+- The file server now supports HEAD requests, the Range header, and conditional
+  requests.
+
+- Addressed an issue causing file server disconnections for slow clients over
+  HTTPS.
+
+- Fix bug preventing users from changing their own passwords.
+
+- Introduced a new 'locks' collection for database locking, replacing the
+  previous use of the 'cache' collection for this purpose.
+
+- Various other minor fixes and enhancements.
+
+## 1.2.9 (2022-08-22)
+
+- New config option `cwmp.skipRootGpn` to enable a workaround for some
+  problematic CPEs that reject GPN requests on data model root.
+- Stream query results and CSV downloads as data becomes available instead of
+  buffering the entire response.
+- Log HTTP/HTTPS client errors in debug log.
+- Fix occasional lock expired errors after updating presets, etc.
+- Fix bug where queries containing `<>` operator may return incorrect results.
+- Fix a performance hit caused by DB calls containing the entire CPE data model
+  rather then just the updated parameters.
+- Fix bug where tags containing special characters are saved in their encoded
+  form when set via a Provision script.
+- Fix issue causing invalid `xsd:dateTime` values to be saved in DB as `NaN`
+  rather than maintain the original string value.
+- Fix bug using `$regex` operator with a numeric or a datetime string.
+- Fix ping not working on certain platforms.
+- Ping requests are now authenticated.
+- Fix error when the `FORWARDED_HEADER` environment variable contains IPv4 CIDR
+  while the listening interface is IPv6 and vice versa.
+- Fix false warning "unexpected parameter in response" showing in
+  `genieacs-cwmp` logs.
+- Other minor fixes and stability improvements.
+
+## 1.2.8 (2021-10-27)
+
+- Fix a remote code execution security vulnerability in genieacs-ui.
+- All UI components can now be configured using fully dynamic expressions.
+  Previously some component types only accept fixed string values as properties.
+- The `container` UI component can now be configured with an `element` property
+  that's either a string or a nested pair of `tag` and `attributes` properties.
+  The various attributes under the `attributes` property can now make use of a
+  new function `ENCODEURICOMPONENT()` to facilitate creating custom hyperlinks
+  in the UI.
+- Improve sorting buttons' behavior in the device listing page and other pages.
+  Sorting by multiple columns should now feel more intuitive.
+- Support the modulo (%) operator in expressions.
+- Fix a regression in the previous release where the config option 'ui.pageSize'
+  no longer works.
+- Fix process crash when a CPE sends an unsupported ACS method.
+
+## 1.2.7 (2021-09-18)
+
+- Fix regression causing frequent invalid session errors.
+- Fix regression breaking digest authentication in connection requests.
+
+## 1.2.6 (2021-09-16)
+
+- New config option `cwmp.downloadSuccessOnTimeout` to enable a workaround for
+  CPEs that neglect to send a TransferComplete message after a successful
+  download.
+- Display a progress bar when uploading new files.
+- Default to dual stack interface binding (i.e. `::` instead of `0.0.0.0`).
+  Unless the binding interface is explicitly set, this will cause IPv4 addresses
+  in the logs to be displayed as IPv4-mapped IPv6 addresses (e.g.
+  `::ffff:127.0.0.1` instead of `127.0.0.1`).
+- Detect and correct for client side (browser) clock skew that would otherwise
+  alter the numbers in the pie charts.
+- Various improvements relating to dealing with buggy TR-069 client
+  implementations.
+- Fix a bug causing "lock expired" exceptions when a CWMP session remains open
+  for a very long time due to slow clients.
+- Fix metadata of uploaded files going missing due to nginx stripping away what
+  it considers to be invalid headers. The nginx directive
+  `ignore_invalid_headers` is no longer required.
+- Fix crash when a CPE is assigned a tag containing a dot.
+- Fix bug preventing the user from closing the preset pop-up after being
+  presented with unsupported preset message.
+- Fix exceptions raised from ext scripts manifesting as timeout faults.
+- Fix download getting triggered repeatedly when the value passed to `Download`
+  parameter is greater than the current timestamp.
+- Fix crash when passing invalid attributes to `declare`.
+- Fix crash in NBI when pushing tasks to non-existing devices.
+- Fix crash in NBI when passing invalid JSON to various API endpoints.
+- Fix crash when the output from `ping` command cannot be parsed in some rare
+  edge cases.
+- A number of other fixes and stability improvements.
+
+## 1.2.5 (2021-03-12)
+
+- Support specifying custom types when uploading files.
+- Fix JS compatibility issue with Safari browser.
+
+## 1.2.4 (2021-02-24)
+
+- The data model state of a CPE is no longer forgotten after unsuccessful
+  session termination (e.g. timeout). This addresses a number of undesired side
+  effects that arise when a CPE does not terminate the session properly.
+- Executing tasks that take a long time to complete (e.g. refreshing the entire
+  data model) no longer shows a timeout error while the task is still being
+  processed.
+- New function `ROUND()` available to expressions. It works similar to the
+  function by the same name in SQLite and PostgreSQL.
+- Log access events for `genieacs-nbi` service.
+- Pipe stdout/stderr from extension scripts to the `genieacs-cwmp` process log.
+- Parameter values of type `xsd:dateTime` are now displayed in the UI and CSV
+  downloads as a date string rather than a numeric value.
+- Add file download link in the files listing page.
+- Display spinner loading animation throughout the UI.
+- Display GenieACS version and build number underneath the logo.
+- New option to specify how many parameters are displayed at a time in the
+  all-parameters component. Simply set `limit` property in the component config.
+- Reduce overly strong Brotli compression level which was causing significant
+  page load slowdown when Brotli is used.
+- Retire dump-data-model tool. `genieacs-sim` can now use a CSV file as its data
+  model.
+- Reduce the number of concurrent database connections from each process.
+- Remove dependency on 'hostInfo' MongoDB command which is a privileged action.
+  It is now possible to use shared MongoDB instances with limited privileged.
+- Fix bug in NBI where querying files returns 404 error.
+- Fix ping not working for devices with an IPv6 address.
+- Fix an elusive memory leak in `genieacs-fs` that slowly eats up memory and can
+  go unnoticed for long periods of time.
+- Fix a rare edge case where a `declare()` call to set a parameter value may not
+  work as intended if the parameter was originally received as part of the
+  Inform message.
+- A number of other fixes and stability improvements.
+
 ## 1.2.3 (2020-10-26)
 
 - New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
-report writable parameters as non-writable. When set to true, the scripts will
+  report writable parameters as non-writable. When set to true, the scripts will
-no longer respect the 'writable' attribute of the CPE parameters and will send
+  no longer respect the 'writable' attribute of the CPE parameters and will send
-a SetParamteerValues, AddObject, or DeleteObject request anyway.
+  a SetParamteerValues, AddObject, or DeleteObject request anyway.
 - Tags no longer restrict what characters are allowed. Any character other than
-alphanumeric characters, hyphen, or underscore is now encoded in the data
+  alphanumeric characters, hyphen, or underscore is now encoded in the data
-model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
+  model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
 - Ask for a confirmation before closing a pop-up dialog with unsaved changes.
 - Better XML validation to avoid crashes caused by invalid CPE requests.
 - Fix confusing 404 error message when the user attempts to modify a resource
-when they don't have the necessary permissions.
+  when they don't have the necessary permissions.
 - Fix a rare issue where genieacs-cwmp stops accepting new connections after
-running for a few weeks.
+  running for a few weeks.
 - Fix exception when IS NULL operator is used in certain situations.
 
 ## 1.2.2 (2020-10-03)
@@ -24,81 +200,80 @@ running for a few weeks.
 - Fix exception that can happen and persist after a Download request.
 - Fix validation bug preventing running refreshObject task on data model root.
 - Fix invalid arguments fault in refresh preset configuration when upgrading
-from v1.1.
+  from v1.1.
 
 ## 1.2.1 (2020-09-08)
 
 - Fix bug causing faults to not be displayed in the UI.
 - Fix bug where deleting objects does not get reflected immediately in the UI.
 - Improve conversion between filters written in the expression format and
-MongoDB queries. There should now be fewer edge cases where the two are not
+  MongoDB queries. There should now be fewer edge cases where the two are not
-equisatisfiable.
+  equisatisfiable.
 
 ## 1.2.0 (2020-09-01)
 
 - Support GetParameterAttributes and SetParameterAttributes TR-069 methods.
 - Support CASE statement and COALESCE function in expressions.
 - Provision arguments can now be a list of expressions that are dynamically
-evaluated.
+  evaluated.
 - Support Forwarded HTTP header to display in the logs the correct IP of CPEs
-behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
+  behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
 - Config expressions can now access all available device parameters, not only
-serial number, product class, and OUI.
+  serial number, product class, and OUI.
-- Use relative URLs throughout the UI to allow serving from a subdirectory
+- Use relative URLs throughout the UI to allow serving from a subdirectory using
-using a reverse proxy.
+  a reverse proxy.
 - Make Date.parse() and Date.UTC() available to provision scripts.
 - libxmljs has been entirely removed in favor of our bespoke XML parser.
-- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now
+- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now used
-used to determine the TCP connection timeout.
+  to determine the TCP connection timeout.
-- The all-parameters component now limits the number of parameters displayed
+- The all-parameters component now limits the number of parameters displayed for
-for better performance.
+  better performance.
 - The process genieacs-cwmp is now much less likely to throw exceptions as a
-result of invalid requests from CPE.
+  result of invalid requests from CPE.
 - A large number of bug fixes and stability improvements.
 
 ## 1.2.0-beta.0 (2019-07-30)
 
 - A brand new UI superseding genieacs-gui.
 - New initialization wizard on first run.
-- New expression/query language used in search filters and preset
+- New expression/query language used in search filters and preset preconditions.
-preconditions.
 - CPE -> ACS authentication is now supported.
 - New config option (CWMP_KEEP_ALIVE_TIMEOUT) to specify how long to wait for a
-reply from the CPE before closing the TCP connection.
+  reply from the CPE before closing the TCP connection.
 - Debug logging has been reimplemented utilizing YAML format for logs.
 - Handle 9005 faults (Invalid Parameter Name) gracefully by attempting to
-rediscover the path of the missing parameter recursively.
+  rediscover the path of the missing parameter recursively.
 - declare() statements not followed by an explicit commit() are now deferred
-until all currently active scripts have been executed.
+  until all currently active scripts have been executed.
 - FS_HOSTNAME now defaults to the server's hostname or IP.
 - The API now validates the structure of task objects before saving.
 - New XML parser implementation for better performance. You can revert to the
-old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
+  old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
-v10.
+  v10.
 - Performance optimizations. While performance has improved for the majority of
-use cases, there may be situations where performance has degraded. It's
+  use cases, there may be situations where performance has degraded. It's
-recommended to revisit your hardware requirements.
+  recommended to revisit your hardware requirements.
 - Connection request authentication no longer uses 'auth.js' file. Instead, the
-connection request authentication behavior can now be customized using an
+  connection request authentication behavior can now be customized using an
-'expression'.
+  'expression'.
 - The config file (config.json) has been deprecated. System configuration (e.g.
-listen ports, worker count) are now recommended to be passed as environment.
+  listen ports, worker count) are now recommended to be passed as environment.
-variables. Other general configuration options are stored in the database so as
+  variables. Other general configuration options are stored in the database so
-to not require service restart for changes to take effect.
+  as to not require service restart for changes to take effect.
 - Optional redis dependency has been removed completely.
 - Tags now allow only alphanumeric characters and underscore.
-- Supported versions of NodeJs and MongoDB are 10.x and up and 2.6 and up
+- Supported versions of Node.js and MongoDB are 10.x and up and 2.6 and up
-respectively.
+  respectively.
 
 ## 1.1.3 (2018-10-23)
 
 - New config option (MAX_COMMIT_ITERATIONS) to avoid max commit iterations
-faults for more complex scripts.
+  faults for more complex scripts.
 - Support base64 and hexBinary parameter types.
 - Strict parsing of number values in queries (e.g. "123abc" no longer accepted
-as 123).
+  as 123).
 - Mixing $ne and $not operators is not allowed. Now it throws an error instead
-of returning incorrect results.
+  of returning incorrect results.
 - When a task expires, any associated fault is also deleted.
 - API now accepts 'timeout' argument when posting a task.
 - A number of stability fixes.
@@ -108,66 +283,68 @@ of returning incorrect results.
 - A large number of bug fixes as well as stability and performance improvements.
 - Three security vulnerabilities disclosed by Maximilian Hils have been patched.
 - New config option UDP_CONNECTION_REQUEST_PORT to specify binding port for UDP
-connection requests.
+  connection requests.
 - New config option DATETIME_MILLISECONDS to strip milliseconds from dateTime
-values.
+  values.
 - New config option BOOLEAN_LITERAL to use 1/0 or true/false for boolean values.
 - Parameter values that cannot be parsed according to the reported type now show
-a warning message.
+  a warning message.
 - Virtual parameter scripts now use the variable 'args' instead of the special
-'TIMESTAMPS' and 'VALUES' variables. The content of the args array is: {declare
+  'TIMESTAMPS' and 'VALUES' variables. The content of the args array is:
-timestamps}, {declare values}, {current timestamps}, {current values}.
+  {declare timestamps}, {declare values}, {current timestamps}, {current
+  values}.
 - Virtual parameter value types are now inferred from the JavaScript type if the
-returned value attribute is not a value-type pair.
+  returned value attribute is not a value-type pair.
 - Show a fault when a virtual parameter script doesn't return the required
-attributes.
+  attributes.
 - Redis is now optional (and disabled by default), reducing the complexity of
-scalable deployments.
+  scalable deployments.
 - Better detection of cyclical presets resulting in fewer faults for complex
-provisioning scripts.
+  provisioning scripts.
 - Math.random() is now deterministic on per-device basis. A function has been
-added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
+  added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
 - Overload spikes are now handled gracefully by refusing to accept new sessions
-temporarily when under abnormal load.
+  temporarily when under abnormal load.
 - Added log messages for session timeouts, connection drops, and XML parsing
-errors.
+  errors.
 - Date.now() now takes an optional argument to specify "time steps" (in
-milliseconds). This can be used to ensure a group of parameters are all
+  milliseconds). This can be used to ensure a group of parameters are all
-refreshed at the same time intervals.
+  refreshed at the same time intervals.
 - Only the non-default configuration options are now logged at process start.
 - Faults caused by errors from extensions now show a cleaner stack trace.
 - Exit main process if there are too many worker crashes (e.g. when DB is down).
 - Updated dependencies and included a lockfile to ensure installations get the
-exact dependencies it was tested against.
+  exact dependencies it was tested against.
 
 ## 1.1.1 (2017-03-23)
 
 - Avoid crashing when connection request credentials are missing.
 - Show a warning instead of crashing when failing to parse parameter values
-according to the expected value type.
+  according to the expected value type.
 - Add missing "Registered" event.
 - Fix bug where in certain cases many more instances than declared are created.
 - Fix parameter discovery bug when declared path timestamp is 1 or is not set.
 - Fix preset precondition failing when testing against datetime parameters and
-certain other parameters like \_deviceId.\_ProductClass.
+  certain other parameters like \_deviceId.\_ProductClass.
 
 ## 1.1.0 (2017-03-10)
 
 - Provisions enable implementing dynamic device configuration or complex device
-provisioning work flow using arbitrary scripts.
+  provisioning work flow using arbitrary scripts.
 - Virtual parameters are user-defined parameters whose values are evaluated from
-a custom script.
+  a custom script.
 - Extensions are sandboxed Node.js scripts that are accessible from provision
-and virtual parameter scripts to facilitate integration with external entities.
+  and virtual parameter scripts to facilitate integration with external
-- Support for UDP/STUN based connection requests for reaching devices behind
+  entities.
-NAT (TR-069 Annex G).
+- Support for UDP/STUN based connection requests for reaching devices behind NAT
+  (TR-069 Annex G).
 - Presets can now be scheduled using a cron-like expression.
 - Presets can now be tied to specific device events (e.g. boot).
-- Presets precondition queries no longer support "$or" or other MongoDB logical
+- Presets precondition queries no longer support "\$or" or other MongoDB logical
-operators.
+  operators.
 - Faults are no longer a part of tasks but are now first class objects.
 - Presets are now assigned to channels. A fault in one channel only blocks
-presets in that channel.
+  presets in that channel.
 - New API CRUD functions for provisions, virtual parameters, and faults.
 - New config options for XML output.
 - API responses now include "GenieACS-Version" header.
@@ -176,19 +353,19 @@ presets in that channel.
 - Supported Node.js versions are 6.x and 7.x.
 - Supported MongoDB versions are 2.6 through 3.4.
 - Expect performance differences due to major under the hood changes. Some
-operations are faster and some are slower. Overall performance is improved.
+  operations are faster and some are slower. Overall performance is improved.
 - GenieACS will no longer fetch the entire device data model upon first contact
-but will instead only fetch the parameters it needs to fulfill the presets.
+  but will instead only fetch the parameters it needs to fulfill the presets.
 - Logs have been overhauled and split into two streams: process log (stderr) and
-access log (stdout). Also added config options to dump logs to files rather than
+  access log (stdout). Also added config options to dump logs to files rather
-standard streams.
+  than standard streams.
 - Connection request authentication credentials are picked up from the device
-data model if available. config/auth.js is still supported as a fallback and now
+  data model if available. config/auth.js is still supported as a fallback and
-supports an optional callback argument.
+  now supports an optional callback argument.
 - Custom commands have been removed. Use virtual parameters and/or extensions.
 - Aliases and value normalizers (config/parameters.json) have been removed. Use
-virtual parameters.
+  virtual parameters.
 - The API /devices/<device_id>/preset has been removed.
 - Rarely used RequestDownload method no longer supported.
 - The TR-069 client simulator has moved to its own repo at
-https://github.com/zaidka/genieacs-sim
+  https://github.com/zaidka/genieacs-sim

CONTRIBUTING.md

@@ -5,8 +5,8 @@ tracker is strictly for bug reports.
 
 # Issues
 
-We use GitHub issues to track bugs. Please ensure your bug description is
+We use GitHub issues to track bugs. Please ensure your bug description is clear
-clear and has sufficient instructions to be able to reproduce the issue.
+and has sufficient instructions to be able to reproduce the issue.
 
 If you face interoperability issues with your CPE, more often than not it's a
 device related issue. Please consult the [forum] before submitting an Issue.
@@ -26,13 +26,14 @@ By submitting a pull request to this repository, you acknowledge that, while
 maintaining copyright, you grant Zaid Abdulla a perpetual, worldwide, non-
 exclusive, no-charge, royalty-free, irrevocable license to reproduce, prepare
 derivative works of, publicly display, publicly perform, sublicense, and
-distribute your contributions and such derivative works under the AGPLv3
+distribute your contributions and such derivative works under the AGPLv3 license
-license or any other license terms, including, but not limited to, proprietary
+or any other license terms, including, but not limited to, proprietary or
-or commercial license terms.
+commercial license terms.
 
-You confirm that you own or have rights to distribute and sublicense the
+You confirm that you own or have rights to distribute and sublicense the source
-source code contained therein, and that your content does not infringe upon
+code contained therein, and that your content does not infringe upon the
-the intellectual property rights of a third party.
+intellectual property rights of a third party.
 
-[commit-guidelines]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
+[commit-guidelines]:
+  http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
 [forum]: https://forum.genieacs.com

README.md

@@ -17,10 +17,10 @@ supported versions are:
 
 Install GenieACS from NPM:
 
-    sudo npm install -g --unsafe-perm genieacs
+    sudo npm install -g genieacs
 
 To build from source instead, clone this repo or download the source archive
-then *cd* into the source directory then run:
+then _cd_ into the source directory then run:
 
     npm install
     npm run build
@@ -30,8 +30,8 @@ source):
 
 ### genieacs-cwmp
 
-This is the service that the CPEs will communicate with. It listens on port
+This is the service that the CPEs will communicate with. It listens on port 7547
-7547 by default. Configure the ACS URL in your devices accordingly.
+by default. Configure the ACS URL in your devices accordingly.
 
 You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
 as a dummy TR-069 simulator if you don't have a CPE at hand.
@@ -50,7 +50,7 @@ such. It listens on port 7567 by default.
 ### genieacs-ui
 
 This serves the web based user interface. It listens on port 3000 by default.
-You must pass *--ui-jwt-secret* argument to supply the secret key used for
+You must pass _--ui-jwt-secret_ argument to supply the secret key used for
 signing browser cookies:
 
     genieacs-ui --ui-jwt-secret secret
@@ -64,14 +64,13 @@ a complete installation guide for production deployments.
 
 ## Support
 
-The [forum](https://forum.genieacs.com) is a good place to get guidance and
+The [forum](https://forum.genieacs.com) is a good place to get guidance and help
-help from the community. Head on over and join the conversation!
+from the community. Head on over and join the conversation!
 
 For commercial support options, please visit
 [genieacs.com](https://genieacs.com/support/).
 
 ## License
 
-Copyright 2013-2020 GenieACS Inc. GenieACS is released under the [AGPLv3
+Copyright 2013-2021 GenieACS Inc. GenieACS is released under the
-license
+[AGPLv3 license terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).
-terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

ava.config.js

@@ -1,5 +0,0 @@
-export default {
-  "extensions": ["ts"],
-  "require": ["esm", "ts-node/register/transpile-only"]
-};
-

bin/genieacs-cwmp.ts

@@ -1,29 +1,10 @@
-/**
+import * as config from "../lib/config.ts";
- * Copyright 2013-2019  GenieACS Inc.
+import * as logger from "../lib/logger.ts";
- *
+import * as cluster from "../lib/cluster.ts";
- * This file is part of GenieACS.
+import * as server from "../lib/server.ts";
- *
+import * as cwmp from "../lib/cwmp.ts";
- * GenieACS is free software: you can redistribute it and/or modify
+import * as db from "../lib/db/db.ts";
- * it under the terms of the GNU Affero General Public License as
+import * as extensions from "../lib/extensions.ts";
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import * as cwmp from "../lib/cwmp";
-import * as db from "../lib/db";
-import * as extensions from "../lib/extensions";
 import { version as VERSION } from "../package.json";
 
 logger.init("cwmp", VERSION);
@@ -37,11 +18,13 @@ function exitWorkerGracefully(): void {
     db.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
-  ]).catch(exitWorkerUngracefully);
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
+  void extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -83,10 +66,17 @@ if (!cluster.worker) {
     host: SERVICE_ADDRESS,
     ssl: key && cert ? { key, cert } : null,
     onConnection: cwmp.onConnection,
+    onClientError: cwmp.onClientError,
     timeout: 30000,
     keepAliveTimeout: 0,
+    requestTimeout: cwmp.REQUEST_TIMEOUT,
   };
 
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
+
   process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
     logger.error({
@@ -94,7 +84,7 @@ if (!cluster.worker) {
       exception: err,
       pid: process.pid,
     });
-    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    server.stop(false).then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
   const initPromise = db
@@ -109,14 +99,20 @@ if (!cluster.worker) {
     });
 
   process.on("SIGINT", () => {
-    initPromise.finally(() => {
+    void initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+      server
+        .stop(false)
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
-    initPromise.finally(() => {
+    void initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+      server
+        .stop(false)
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
     });
   });
 }

bin/genieacs-ext.ts

@@ -1,23 +1,4 @@
-/**
+import { Fault } from "../lib/types.ts";
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { Fault } from "../lib/types";
 
 const jobs = new Set();
 const fileName = process.argv[2];
@@ -41,12 +22,12 @@ function errorToFault(err: Error): Fault {
     fault.detail["stack"] = err.stack;
     // Trim the stack trace
     const stackTrimIndex = fault.detail["stack"].match(
-      /\s+at\s[^\s]+\s\(.*genieacs-ext:.+\)/
+      /\s+at\s[^\s]+\s\(.*genieacs-ext:.+\)/,
     );
     if (stackTrimIndex) {
       fault.detail["stack"] = fault.detail["stack"].slice(
         0,
-        stackTrimIndex.index
+        stackTrimIndex.index,
       );
     }
   }
@@ -54,6 +35,11 @@ function errorToFault(err: Error): Fault {
   return fault;
 }
 
+// Need this for Node < 15
+process.on("unhandledRejection", (err) => {
+  throw err;
+});
+
 process.on("uncaughtException", (err) => {
   const fault = errorToFault(err);
   jobs.forEach((jobId) => {

bin/genieacs-fs.ts

@@ -1,28 +1,9 @@
-/**
+import * as config from "../lib/config.ts";
- * Copyright 2013-2019  GenieACS Inc.
+import * as logger from "../lib/logger.ts";
- *
+import * as cluster from "../lib/cluster.ts";
- * This file is part of GenieACS.
+import * as server from "../lib/server.ts";
- *
+import { listener } from "../lib/fs.ts";
- * GenieACS is free software: you can redistribute it and/or modify
+import * as db from "../lib/db/db.ts";
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/fs";
-import * as db from "../lib/db";
 import { version as VERSION } from "../package.json";
 
 logger.init("fs", VERSION);
@@ -32,9 +13,9 @@ const SERVICE_PORT = config.get("FS_PORT") as number;
 
 function exitWorkerGracefully(): void {
   setTimeout(exitWorkerUngracefully, 5000).unref();
-  Promise.all([db.disconnect(), cluster.worker.disconnect()]).catch(
+  Promise.all([db.disconnect(), cluster.worker.disconnect()])
-    exitWorkerUngracefully
+    .then(logger.close)
-  );
+    .catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
@@ -79,7 +60,10 @@ if (!cluster.worker) {
     timeout: 30000,
   };
 
-  let stopping = false;
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
 
   process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
@@ -88,19 +72,13 @@ if (!cluster.worker) {
       exception: err,
       pid: process.pid,
     });
-    stopping = true;
     server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
-  };
-
   const initPromise = db
     .connect()
     .then(() => {
-      server.start(options, _listener);
+      server.start(options, listener);
     })
     .catch((err) => {
       setTimeout(() => {
@@ -109,15 +87,13 @@ if (!cluster.worker) {
     });
 
   process.on("SIGINT", () => {
-    stopping = true;
+    void initPromise.finally(() => {
-    initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
-    stopping = true;
+    void initPromise.finally(() => {
-    initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });

bin/genieacs-nbi.ts

@@ -1,29 +1,10 @@
-/**
+import * as config from "../lib/config.ts";
- * Copyright 2013-2019  GenieACS Inc.
+import * as logger from "../lib/logger.ts";
- *
+import * as cluster from "../lib/cluster.ts";
- * This file is part of GenieACS.
+import * as server from "../lib/server.ts";
- *
+import { listener } from "../lib/nbi.ts";
- * GenieACS is free software: you can redistribute it and/or modify
+import * as db from "../lib/db/db.ts";
- * it under the terms of the GNU Affero General Public License as
+import * as extensions from "../lib/extensions.ts";
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/nbi";
-import * as db from "../lib/db";
-import * as extensions from "../lib/extensions";
 import { version as VERSION } from "../package.json";
 
 logger.init("nbi", VERSION);
@@ -37,11 +18,13 @@ function exitWorkerGracefully(): void {
     db.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
-  ]).catch(exitWorkerUngracefully);
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
+  void extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -81,10 +64,13 @@ if (!cluster.worker) {
     port: SERVICE_PORT,
     host: SERVICE_ADDRESS,
     ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
+    timeout: 120000,
   };
 
-  let stopping = false;
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
 
   process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
@@ -93,19 +79,13 @@ if (!cluster.worker) {
       exception: err,
       pid: process.pid,
     });
-    stopping = true;
     server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
-  };
-
   const initPromise = db
     .connect()
     .then(() => {
-      server.start(options, _listener);
+      server.start(options, listener);
     })
     .catch((err) => {
       setTimeout(() => {
@@ -114,15 +94,13 @@ if (!cluster.worker) {
     });
 
   process.on("SIGINT", () => {
-    stopping = true;
+    void initPromise.finally(() => {
-    initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
-    stopping = true;
+    void initPromise.finally(() => {
-    initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });

bin/genieacs-ui.ts

@@ -1,29 +1,10 @@
-/**
+import * as config from "../lib/config.ts";
- * Copyright 2013-2019  GenieACS Inc.
+import * as logger from "../lib/logger.ts";
- *
+import * as cluster from "../lib/cluster.ts";
- * This file is part of GenieACS.
+import * as server from "../lib/server.ts";
- *
+import { listener } from "../lib/ui.ts";
- * GenieACS is free software: you can redistribute it and/or modify
+import * as extensions from "../lib/extensions.ts";
- * it under the terms of the GNU Affero General Public License as
+import * as db from "../lib/db/db.ts";
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/ui";
-import * as extensions from "../lib/extensions";
-import * as db from "../lib/db";
 import { version as VERSION } from "../package.json";
 
 logger.init("ui", VERSION);
@@ -37,11 +18,13 @@ function exitWorkerGracefully(): void {
     db.disconnect(),
     extensions.killAll(),
     cluster.worker.disconnect(),
-  ]).catch(exitWorkerUngracefully);
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
 }
 
 function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
+  void extensions.killAll().finally(() => {
     process.exit(1);
   });
 }
@@ -81,10 +64,13 @@ if (!cluster.worker) {
     port: SERVICE_PORT,
     host: SERVICE_ADDRESS,
     ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
+    timeout: 120000,
   };
 
-  let stopping = false;
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
 
   process.on("uncaughtException", (err) => {
     if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
@@ -93,19 +79,13 @@ if (!cluster.worker) {
       exception: err,
       pid: process.pid,
     });
-    stopping = true;
     server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
   });
 
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
-  };
-
   const initPromise = db
     .connect()
     .then(() => {
-      server.start(options, _listener);
+      server.start(options, async (req, res) => listener(req, res));
     })
     .catch((err) => {
       setTimeout(() => {
@@ -114,15 +94,13 @@ if (!cluster.worker) {
     });
 
   process.on("SIGINT", () => {
-    stopping = true;
+    void initPromise.finally(() => {
-    initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });
 
   process.on("SIGTERM", () => {
-    stopping = true;
+    void initPromise.finally(() => {
-    initPromise.finally(() => {
       server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
     });
   });

build/assets.ts

@@ -0,0 +1,5 @@
+export const APP_JS = "app.js";
+export const APP_CSS = "app.css";
+export const ICONS_SVG = "icons.svg";
+export const LOGO_SVG = "logo.svg";
+export const FAVICON_PNG = "favicon.png";

build/build.ts

@@ -1,100 +1,46 @@
-/**
+import path from "node:path";
- * Copyright 2013-2019  GenieACS Inc.
+import fs from "node:fs";
- *
+import { createHash } from "node:crypto";
- * This file is part of GenieACS.
+import { promisify } from "node:util";
- *
+import { exec } from "node:child_process";
- * GenieACS is free software: you can redistribute it and/or modify
+import * as esbuild from "esbuild";
- * it under the terms of the GNU Affero General Public License as
+import { optimize } from "svgo";
- * published by the Free Software Foundation, either version 3 of the
+import * as xmlParser from "../lib/xml-parser.ts";
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
 
-import * as path from "path";
+const fsAsync = {
-import * as fs from "fs";
+  readdir: promisify(fs.readdir),
-import { rollup, WarningHandler } from "rollup";
+  readFile: promisify(fs.readFile),
-import rollupJson from "@rollup/plugin-json";
+  writeFile: promisify(fs.writeFile),
-import typescript from "@rollup/plugin-typescript";
+  copyFile: promisify(fs.copyFile),
-import { terser } from "rollup-plugin-terser";
+  rename: promisify(fs.rename),
-import nodeResolve from "@rollup/plugin-node-resolve";
+  chmod: promisify(fs.chmod),
-import commonjs from "@rollup/plugin-commonjs";
+  lstat: promisify(fs.lstat),
-import webpack, { Configuration } from "webpack";
+  exists: promisify(fs.exists),
-import postcss from "postcss";
+  rmdir: promisify(fs.rmdir),
-import postcssImport from "postcss-import";
+  unlink: promisify(fs.unlink),
-import postcssPresetEnv from "postcss-preset-env";
+  mkdir: promisify(fs.mkdir),
-import cssnano from "cssnano";
+};
-import SVGO from "svgo";
+
-import * as xmlParser from "../lib/xml-parser";
+const execAsync = promisify(exec);
 
 const MODE = process.env["NODE_ENV"] || "production";
 
-const BUILD_METADATA = new Date()
+const INPUT_DIR = process.cwd();
-  .toISOString()
+const OUTPUT_DIR = path.join(INPUT_DIR, "dist");
-  .split(".")[0]
-  .replace(/[^0-9]/g, "");
 
-const INPUT_DIR = path.resolve(__dirname, "..");
+async function rmDir(dirPath: string): Promise<void> {
-const OUTPUT_DIR = path.resolve(__dirname, "../dist");
+  if (!(await fsAsync.exists(dirPath))) return;
-
+  const files = await fsAsync.readdir(dirPath);
-const externals = [
-  "path",
-  "fs",
-  "cluster",
-  "os",
-  "tls",
-  "http",
-  "https",
-  "zlib",
-  "crypto",
-  "util",
-  "mongodb",
-  "vm",
-  "later",
-  "parsimmon",
-  "seedrandom",
-  "querystring",
-  "child_process",
-  "dgram",
-  "url",
-  "iconv-lite",
-  "koa",
-  "koa-router",
-  "koa-compress",
-  "koa-bodyparser",
-  "koa-jwt",
-  "koa-static",
-  "jsonwebtoken",
-  "stream",
-  "mithril",
-  "parsimmon",
-  "yaml",
-  "codemirror",
-  "codemirror/mode/javascript/javascript",
-  "codemirror/mode/yaml/yaml",
-  "ipaddr.js",
-  "jsbi",
-  "espresso-iisojs",
-];
-
-function rmDirSync(dirPath): void {
-  if (!fs.existsSync(dirPath)) return;
-  const files = fs.readdirSync(dirPath);
 
   for (const file of files) {
-    const filePath = `${dirPath}/${file}`;
+    const filePath = path.join(dirPath, file);
-    if (fs.statSync(filePath).isFile()) fs.unlinkSync(filePath);
+    if ((await fsAsync.lstat(filePath)).isDirectory()) await rmDir(filePath);
-    else rmDirSync(filePath);
+    else await fsAsync.unlink(filePath);
   }
-  fs.rmdirSync(dirPath);
+  await fsAsync.rmdir(dirPath);
 }
 
+// For lockfileVersion = 1
 function stripDevDeps(deps): void {
   if (!deps["dependencies"]) return;
   for (const [k, v] of Object.entries(deps["dependencies"])) {
@@ -104,6 +50,15 @@ function stripDevDeps(deps): void {
   if (!Object.keys(deps["dependencies"]).length) delete deps["dependencies"];
 }
 
+// For lockfileVersion = 2
+function stripDevDeps2(deps): void {
+  if (!deps["packages"]) return;
+  for (const [k, v] of Object.entries(deps["packages"])) {
+    delete v["devDependencies"];
+    if (v["dev"]) delete deps["packages"][k];
+  }
+}
+
 function xmlTostring(xml): string {
   const children = [];
   for (const c of xml.children || []) children.push(xmlTostring(c));
@@ -113,6 +68,59 @@ function xmlTostring(xml): string {
     : `<${xml.name} ${xml.attrs}>${children.join("")}</${xml.name}>`;
 }
 
+function assetHash(buffer: Buffer | string): string {
+  return createHash("md5").update(buffer).digest("hex").slice(0, 8);
+}
+
+const ASSETS = {} as {
+  APP_JS?: string;
+  APP_CSS?: string;
+  ICONS_SVG?: string;
+  LOGO_SVG?: string;
+  FAVICON_PNG?: string;
+};
+
+const assetsPlugin = {
+  name: "assets",
+  setup(build) {
+    build.onLoad({ filter: /\/build\/assets.ts$/ }, () => {
+      const lines = Object.entries(ASSETS).map(
+        ([k, v]) => `export const ${k} = ${JSON.stringify(v)};`,
+      );
+      return { contents: lines.join("\n") };
+    });
+  },
+} as esbuild.Plugin;
+
+const packageDotJsonPlugin = {
+  name: "packageDotJson",
+  setup(build) {
+    const sourcePath = path.join(INPUT_DIR, "package.json");
+    build.onResolve({ filter: /\/package.json$/ }, (args) => {
+      const p = path.join(args.resolveDir, args.path);
+      if (p !== sourcePath) return undefined;
+      return { path: path.join(OUTPUT_DIR, "package.json") };
+    });
+  },
+} as esbuild.Plugin;
+
+const inlineDepsPlugin = {
+  name: "inlineDeps",
+  setup(build) {
+    const deps = [
+      "parsimmon",
+      "espresso-iisojs",
+      "codemirror",
+      "mithril",
+      "yaml",
+    ];
+    build.onResolve({ filter: /^[^.]/ }, async (args) => {
+      if (deps.some((d) => args.path.startsWith(d))) return undefined;
+      return { sideEffects: false, external: true };
+    });
+  },
+} as esbuild.Plugin;
+
 function generateSymbol(id: string, svgStr: string): string {
   const xml = xmlParser.parseXml(svgStr);
   const svg = xml.children[0];
@@ -130,41 +138,60 @@ function generateSymbol(id: string, svgStr: string): string {
   return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
 }
 
+async function getBuildMetadata(): Promise<string> {
+  const date = new Date().toISOString().slice(2, 10).replaceAll("-", "");
+
+  const [commit, diff, newFiles] = await Promise.all([
+    execAsync("git rev-parse HEAD"),
+    execAsync("git diff HEAD"),
+    execAsync("git ls-files --others --exclude-standard"),
+  ]).then((res) => res.map((r) => r.stdout.trim()));
+
+  if (!diff && !newFiles) return date + commit.slice(0, 4);
+
+  const hash = createHash("md5");
+  hash.update(commit).update(diff).update(newFiles);
+  for (const file of newFiles.split("\n").filter((f) => f))
+    hash.update(await fsAsync.readFile(file));
+  return date + hash.digest("hex").slice(0, 4);
+}
+
 async function init(): Promise<void> {
-  // Delete any old output directory
+  const [buildMetadata, packageJsonFile, npmShrinkwrapFile] = await Promise.all(
-  rmDirSync(OUTPUT_DIR);
+    [
-
+      getBuildMetadata(),
-  // Create output directory layout
+      fsAsync.readFile(path.join(INPUT_DIR, "package.json")),
-  fs.mkdirSync(OUTPUT_DIR);
+      fsAsync.readFile(path.join(INPUT_DIR, "npm-shrinkwrap.json")),
-  fs.mkdirSync(OUTPUT_DIR + "/bin");
+    ],
-  fs.mkdirSync(OUTPUT_DIR + "/public");
-  fs.mkdirSync(OUTPUT_DIR + "/tools");
-
-  // Create package.json
-  const packageJson = JSON.parse(
-    fs.readFileSync(path.resolve(INPUT_DIR, "package.json")).toString()
   );
+
+  const packageJson = JSON.parse(packageJsonFile.toString());
   delete packageJson["devDependencies"];
-  packageJson["scripts"] = {
+  delete packageJson["private"];
-    install: packageJson["scripts"].install,
+  delete packageJson["scripts"];
-    configure: packageJson["scripts"].configure,
+  packageJson["version"] = `${packageJson["version"]}+${buildMetadata}`;
-  };
-  packageJson["version"] = `${packageJson["version"]}+${BUILD_METADATA}`;
-  fs.writeFileSync(
-    path.resolve(OUTPUT_DIR, "package.json"),
-    JSON.stringify(packageJson, null, 2)
-  );
 
-  // Create npm-shrinkwrap.json
+  const npmShrinkwrap = JSON.parse(npmShrinkwrapFile.toString());
-  const npmShrinkwrapJson = JSON.parse(
+  npmShrinkwrap["version"] = packageJson["version"];
-    fs.readFileSync(path.resolve(INPUT_DIR, "npm-shrinkwrap.json")).toString()
+  stripDevDeps(npmShrinkwrap);
-  );
+  stripDevDeps2(npmShrinkwrap);
-  npmShrinkwrapJson["version"] = packageJson["version"];
+
-  stripDevDeps(npmShrinkwrapJson);
+  await rmDir(OUTPUT_DIR);
-  fs.writeFileSync(
+
-    path.resolve(OUTPUT_DIR, "npm-shrinkwrap.json"),
+  await fsAsync.mkdir(OUTPUT_DIR);
-    JSON.stringify(npmShrinkwrapJson, null, 2)
+
-  );
+  await Promise.all([
+    fsAsync.mkdir(path.join(OUTPUT_DIR, "bin")),
+    fsAsync.mkdir(path.join(OUTPUT_DIR, "public")),
+    fsAsync.writeFile(
+      path.join(OUTPUT_DIR, "package.json"),
+      JSON.stringify(packageJson, null, 2),
+    ),
+    fsAsync.writeFile(
+      path.join(OUTPUT_DIR, "npm-shrinkwrap.json"),
+      JSON.stringify(npmShrinkwrap, null, 2),
+    ),
+  ]);
 }
 
 async function copyStatic(): Promise<void> {
@@ -176,233 +203,157 @@ async function copyStatic(): Promise<void> {
     "public/favicon.png",
   ];
 
-  for (const file of files) {
+  const [logo, favicon] = await Promise.all([
-    fs.copyFileSync(
+    fsAsync.readFile(path.join(INPUT_DIR, "public/logo.svg")),
-      path.resolve(INPUT_DIR, file),
+    fsAsync.readFile(path.join(INPUT_DIR, "public/favicon.png")),
-      path.resolve(OUTPUT_DIR, file)
+  ]);
+
+  ASSETS.LOGO_SVG = `logo-${assetHash(logo)}.svg`;
+  ASSETS.FAVICON_PNG = `favicon-${assetHash(favicon)}.png`;
+
+  const filenames = {} as Record<string, string>;
+  filenames["public/logo.svg"] = path.join("public", ASSETS.LOGO_SVG);
+  filenames["public/favicon.png"] = path.join("public", ASSETS.FAVICON_PNG);
+
+  await Promise.all(
+    files.map((f) =>
+      fsAsync.copyFile(
+        path.join(INPUT_DIR, f),
+        path.join(OUTPUT_DIR, filenames[f] || f),
+      ),
+    ),
   );
-  }
 }
 
 async function generateCss(): Promise<void> {
-  const cssInPath = path.resolve(INPUT_DIR, "ui/css/app.css");
+  const res = await esbuild.build({
-  const cssOutPath = path.resolve(OUTPUT_DIR, "public/app.css");
+    bundle: true,
-  const cssIn = fs.readFileSync(cssInPath);
+    absWorkingDir: INPUT_DIR,
-  const cssOut = await postcss([
+    minify: MODE === "production",
-    postcssImport,
+    sourcemap: "linked",
-    postcssPresetEnv({
+    sourcesContent: false,
-      stage: 3,
+    entryPoints: ["ui/css/app.css"],
-      features: {
+    entryNames: "[dir]/[name]-[hash]",
-        "nesting-rules": true,
+    outfile: path.join(OUTPUT_DIR, "public/app.css"),
-        "color-mod-function": true,
+    target: ["chrome109", "safari15.6", "firefox115", "opera102", "edge118"],
-      },
+    metafile: true,
-    }),
-    cssnano,
-  ]).process(cssIn, { from: cssInPath, to: cssOutPath });
-  fs.writeFileSync(cssOutPath, cssOut.css);
-}
-
-async function generateToolsJs(): Promise<void> {
-  for (const bin of ["dump-data-model"]) {
-    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}.ts`);
-    const outputFile = path.resolve(OUTPUT_DIR, `tools/${bin}`);
-    const bundle = await rollup({
-      input: inputFile,
-      external: externals,
-      acorn: {
-        allowHashBang: true,
-      },
-      plugins: [
-        typescript({
-          tsconfig: "./tsconfig.json",
-          include: [`tools/${bin}.ts`, "lib/**/*.ts"],
-        }),
-        MODE === "production" ? terser() : null,
-      ],
   });
 
-    await bundle.write({
+  for (const [k, v] of Object.entries(res.metafile.outputs)) {
-      format: "cjs",
+    if (v.entryPoint === "ui/css/app.css") {
-      preferConst: true,
+      ASSETS.APP_CSS = path.relative(
-      sourcemap: "inline",
+        path.join(OUTPUT_DIR, "public"),
-      sourcemapExcludeSources: true,
+        path.join(INPUT_DIR, k),
-      banner: "#!/usr/bin/env node",
+      );
-      file: outputFile,
+      break;
-    });
+    }
-
-    // Mark as executable
-    const mode = fs.statSync(outputFile).mode;
-    fs.chmodSync(outputFile, mode | 73);
   }
 }
 
 async function generateBackendJs(): Promise<void> {
-  for (const bin of [
+  const services = [
     "genieacs-cwmp",
     "genieacs-ext",
     "genieacs-nbi",
     "genieacs-fs",
     "genieacs-ui",
-  ]) {
+  ];
-    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}.ts`);
-    const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
-    const bundle = await rollup({
-      input: inputFile,
-      external: externals,
-      acorn: {
-        allowHashBang: true,
-      },
-      treeshake: {
-        propertyReadSideEffects: false,
-        moduleSideEffects: false,
-      },
-      plugins: [
-        rollupJson({ preferConst: true }),
-        {
-          name: "",
-          resolveId: (importee, importer) => {
-            if (importee.endsWith("/package.json")) {
-              const p = path.resolve(path.dirname(importer), importee);
-              if (p === path.resolve(INPUT_DIR, "package.json"))
-                return path.resolve(OUTPUT_DIR, "package.json");
-            }
-            return null;
-          },
-        },
-        typescript({
-          tsconfig: "./tsconfig.json",
-          include: [`bin/${bin}.ts`, "lib/**/*.ts"],
-        }),
-        MODE === "production" ? terser() : null,
-      ],
-    });
 
-    await bundle.write({
+  await esbuild.build({
-      format: "cjs",
+    bundle: true,
-      preferConst: true,
+    absWorkingDir: INPUT_DIR,
+    minify: MODE === "production",
     sourcemap: "inline",
-      sourcemapExcludeSources: true,
+    sourcesContent: false,
-      banner: "#!/usr/bin/env node",
+    platform: "node",
-      file: outputFile,
+    target: "node12.13.0",
+    packages: "external",
+    banner: { js: "#!/usr/bin/env node" },
+    entryPoints: services.map((s) => `bin/${s}.ts`),
+    outdir: path.join(OUTPUT_DIR, "bin"),
+    plugins: [packageDotJsonPlugin, assetsPlugin],
   });
 
+  for (const bin of services) {
+    const p = path.join(OUTPUT_DIR, "bin", bin);
+    await fsAsync.rename(`${p}.js`, p);
     // Mark as executable
-    const mode = fs.statSync(outputFile).mode;
+    const mode = (await fsAsync.lstat(p)).mode;
-    fs.chmodSync(outputFile, mode | 73);
+    await fsAsync.chmod(p, mode | 73);
   }
 }
 
 async function generateFrontendJs(): Promise<void> {
-  const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
+  const res = await esbuild.build({
-  const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
+    bundle: true,
+    absWorkingDir: INPUT_DIR,
+    splitting: true,
+    minify: MODE === "production",
+    sourcemap: "linked",
+    sourcesContent: false,
+    platform: "browser",
+    format: "esm",
+    target: ["chrome109", "safari15.6", "firefox115", "opera102", "edge118"],
+    entryPoints: ["ui/app.ts"],
+    entryNames: "[dir]/[name]-[hash]",
+    outdir: path.join(OUTPUT_DIR, "public"),
+    plugins: [packageDotJsonPlugin, inlineDepsPlugin, assetsPlugin],
+    metafile: true,
+  });
 
-  const inlineDeps = ["mithril", "parsimmon", "jsbi", "espresso-iisojs"];
+  for (const [k, v] of Object.entries(res.metafile.outputs)) {
-  const bundle = await rollup({
+    for (const imp of v.imports)
-    input: inputFile,
+      if (imp.external) throw new Error(`External import found: ${imp.path}`);
-    external: externals.filter((e) => !inlineDeps.includes(e)),
+
-    plugins: [
+    if (v.entryPoint === "ui/app.ts") {
-      rollupJson({ preferConst: true }),
+      ASSETS.APP_JS = path.relative(
-      {
+        path.join(OUTPUT_DIR, "public"),
-        name: "",
+        path.join(INPUT_DIR, k),
-        resolveId: function (importee, importer) {
-          if (importee.endsWith("/bigint")) {
-            return this.resolve(importee + "-jsbi", importer);
-          } else if (importee === "espresso-iisojs") {
-            return this.resolve(
-              "espresso-iisojs/dist/espresso-iisojs-jsbi.mjs",
-              importer
       );
     }
-          return null;
+  }
-        },
-      },
-      typescript({ tsconfig: "./tsconfig.json" }),
-      nodeResolve(),
-      commonjs(),
-    ],
-    inlineDynamicImports: true,
-    treeshake: {
-      propertyReadSideEffects: false,
-      moduleSideEffects: false,
-    },
-    onwarn: ((warning, warn) => {
-      // Ignore circular dependency warnings
-      if (warning.code !== "CIRCULAR_DEPENDENCY") warn(warning);
-    }) as WarningHandler,
-  });
-
-  await bundle.write({
-    preferConst: true,
-    format: "es",
-    sourcemap: "inline",
-    sourcemapExcludeSources: true,
-    file: outputFile,
-  });
-
-  const webpackConf: Configuration = {
-    mode: MODE as "development" | "production",
-    entry: outputFile,
-    resolve: {
-      aliasFields: ["module"],
-    },
-    devtool: "nosources-source-map",
-    module: {
-      rules: [
-        {
-          test: /\.js$/,
-          use: ["source-map-loader"],
-          enforce: "pre",
-        },
-      ],
-    },
-    output: {
-      path: path.resolve(OUTPUT_DIR, "public"),
-      filename: "app.js",
-    },
-  };
-
-  return new Promise((resolve, reject) => {
-    webpack(webpackConf, (err, stats) => {
-      if (err) return reject(err);
-      process.stdout.write(stats.toString({ colors: true }) + "\n");
-      resolve();
-    });
-  });
 }
 
 async function generateIconsSprite(): Promise<void> {
-  const svgo = new SVGO({ plugins: [{ removeViewBox: false }] });
+  const symbols = [] as string[];
-  const symbols = [];
+  const iconsDir = path.join(INPUT_DIR, "ui/icons");
-  const iconsDir = path.resolve(INPUT_DIR, "ui/icons");
+  for (const file of await fsAsync.readdir(iconsDir)) {
-  for (const file of fs.readdirSync(iconsDir)) {
     const id = path.parse(file).name;
     const filePath = path.join(iconsDir, file);
-    const { data } = await svgo.optimize(fs.readFileSync(filePath).toString());
+    const src = (await fsAsync.readFile(filePath)).toString();
+    const { data } = await optimize(src, {
+      plugins: [
+        {
+          name: "preset-default",
+          params: {
+            overrides: {
+              removeViewBox: false,
+            },
+          },
+        },
+      ],
+    });
     symbols.push(generateSymbol(id, data));
   }
-  fs.writeFileSync(
+  const data = `<svg xmlns="http://www.w3.org/2000/svg">${symbols.join(
-    path.resolve(OUTPUT_DIR, "public/icons.svg"),
+    "",
-    `<svg xmlns="http://www.w3.org/2000/svg">${symbols.join("")}</svg>`
+  )}</svg>`;
+  ASSETS.ICONS_SVG = `icons-${assetHash(data)}.svg`;
+  await fsAsync.writeFile(
+    path.join(OUTPUT_DIR, "public", ASSETS.ICONS_SVG),
+    data,
   );
 }
 
 init()
-  .then(() => {
+  .then(() =>
     Promise.all([
-      copyStatic(),
+      Promise.all([generateIconsSprite(), copyStatic()]).then(
+        generateFrontendJs,
+      ),
       generateCss(),
-      generateIconsSprite(),
+    ]).then(generateBackendJs),
-      generateToolsJs(),
+  )
-      generateBackendJs(),
-      generateFrontendJs(),
-    ])
-      .then(() => {
-        // Ignore
-      })
-      .catch((err) => {
-        process.stderr.write(err.stack + "\n");
-      });
-  })
   .catch((err) => {
     process.stderr.write(err.stack + "\n");
   });

build/lint.ts

@@ -0,0 +1,55 @@
+import { exec } from "node:child_process";
+import { promisify } from "node:util";
+
+const execPromise = promisify(exec);
+
+async function runEslint(): Promise<string> {
+  const CMD =
+    "eslint 'bin/*.ts' 'lib/**/*.ts' 'ui/**/*.ts' 'test/**/*.ts' 'build/**/*.ts'";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  try {
+    const { stdout, stderr } = await execPromise(CMD, { env });
+    if (stderr) throw new Error(stderr);
+    return stdout;
+  } catch (err) {
+    if (err.killed || err.signal || err.stderr || err.code !== 1) throw err;
+    return err.stdout;
+  }
+}
+
+async function runTsc(): Promise<string> {
+  const CMD = "tsc --noEmit";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  const { stdout, stderr } = await execPromise(CMD, { env });
+  if (stderr) throw new Error(stderr);
+  return stdout;
+}
+
+async function runPrettier(): Promise<string> {
+  const CMD = "prettier --prose-wrap always --write .";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  const { stdout, stderr } = await execPromise(CMD, { env });
+  if (stderr) throw new Error(stderr);
+  return stdout;
+}
+
+async function runAll(): Promise<void> {
+  const prom1 = runPrettier();
+  const prom2 = runEslint();
+  const prom3 = runTsc();
+
+  console.log(await prom1);
+  console.log(await prom2);
+  console.log(await prom3);
+}
+
+runAll().catch(console.error);

build/spellcheck-dict.pws

@@ -0,0 +1,180 @@
+personal_ws-1.1 en 0
+genieacs
+GenieACS
+javascript
+sudo
+config
+cwmp
+CPE
+systemctl
+nbi
+CWMP
+NBI
+ACS
+SSL
+xsd
+args
+DeviceID
+GENIEACS
+env
+param
+arg
+AUTH
+igd
+TCP
+toctree
+auth
+EnvironmentFile
+JSON
+json
+parameterValues
+setParameterValues
+stderr
+stdout
+attr
+cfg
+CPEs
+ExecStart
+faq
+FileType
+GenieACS's
+JWT
+maxdepth
+npm
+objectName
+refreshObject
+SerialNumber
+systemd
+usr
+WantedBy
+yaml
+chown
+Config
+getParameterValues
+https
+InternetGatewayDevice
+mkdir
+OUI
+sql
+TODO
+UDP
+addObject
+AGPLv
+const
+cpe
+cron
+deleteObject
+FactoryReset
+factoryReset
+FileName
+fileType
+Github
+hostname
+HTTPS
+jwt
+latlong
+NPM
+oui
+parameterNames
+ProductClass
+productClass
+repo
+SSID
+sublicense
+TLS
+WANIPConnection
+WPA
+YourProvisionName
+abc
+Abdulla
+accessors
+AddObject
+AddressingType
+api
+APIs
+chmod
+cwmp's
+dateext
+DATETIME
+dateTime
+datetime
+declaratively
+delaycompress
+DeleteObject
+deviceId
+DHCP
+dir
+distro
+equisatisfiable
+ExternalIPAddress
+failover
+favor
+func
+GetParameterAttributes
+getPassword
+gte
+gui
+hexBinary
+Hils
+HOSTNAME
+init
+journalctl
+journald
+LastFileName
+LastFileType
+lastInform
+LIBXMLJS
+libxmljs
+literalinclude
+lockfile
+logrotate
+lte
+mipsbe
+normalizers
+pre
+quickstart
+README
+redis
+reimplemented
+RequestDownload
+RESTful
+rollout
+sandboxed
+scalable
+SetParameterAttributes
+SetParamteerValues
+SIGINT
+SIGTERM
+skipWritableCheck
+stringify
+subdirectory
+TargetFileName
+useradd
+VirtualParameters
+vparam
+xml
+YAML
+Zaid
+cacheExpire
+iss
+http
+statusCode
+rawData
+pos
+Brotli
+CSV
+hostInfo
+IPv
+downloadSuccessOnTimeout
+TransferComplete
+nginx
+ENCODEURICOMPONENT
+pageSize
+skipRootGpn
+GPN
+NaN
+CIDR
+XMPP
+PEM
+JID
+unindexed

build/spellcheck.sh

@@ -0,0 +1,12 @@
+#! /bin/sh
+
+cd "$(dirname "$0")"
+
+FILES=`ls ../docs/*.rst ../docs/*.js ../*.md`
+
+for FILE in $FILES
+do
+    echo $FILE
+    cat $FILE | aspell list --lang=en --add-extra-dicts=./spellcheck-dict.pws --ignore 2
+    echo
+done

docs/.readthedocs.yaml

@@ -0,0 +1,18 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+
+sphinx:
+  configuration: docs/conf.py
+
+formats: all
+
+python:
+  install:
+    - requirements: docs/requirements.txt

docs/api-reference.rst

@@ -1,7 +1,7 @@
 API Reference
 =============
 
-GenieACS expoeses a rich RESTful API through its NBI component. This document
+GenieACS exposes a rich RESTful API through its NBI component. This document
 serves as a reference for the available APIs.
 
 This API makes use of MongoDB's query language in some of its endpoints. Refer
@@ -16,7 +16,7 @@ to MongoDB's documentation for details.
 
 .. warning::
 
-  A common pitfll is not properly percent-encoding special characters in the
+  A common pitfall is not properly percent-encoding special characters in the
   device ID or query in the URL.
 
 Endpoints
@@ -41,11 +41,11 @@ Examples
 
 .. code:: javascript
 
-  query = {"_id": "202BC1-BM632w-0000000"}
+  query = {"_id": "202BC1-BM632w-000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D'
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D'
 
 - Find a device by its MAC address:
 
@@ -77,21 +77,21 @@ Examples
 
 .. code:: javascript
 
-  query = {"device": "202BC1-BM632w-0000000"}
+  query = {"device": "202BC1-BM632w-000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-0000000%22%7D'
+  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-000000%22%7D'
 
 - Return specific parameters for a given device:
 
 .. code:: javascript
 
-  query = {"_id": "202BC1-BM632w-0000000"}
+  query = {"_id": "202BC1-BM632w-000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
+  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
 
 The ``projection`` URL param is a comma-separated list of the parameters to receive.
 
@@ -106,7 +106,7 @@ if the tasks have been queued to be executed at the next inform.
 *device_id*: The ID of the device.
 
 *connection_request*: Indicates that a connection request will be triggered to
-execute the tasks immediatly. Otherwise, the tasks will be queued and be
+execute the tasks immediately. Otherwise, the tasks will be queued and be
 processed at the next inform.
 
 The response body is the task object as it is inserted in the database. The
@@ -120,7 +120,7 @@ Examples
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
   -X POST \
   --data '{"name": "refreshObject", "objectName": ""}'
 
@@ -138,7 +138,7 @@ Examples
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
   -X POST \
   --data '{"name":"setParameterValues", "parameterValues": [["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]]}'
 
@@ -183,7 +183,7 @@ Example
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/faults/202BC1-BM632w-0000000:default' -X DELETE
+  curl -i 'http://localhost:7557/faults/202BC1-BM632w-000000:default' -X DELETE
 
 DELETE /devices/\<device_id\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -218,7 +218,7 @@ Assign the tag "testing" to a device:
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X POST
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X POST
 
 DELETE /devices/\<device_id\>/tags/\<tag\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -236,7 +236,7 @@ Remove the tag "testing" from a device:
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X DELETE
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X DELETE
 
 PUT /presets/\<preset_name\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -299,7 +299,7 @@ The following file metadata may be sent as request headers:
 - ``fileType``: For firmware images it should be "1 Firmware Upgrade Image".
   Other common types are "2 Web Content" and "3 Vendor Configuration File".
 
-- ``oui``: The OUI of the device model that this file belogs to.
+- ``oui``: The OUI of the device model that this file belongs to.
 
 - ``productClass``: The product class of the device.
 
@@ -344,7 +344,7 @@ Find files using a query.
 Tasks
 -----
 
-Find the different availabe tasks and their object structure.
+Find the different available tasks and their object structure.
 
 ``getParameterValues``
 ~~~~~~~~~~~~~~~~~~~~~~

docs/conf.py

@@ -19,7 +19,7 @@ import json
 # -- Project information -----------------------------------------------------
 
 project = 'GenieACS Documentation'
-copyright = '2019, GenieACS Inc.'
+copyright = '2024, GenieACS Inc.'
 author = 'GenieACS Inc.'
 
 # The full version, including alpha/beta/rc tags
@@ -58,7 +58,7 @@ html_theme_options = {
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 master_doc = 'index'
 

docs/cpe-authentication.rst

@@ -3,4 +3,70 @@
 CPE Authentication
 ==================
 
+CPE to ACS
+----------
+
+.. note::
+
+  By default GenieACS will accept any incoming connection via HTTP/HTTPS and
+  respond to it.
+
+The following parameters are used to set and get (password is redacted but
+can be set) the username/password used to authenticate against the ACS:
+
+Username: ``Device.ManagementServer.Username`` or ``InternetGatewayDevice.ManagementServer.Username``
+
+Password: ``Device.ManagementServer.Password`` or ``InternetGatewayDevice.ManagementServer.Password``
+
+Enable CPE to ACS Authentication
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+CPE to ACS authentication can be configured in the web interface by using the
+`Config` option in the `Admin` tab.
+
+Go to the `Admin` -> `Config` page and click on `New config` button at the
+bottom of the page. This will open pop-up which requires you to fill in a key
+and value. The key should be ``cwmp.auth``. The value accepts a boolean.
+Setting the value to ``true`` makes it so that GenieACS accepts any incoming
+connection, setting it to ``false`` makes GenieACS deny all incoming
+connections. This can be further configured using the ``AUTH()`` and ``EXT()``
+functions.
+
+The ``AUTH()`` function
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``AUTH()`` function accepts two parameters, username and password. It
+checks the given username and password with the incoming request to determine
+whether to return true or false.
+
+Basic usage of the ``AUTH()`` function could be as follows:
+
+.. code:: sql
+
+   AUTH("fixed-username", "fixed-password")
+
+This will only accept incoming request who authenticate with
+"fixed-username" and "fixed-password".
+
+The various device parameters can be referenced from within the ``cwmp.auth``
+expression. For example:
+
+.. code:: sql
+
+   AUTH(Device.ManagementServer.Username, Device.ManagementServer.Password)
+
+The ``EXT()`` function
+~~~~~~~~~~~~~~~~~~~~~~
+
+The ``EXT()`` function makes it possible to call an :ref:`extension
+<extensions>` script from the auth expression. This can be used to fetch
+the credentials from an external source:
+
+.. code:: sql
+
+   AUTH(DeviceID.SerialNumber, EXT("authenticate", "getPassword", DeviceID.SerialNumber))
+
+ACS to CPE
+----------
+
 TODO

docs/environment-variables.rst

@@ -64,7 +64,7 @@ CWMP_PORT
 CWMP_INTERFACE
   The network interface that genieacs-cwmp binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 CWMP_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
@@ -86,7 +86,7 @@ CWMP_ACCESS_LOG_FILE
   File to log incoming requests for genieacs-cwmp. If omitted, logs will go to
   stdout.
 
-  Default: unest
+  Default: unset
 
 NBI_WORKER_PROCESSES
   The number of worker processes to spawn for genieacs-nbi. A value of 0 means
@@ -102,12 +102,12 @@ NBI_PORT
 NBI_INTERFACE
   The network interface that genieacs-nbi binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 NBI_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
 
-  Default: unest
+  Default: unset
 
 NBI_SSL_KEY
   Path to certificate key file. If omitted, non-secure HTTP will be used.
@@ -140,7 +140,7 @@ FS_PORT
 FS_INTERFACE
   The network interface that genieacs-fs binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 FS_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.
@@ -165,7 +165,7 @@ FS_ACCESS_LOG_FILE
   Default: unset
 
 FS_URL_PREFIX
-  The URL prefix (e.g. 'https://example.com:7657/') to use when generating the
+  The URL prefix (e.g. 'https://example.com:7567/') to use when generating the
   file URL for TR-069 Download requests. Set this if genieacs-fs and
   genieacs-cwmp are behind a proxy or running on different servers.
 
@@ -186,7 +186,7 @@ UI_PORT
 UI_INTERFACE
   The network interface that genieacs-ui binds to.
 
-  Default: ``0.0.0.0``
+  Default: ``::``
 
 UI_SSL_CERT
   Path to certificate file. If omitted, non-secure HTTP will be used.

docs/ext-sample.js

@@ -1,4 +1,4 @@
-// This is an exmaple GenieACS extension to get the current latitude/longitude
+// This is an example GenieACS extension to get the current latitude/longitude
 // of the International Space Station. Why, you ask? Because why not.
 // To install, copy this file to config/ext/iss.js.
 
@@ -10,15 +10,17 @@ let cache = null;
 let cacheExpire = 0;
 
 function latlong(args, callback) {
-  if (Date.now() < cacheExpire)
+  if (Date.now() < cacheExpire) return callback(null, cache);
-    return callback(null, cache);
 
-  http.get("http://api.open-notify.org/iss-now.json", (res) => {
+  http
+    .get("http://api.open-notify.org/iss-now.json", (res) => {
       if (res.statusCode !== 200)
-      return callback(new Error(`Request failed (status code: ${res.statusCode})`));
+        return callback(
+          new Error(`Request failed (status code: ${res.statusCode})`),
+        );
 
       let rawData = "";
-    res.on("data", (chunk) => rawData += chunk);
+      res.on("data", (chunk) => (rawData += chunk));
 
       res.on("end", () => {
         let pos = JSON.parse(rawData)["iss_position"];
@@ -26,7 +28,8 @@ function latlong(args, callback) {
         cacheExpire = Date.now() + 10000;
         callback(null, cache);
       });
-  }).on("error", (err) => {
+    })
+    .on("error", (err) => {
       callback(err);
     });
 }

docs/installation-guide.rst

@@ -4,7 +4,7 @@ Installation Guide
 This guide is for installing GenieACS on a single server on any Linux distro
 that uses *systemd* as its init system.
 
-The various GenieACS services are independant of each other and may be
+The various GenieACS services are independent of each other and may be
 installed on different servers. You may also run multiple instances of each in
 a load-balancing/failover setup.
 
@@ -19,7 +19,7 @@ Prerequisites
 
 .. topic:: Node.js
 
-  GenieACS requires Node.js 10.x and up. Refer to https://nodejs.org/ for
+  GenieACS requires Node.js 12.13 and up. Refer to https://nodejs.org/ for
   instructions.
 
 .. topic:: MongoDB
@@ -34,7 +34,7 @@ Install GenieACS
 
   .. parsed-literal::
 
-    sudo npm install -g --unsafe-perm genieacs@\ |release|
+    sudo npm install -g genieacs@\ |release|
 
 .. topic:: Installing from source
 
@@ -73,8 +73,14 @@ Configure systemd
     GENIEACS_FS_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-fs-access.log
     GENIEACS_UI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-ui-access.log
     GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
+    NODE_OPTIONS=--enable-source-maps
     GENIEACS_EXT_DIR=/opt/genieacs/ext
-    GENIEACS_UI_JWT_SECRET=secret
+
+  Generate a secure JWT secret and append to :file:`/opt/genieacs/genieacs.env`:
+
+  .. code:: bash
+
+    node -e "console.log(\"GENIEACS_UI_JWT_SECRET=\" + require('crypto').randomBytes(128).toString('hex'))" >> /opt/genieacs/genieacs.env
   
   Set file ownership and permissions:
 

docs/requirements.txt

@@ -0,0 +1 @@
+sphinx_rtd_theme==2.0.0

lib/api-functions.ts

@@ -1,50 +1,61 @@
-/**
+import { ObjectId } from "mongodb";
- * Copyright 2013-2019  GenieACS Inc.
+import { collections } from "./db/db.ts";
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { parse } from "url";
-import * as db from "./db";
-import * as common from "./common";
-import * as cache from "./cache";
 import {
-  getCurrentSnapshot,
+  deleteConfig,
+  deleteFault as dbDeleteFault,
+  deleteFile,
+  deletePermission,
+  deletePreset,
+  deleteProvision,
+  deleteTask,
+  deleteUser,
+  deleteVirtualParameter,
+  putConfig,
+  putPermission,
+  putPreset,
+  putProvision,
+  putUser,
+  putVirtualParameter,
+} from "./ui/db.ts";
+import * as common from "./util.ts";
+import * as cache from "./cache.ts";
+import { acquireLock, getToken, releaseLock } from "./lock.ts";
+import {
+  getRevision,
   getConfig,
   getConfigExpression,
-} from "./local-cache";
+  getUsers,
+} from "./ui/local-cache.ts";
 import {
   httpConnectionRequest,
   udpConnectionRequest,
-} from "./connection-request";
+  xmppConnectionRequest,
-import { Expression, Task } from "./types";
+} from "./connection-request.ts";
-import { flattenDevice } from "./mongodb-functions";
+import { Expression, Task } from "./types.ts";
-import { evaluate } from "./common/expression";
+import { evaluate } from "./common/expression/util.ts";
+import { hashPassword } from "./auth.ts";
+import { flattenDevice } from "./ui/db.ts";
+import { ResourceLockedError } from "./common/errors.ts";
+import * as config from "../lib/config.ts";
+
+const XMPP_CONFIGURED = !!config.get("XMPP_JID");
 
 export async function connectionRequest(
   deviceId: string,
-  device?: Record<string, { value?: [boolean | number | string, string] }>
+  device?: Record<string, { value?: [boolean | number | string, string] }>,
 ): Promise<string> {
   if (!device) {
-    const res = await db.devicesCollection.findOne({ _id: deviceId });
+    const res = await collections.devices.findOne({ _id: deviceId });
     if (!res) throw new Error("No such device");
     device = flattenDevice(res);
   }
 
-  let connectionRequestUrl, udpConnectionRequestAddress, username, password;
+  let connectionRequestUrl,
+    udpConnectionRequestAddress,
+    stunEnable,
+    connReqJabberId,
+    username,
+    password;
 
   if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
     connectionRequestUrl = (device[
@@ -55,6 +66,12 @@ export async function connectionRequest(
         "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
       ] || {}
     ).value || [""])[0];
+    stunEnable = ((
+      device["InternetGatewayDevice.ManagementServer.STUNEnable"] || {}
+    ).value || [""])[0];
+    connReqJabberId = ((
+      device["InternetGatewayDevice.ManagementServer.ConnReqJabberID"] || {}
+    ).value || [""])[0];
     username = ((
       device[
         "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
@@ -72,6 +89,10 @@ export async function connectionRequest(
     udpConnectionRequestAddress = ((
       device["Device.ManagementServer.UDPConnectionRequestAddress"] || {}
     ).value || [""])[0];
+    stunEnable = ((device["Device.ManagementServer.STUNEnable"] || {})
+      .value || [""])[0];
+    connReqJabberId = ((device["Device.ManagementServer.ConnReqJabberID"] || {})
+      .value || [""])[0];
     username = ((
       device["Device.ManagementServer.ConnectionRequestUsername"] || {}
     ).value || [""])[0];
@@ -79,8 +100,12 @@ export async function connectionRequest(
       device["Device.ManagementServer.ConnectionRequestPassword"] || {}
     ).value || [""])[0];
   }
-
+  let remoteAddress;
-  const remoteAddress = parse(connectionRequestUrl).host;
+  try {
+    remoteAddress = new URL(connectionRequestUrl).hostname;
+  } catch (err) {
+    return "Invalid connection request URL";
+  }
 
   const evalCallback = (exp): Expression => {
     if (!Array.isArray(exp)) return exp;
@@ -95,7 +120,7 @@ export async function connectionRequest(
       else if (name === "password") return password;
 
       const p = device[name];
-      if (p && p.value) return p.value[0];
+      if (p?.value) return p.value[0];
     } else if (exp[0] === "FUNC") {
       if (exp[1] === "REMOTE_ADDRESS") return remoteAddress;
       else if (exp[1] === "USERNAME") return username;
@@ -104,32 +129,32 @@ export async function connectionRequest(
     return exp;
   };
 
-  const snapshot = await getCurrentSnapshot();
+  const snapshot = await getRevision();
   const now = Date.now();
   const UDP_CONNECTION_REQUEST_PORT = +getConfig(
     snapshot,
     "cwmp.udpConnectionRequestPort",
     {},
     now,
-    evalCallback
+    evalCallback,
   );
   const CONNECTION_REQUEST_TIMEOUT = +getConfig(
     snapshot,
     "cwmp.connectionRequestTimeout",
     {},
     now,
-    evalCallback
+    evalCallback,
   );
   const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = !!getConfig(
     snapshot,
     "cwmp.connectionRequestAllowBasicAuth",
     {},
     now,
-    evalCallback
+    evalCallback,
   );
   let authExp: Expression = getConfigExpression(
     snapshot,
-    "cwmp.connectionRequestAuth"
+    "cwmp.connectionRequestAuth",
   );
 
   if (authExp === undefined) {
@@ -145,48 +170,66 @@ export async function connectionRequest(
 
   const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
 
-  let udpProm;
+  let udpProm = Promise.resolve(false);
-  if (udpConnectionRequestAddress) {
+  if (udpConnectionRequestAddress && +stunEnable) {
+    try {
+      const u = new URL("udp://" + udpConnectionRequestAddress);
       udpProm = udpConnectionRequest(
-      udpConnectionRequestAddress,
+        u.hostname,
+        parseInt(u.port || "80"),
         authExp,
         UDP_CONNECTION_REQUEST_PORT,
         debug,
-      deviceId
+        deviceId,
+      ).then(
+        () => true,
+        () => false,
       );
+    } catch (err) {
+      // Ignore invalid address
+    }
   }
 
-  const status = await httpConnectionRequest(
+  let status;
+
+  if (connReqJabberId && XMPP_CONFIGURED) {
+    status = await xmppConnectionRequest(
+      connReqJabberId,
+      authExp,
+      CONNECTION_REQUEST_TIMEOUT,
+      debug,
+      deviceId,
+    );
+  } else {
+    status = await httpConnectionRequest(
       connectionRequestUrl,
       authExp,
       CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
       CONNECTION_REQUEST_TIMEOUT,
       debug,
-    deviceId
+      deviceId,
     );
-
-  if (udpProm) {
-    await udpProm;
-    return "";
   }
 
+  if (await udpProm) return "";
+
   return status;
 }
 
 export async function awaitSessionStart(
   deviceId: string,
   lastInform: number,
-  timeout: number
+  timeout: number,
 ): Promise<boolean> {
   const now = Date.now();
-  const device = await db.devicesCollection.findOne(
+  const device = await collections.devices.findOne(
     { _id: deviceId },
-    { projection: { _lastInform: 1 } }
+    { projection: { _lastInform: 1 } },
   );
   const li = (device["_lastInform"] as Date).getTime();
   if (li > lastInform) return true;
-  const token = await cache.get(`cwmp_session_${deviceId}`);
+  const token = await getToken(`cwmp_session_${deviceId}`);
-  if (token) return true;
+  if (token?.startsWith("cwmp_session_")) return true;
   if (timeout < 500) return false;
   await new Promise((resolve) => setTimeout(resolve, 500));
   timeout -= Date.now() - now;
@@ -195,11 +238,11 @@ export async function awaitSessionStart(
 
 export async function awaitSessionEnd(
   deviceId: string,
-  timeout: number
+  timeout: number,
 ): Promise<boolean> {
   const now = Date.now();
-  const token = await cache.get(`cwmp_session_${deviceId}`);
+  const token = await getToken(`cwmp_session_${deviceId}`);
-  if (!token) return true;
+  if (!token?.startsWith("cwmp_session_")) return true;
   if (timeout < 500) return false;
   await new Promise((resolve) => setTimeout(resolve, 500));
   timeout -= Date.now() - now;
@@ -290,12 +333,21 @@ function sanitizeTask(task): void {
         !task.provisions.every((arr) =>
           arr.every(
             (s) =>
-              s == null || ["boolean", "number", "string"].includes(typeof s)
+              s == null || ["boolean", "number", "string"].includes(typeof s),
-          )
+          ),
         )
       )
         throw new Error("Invalid 'provisions' property");
       break;
+
+    case "reboot":
+      break;
+
+    case "factoryReset":
+      break;
+
+    default:
+      throw new Error("Invalid task name");
   }
 
   return task;
@@ -303,35 +355,145 @@ function sanitizeTask(task): void {
 
 export async function insertTasks(tasks: any[]): Promise<Task[]> {
   if (tasks && !Array.isArray(tasks)) tasks = [tasks];
-  else if (!tasks || tasks.length === 0) return tasks || [];
+  else if (!tasks?.length) return tasks || [];
 
   for (const task of tasks) {
     sanitizeTask(task);
     if (task.uniqueKey) {
-      await db.tasksCollection.deleteOne({
+      await collections.tasks.deleteOne({
         device: task.device,
         uniqueKey: task.uniqueKey,
       });
     }
   }
-  await db.tasksCollection.insertMany(tasks);
+  await collections.tasks.insertMany(tasks);
+  for (const task of tasks) task._id = task._id.toString();
   return tasks;
 }
 
 export async function deleteDevice(deviceId: string): Promise<void> {
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+  if (!token) throw new ResourceLockedError("Device is in session");
+  try {
     await Promise.all([
-    db.tasksCollection.deleteMany({ device: deviceId }),
+      collections.tasks.deleteMany({ device: deviceId }),
-    db.devicesCollection.deleteOne({ _id: deviceId }),
+      collections.devices.deleteOne({ _id: deviceId }),
-    db.faultsCollection.deleteMany({
+      collections.faults.deleteMany({
         _id: {
           $regex: `^${common.escapeRegExp(deviceId)}\\:`,
         },
       }),
-    db.operationsCollection.deleteMany({
+      collections.operations.deleteMany({
         _id: {
           $regex: `^${common.escapeRegExp(deviceId)}\\:`,
         },
       }),
-    cache.del(`${deviceId}_tasks_faults_operations`),
     ]);
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
+}
+
+export async function deleteFault(id: string): Promise<void> {
+  const deviceId = id.split(":", 1)[0];
+  const channel = id.slice(deviceId.length + 1);
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+  if (!token) throw new ResourceLockedError("Device is in session");
+  try {
+    const proms = [dbDeleteFault(id)];
+    if (channel.startsWith("task_"))
+      proms.push(deleteTask(new ObjectId(channel.slice(5))));
+    await Promise.all(proms);
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
+}
+
+export async function deleteResource(
+  resource: string,
+  id: string,
+): Promise<void> {
+  if (resource === "devices") {
+    await deleteDevice(id);
+  } else if (resource === "files") {
+    await deleteFile(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "faults") {
+    await deleteFault(id);
+  } else if (resource === "provisions") {
+    await deleteProvision(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "presets") {
+    await deletePreset(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "virtualParameters") {
+    await deleteVirtualParameter(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "config") {
+    await deleteConfig(id);
+    await Promise.all([
+      cache.del("ui-local-cache-hash"),
+      cache.del("cwmp-local-cache-hash"),
+    ]);
+  } else if (resource === "permissions") {
+    await deletePermission(id);
+    await cache.del("ui-local-cache-hash");
+  } else if (resource === "users") {
+    await deleteUser(id);
+    await cache.del("ui-local-cache-hash");
+  } else {
+    throw new Error(`Unknown resource ${resource}`);
+  }
+}
+
+// TODO Implement validation
+export async function putResource(
+  resource: string,
+  id: string,
+  data: any,
+): Promise<void> {
+  if (resource === "presets") {
+    await putPreset(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "provisions") {
+    await putProvision(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "virtualParameters") {
+    await putVirtualParameter(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "config") {
+    await putConfig(id, data);
+    await Promise.all([
+      cache.del("ui-local-cache-hash"),
+      cache.del("cwmp-local-cache-hash"),
+    ]);
+  } else if (resource === "permissions") {
+    await putPermission(id, data);
+    await cache.del("ui-local-cache-hash");
+  } else if (resource === "users") {
+    delete data["password"];
+    delete data["salt"];
+    await putUser(id, data);
+    await cache.del("ui-local-cache-hash");
+  } else {
+    throw new Error(`Unknown resource ${resource}`);
+  }
+}
+
+export function authLocal(
+  snapshot: string,
+  username: string,
+  password: string,
+): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    const users = getUsers(snapshot);
+    const user = users[username];
+    if (!user?.password) return void resolve(null);
+    hashPassword(password, user.salt)
+      .then((hash) => {
+        if (hash === user.password) resolve(true);
+        else resolve(false);
+      })
+      .catch(reject);
+  });
 }

lib/auth.ts

@@ -1,23 +1,4 @@
-/**
+import { createHash, randomBytes, pbkdf2 } from "node:crypto";
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { createHash, randomBytes, pbkdf2 } from "crypto";
 
 function parseHeaderFeilds(str: string): Record<string, string> {
   const res = {};
@@ -52,9 +33,9 @@ function parseHeaderFeilds(str: string): Record<string, string> {
   return res;
 }
 
-export function parseAuthorizationHeader(
+export function parseAuthorizationHeader(authHeader: string): {
-  authHeader: string
+  method: string;
-): { method: string } {
+} {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };
@@ -63,7 +44,7 @@ export function parseAuthorizationHeader(
     // Inspired by https://github.com/jshttp/basic-auth
     const USER_PASS_REGEX = /^([^:]*):(.*)$/;
     const creds = USER_PASS_REGEX.exec(
-      Buffer.from(authHeader.slice(method.length + 1), "base64").toString()
+      Buffer.from(authHeader.slice(method.length + 1), "base64").toString(),
     );
 
     if (!creds) throw new Error("Unable to parse auth header");
@@ -77,7 +58,7 @@ export function parseAuthorizationHeader(
 }
 
 export function parseWwwAuthenticateHeader(
-  authHeader: string
+  authHeader: string,
 ): Record<string, string> {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
@@ -100,7 +81,7 @@ export function digest(
   qop?: string | Buffer,
   body?: string | Buffer,
   cnonce?: string | Buffer,
-  nc?: string | Buffer
+  nc?: string | Buffer,
 ): string {
   const ha1 = createHash("md5");
   ha1.update(username).update(":").update(realm).update(":").update(password);
@@ -141,7 +122,7 @@ export function solveDigest(
   uri: string | Buffer,
   httpMethod: string | Buffer,
   body: string | Buffer,
-  authHeader: Record<string, string>
+  authHeader: Record<string, string>,
 ): string {
   const cnonce = randomBytes(8).toString("hex");
   const nc = "00000001";
@@ -163,7 +144,7 @@ export function solveDigest(
     qop,
     body,
     cnonce,
-    nc
+    nc,
   );
 
   let authString = `Digest username="${username}"`;

lib/cache.ts

@@ -1,118 +1,35 @@
-/**
+import { collections } from "./db/db.ts";
- * Copyright 2013-2019  GenieACS Inc.
+import * as config from "./config.ts";
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { Collection } from "mongodb";
-import { onConnect } from "./db";
-import * as config from "./config";
 
 const CLOCK_SKEW_TOLERANCE = 30000;
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let cacheCollection: Collection;
-
-onConnect(async (db) => {
-  cacheCollection = db.collection("cache");
-  await cacheCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
-});
-
 export async function get(key: string): Promise<string> {
-  const res = await cacheCollection.findOne({ _id: key });
+  const res = await collections.cache.findOne({ _id: key });
-  if (res) return res["value"];
+  return res?.value;
-  return null;
 }
 
 export async function del(key: string): Promise<void> {
-  await cacheCollection.deleteOne({ _id: key });
+  await collections.cache.deleteOne({ _id: key });
 }
 
 export async function set(
   key: string,
   value: string,
-  ttl: number = MAX_CACHE_TTL
+  ttl: number = MAX_CACHE_TTL,
 ): Promise<void> {
   const timestamp = new Date();
   const expire = new Date(
-    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000
+    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000,
   );
-  await cacheCollection.replaceOne(
+  await collections.cache.replaceOne(
     { _id: key },
-    { _id: key, value, expire, timestamp },
+    { value, expire, timestamp },
-    { upsert: true }
+    { upsert: true },
   );
 }
 
-export async function pop(key: string): Promise<any> {
+export async function pop(key: string): Promise<string> {
-  const res = await cacheCollection.findOneAndDelete({ _id: key });
+  const res = await collections.cache.findOneAndDelete({ _id: key });
-  if (res && res["value"]) return res["value"]["value"];
+  return res.value?.value;
-  return null;
-}
-
-export async function acquireLock(
-  lockName: string,
-  ttl: number,
-  timeout = 0,
-  token = Math.random().toString(36).slice(2)
-): Promise<string> {
-  try {
-    const now = Date.now();
-    const r = await cacheCollection.findOneAndUpdate(
-      { _id: lockName, token },
-      {
-        $set: {
-          value: token,
-          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
-        },
-        $currentDate: { timestamp: true },
-      },
-      { upsert: true, returnOriginal: false }
-    );
-    const v = r.value;
-    if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
-      throw new Error("Database clock skew too great");
-  } catch (err) {
-    if (err.code === 11000) {
-      if (timeout > 0) {
-        return new Promise((resolve, reject) => {
-          const w = 50 + Math.random() * 50;
-          setTimeout(() => {
-            acquireLock(lockName, ttl, timeout - w, token).then(
-              resolve,
-              reject
-            );
-          }, w);
-        });
-      }
-      throw new Error("Failed to acquire lock");
-    }
-    throw err;
-  }
-
-  return token;
-}
-
-export async function releaseLock(
-  lockName: string,
-  token: string
-): Promise<void> {
-  const res = await cacheCollection.deleteOne({
-    _id: lockName,
-    value: token,
-  });
-  if (res["result"]["n"] !== 1) throw new Error("Lock expired");
 }

lib/cluster.ts

@@ -1,30 +1,11 @@
-/**
+import cluster, { Worker } from "node:cluster";
- * Copyright 2013-2019  GenieACS Inc.
+import { cpus } from "node:os";
- *
+import * as logger from "./logger.ts";
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import cluster from "cluster";
-import { cpus } from "os";
-import * as logger from "./logger";
 
 let respawnTimestamp = 0;
 let crashes: number[] = [];
 
-function fork(): cluster.Worker {
+function fork(): Worker {
   const w = cluster.fork();
   w.on("error", (err: NodeJS.ErrnoException) => {
     // Avoid exception when attempting to kill the process just as it's exiting
@@ -92,7 +73,7 @@ function restartWorker(worker, code, signal): void {
 export function start(
   workerCount: number,
   servicePort: number,
-  serviceAddress: string
+  serviceAddress: string,
 ): void {
   cluster.on("listening", (worker, address) => {
     if (

lib/common/authorizer.ts

@@ -1,33 +1,14 @@
-/**
+import { PermissionSet, Expression } from "../types.ts";
- * Copyright 2013-2019  GenieACS Inc.
+import { evaluate, or } from "./expression/util.ts";
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { PermissionSet, Expression } from "../types";
-import { evaluate, or } from "./expression";
 
 export default class Authorizer {
-  private permissionSets: PermissionSet[];
+  private declare permissionSets: PermissionSet[];
-  private validatorCache: WeakMap<
+  private declare validatorCache: WeakMap<
     any,
     (mutationType, mutation, any) => boolean
   >;
-  private hasAccessCache: Map<string, boolean>;
+  private declare hasAccessCache: Map<string, boolean>;
-  private getFilterCache: Map<string, Expression>;
+  private declare getFilterCache: Map<string, Expression>;
 
   public constructor(permissionSets: PermissionSet[]) {
     this.permissionSets = permissionSets;
@@ -78,7 +59,7 @@ export default class Authorizer {
 
   public getValidator(
     resourceType: string,
-    resource: unknown
+    resource: unknown,
   ): (mutationType: string, mutation?: any, args?: any) => boolean {
     if (this.validatorCache.has(resource))
       return this.validatorCache.get(resource);
@@ -99,7 +80,7 @@ export default class Authorizer {
     const validator = (
       mutationType: string,
       mutation: any,
-      any: any
+      any: any,
     ): boolean => {
       if (!validators.length) return false;
 
@@ -118,10 +99,9 @@ export default class Authorizer {
         if (["mutation", "options"].includes(entry)) {
           value = object[entry];
           for (const seg of paramName.split(".")) {
-            // typeof null is "object"
-            if (value != null && typeof value !== "object") value = null;
-            else value = value[seg];
             if (value == null) break;
+            if (typeof value !== "object") value = null;
+            else value = value[seg];
           }
         } else if (object[entry]) {
           if (paramName) value = object[entry][paramName];
@@ -134,7 +114,7 @@ export default class Authorizer {
       const res = evaluate(
         validators.length > 1 ? ["OR", validators] : validators[0],
         valueFunction,
-        Date.now()
+        Date.now(),
       );
       return !Array.isArray(res) && !!res;
     };

lib/common/bigint-jsbi.ts

@@ -1,108 +0,0 @@
-/**
- * Copyright 2013-2020  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import JSBI from "jsbi";
-
-export function and(a: JSBI, b: JSBI): JSBI {
-  return JSBI.bitwiseAnd(a, b);
-}
-
-export function or(a: JSBI, b: JSBI): JSBI {
-  return JSBI.bitwiseOr(a, b);
-}
-
-export function xor(a: JSBI, b: JSBI): JSBI {
-  return JSBI.bitwiseXor(a, b);
-}
-
-export function not(a: JSBI): JSBI {
-  return JSBI.bitwiseNot(a);
-}
-
-export function lshift(a: JSBI, b: JSBI): JSBI {
-  return JSBI.leftShift(a, b);
-}
-
-export function rshift(a: JSBI, b: JSBI): JSBI {
-  return JSBI.signedRightShift(a, b);
-}
-
-export function add(a: JSBI, b: JSBI): JSBI {
-  return JSBI.add(a, b);
-}
-
-export function sub(a: JSBI, b: JSBI): JSBI {
-  return JSBI.subtract(a, b);
-}
-
-export function mul(a: JSBI, b: JSBI): JSBI {
-  return JSBI.multiply(a, b);
-}
-
-export function div(a: JSBI, b: JSBI): JSBI {
-  return JSBI.divide(a, b);
-}
-
-export function exp(a: JSBI, b: JSBI): JSBI {
-  return JSBI.exponentiate(a, b);
-}
-
-export function rem(a: JSBI, b: JSBI): JSBI {
-  return JSBI.remainder(a, b);
-}
-
-export function toNumber(a: JSBI): number {
-  return JSBI.toNumber(a);
-}
-
-export function eq(a: JSBI, b: JSBI): boolean {
-  return JSBI.equal(a, b);
-}
-
-export function ne(a: JSBI, b: JSBI): boolean {
-  return JSBI.notEqual(a, b);
-}
-
-export function lt(a: JSBI, b: JSBI): boolean {
-  return JSBI.lessThan(a, b);
-}
-
-export function lte(a: JSBI, b: JSBI): boolean {
-  return JSBI.lessThanOrEqual(a, b);
-}
-
-export function gt(a: JSBI, b: JSBI): boolean {
-  return JSBI.greaterThan(a, b);
-}
-
-export function gte(a: JSBI, b: JSBI): boolean {
-  return JSBI.greaterThanOrEqual(a, b);
-}
-
-export function asUintN(a: number, b: JSBI): JSBI {
-  return JSBI.asUintN(a, b);
-}
-
-export function asIntN(a: number, b: JSBI): JSBI {
-  return JSBI.asIntN(a, b);
-}
-
-const _BigInt = JSBI.BigInt;
-type _bigint = JSBI;
-export { _BigInt as BigInt, _bigint as bigint };

lib/common/debounce.ts

@@ -0,0 +1,16 @@
+export default function debounce<T>(
+  func: (args: T[]) => void,
+  timeout: number,
+): (arg: T) => void {
+  let timer: ReturnType<typeof setTimeout>;
+  let args: T[] = [];
+  return (arg: T) => {
+    args.push(arg);
+    clearTimeout(timer);
+    timer = setTimeout(() => {
+      const argscopy = args;
+      args = [];
+      func(argscopy);
+    }, timeout);
+  };
+}

lib/common/errors.ts

@@ -0,0 +1,6 @@
+export class ResourceLockedError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ResourceLockedError";
+  }
+}

lib/common/expression/bigint.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2020  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 export function and(a: bigint, b: bigint): bigint {
   return a & b;
 }
@@ -103,4 +84,4 @@ export function asIntN(a: number, b: bigint): bigint {
 
 const _BigInt = BigInt;
 type _bigint = bigint;
-export { _BigInt as BigInt, _bigint as bigint };
+export { _BigInt as BigInt, type _bigint as bigint };

lib/common/expression/normalize.ts

@@ -0,0 +1,492 @@
+import * as BI from "./bigint.ts";
+import { Expression } from "../../types.ts";
+import { map } from "./parser.ts";
+import { and, evaluateCallback } from "./util.ts";
+
+const ZERO = BI.BigInt(0);
+const ONE = BI.BigInt(1);
+const TWO = BI.BigInt(2);
+const NEGATIVE_ONE = BI.BigInt(-1);
+
+class Indeterminates {
+  public declare map: Map<string, number>;
+  public declare sortedKeys: string[];
+
+  public constructor(str?: string) {
+    this.map = new Map();
+    if (str) {
+      this.map.set(str, 1);
+      this.sortedKeys = [str];
+    } else {
+      this.sortedKeys = [];
+    }
+  }
+
+  public reciprocal(): Indeterminates {
+    const res = new Indeterminates();
+    res.sortedKeys = this.sortedKeys;
+    res.map = new Map();
+    for (const [k, v] of this.map) res.map.set(k, 0 - v);
+    return res;
+  }
+
+  public static multiply(
+    indeterminates1: Indeterminates,
+    indeterminates2: Indeterminates,
+  ): Indeterminates {
+    const res = new Indeterminates();
+    res.sortedKeys = indeterminates1.sortedKeys.slice();
+    res.map = new Map(indeterminates1.map);
+
+    for (const [key, val] of indeterminates2.map) {
+      const v = res.map.get(key);
+      if (!v) {
+        res.map.set(key, val);
+        res.sortedKeys.push(key);
+      } else {
+        const v2 = val + v;
+        if (!v2) {
+          res.map.delete(key);
+          res.sortedKeys = res.sortedKeys.filter((s) => s !== key);
+        } else {
+          res.map.set(key, v2);
+        }
+      }
+    }
+
+    res.sortedKeys.sort((a, b) => {
+      if (a.length !== b.length) return b.length - a.length;
+      else if (a > b) return 1;
+      else if (a < b) return -1;
+      return 0;
+    });
+
+    return res;
+  }
+
+  public static compare(a: Indeterminates, b: Indeterminates): number {
+    if (a.sortedKeys.length !== b.sortedKeys.length)
+      return b.sortedKeys.length - a.sortedKeys.length;
+    for (let i = 0; i < a.sortedKeys.length; ++i) {
+      const k1 = a.sortedKeys[i];
+      const w1 = a.map.get(k1);
+      const k2 = b.sortedKeys[i];
+      const w2 = b.map.get(k2);
+      if (w1 !== w2) return w2 - w1;
+      if (k1.length > k2.length) return -1;
+      else if (k1.length < k2.length) return 1;
+      else if (k1 > k2) return 1;
+      else if (k1 < k2) return -1;
+    }
+    return 0;
+  }
+}
+
+interface Term {
+  indeterminates: Indeterminates;
+  coefficientNumerator: BI.bigint;
+  coefficientDenominator: BI.bigint;
+}
+
+function findGcd(a: BI.bigint, b: BI.bigint): BI.bigint {
+  while (BI.ne(b, ZERO)) {
+    const t = b;
+    b = BI.rem(a, b);
+    a = t;
+  }
+  return a;
+}
+
+class Polynomial {
+  public declare terms: Term[];
+
+  public constructor(terms: Term[]) {
+    this.terms = terms;
+  }
+
+  public static simplifyTerms(terms: Term[]): Term[] {
+    const ts = terms
+      .slice()
+      .sort((a: Term, b: Term) =>
+        Indeterminates.compare(a.indeterminates, b.indeterminates),
+      );
+
+    for (let i = 1; i < ts.length; ++i) {
+      const t1 = ts[i - 1];
+      const t2 = ts[i];
+      if (Indeterminates.compare(t1.indeterminates, t2.indeterminates) === 0) {
+        const numerator = BI.add(
+          BI.mul(t1.coefficientNumerator, t2.coefficientDenominator),
+          BI.mul(t2.coefficientNumerator, t1.coefficientDenominator),
+        );
+
+        const denominator = BI.mul(
+          t1.coefficientDenominator,
+          t2.coefficientDenominator,
+        );
+
+        const gcd = findGcd(numerator, denominator);
+
+        ts[i] = {
+          indeterminates: t2.indeterminates,
+          coefficientNumerator: BI.div(numerator, gcd),
+          coefficientDenominator: BI.div(denominator, gcd),
+        };
+        ts[i - 1] = {
+          indeterminates: t1.indeterminates,
+          coefficientNumerator: ZERO,
+          coefficientDenominator: t1.coefficientDenominator,
+        };
+      }
+    }
+    return ts.filter((v) => BI.ne(v.coefficientNumerator, ZERO));
+  }
+
+  public static fromIndeterminate(indeterminate: Expression): Polynomial {
+    const indeterminates = new Indeterminates(JSON.stringify(indeterminate));
+    const terms = [
+      {
+        indeterminates: indeterminates,
+        coefficientNumerator: ONE,
+        coefficientDenominator: ONE,
+      },
+    ];
+    return new Polynomial(terms);
+  }
+
+  public static fromConstant(constant: number): Polynomial {
+    const [int, frac] = Math.abs(constant).toString(2).split(".", 2);
+    let numerator = BI.BigInt("0b" + int);
+    if (constant < 0) numerator = BI.mul(numerator, NEGATIVE_ONE);
+    let denominator = ONE;
+    if (frac) {
+      denominator = BI.exp(TWO, BI.BigInt(frac.length));
+      numerator = BI.add(
+        BI.mul(numerator, denominator),
+        BI.BigInt("0b" + frac),
+      );
+    }
+
+    const terms = [
+      {
+        indeterminates: new Indeterminates(),
+        coefficientNumerator: numerator,
+        coefficientDenominator: denominator,
+      },
+    ];
+
+    return new Polynomial(terms);
+  }
+
+  public negation(): Polynomial {
+    const terms = this.terms.map((t) => ({
+      indeterminates: t.indeterminates,
+      coefficientNumerator: BI.mul(t.coefficientNumerator, NEGATIVE_ONE),
+      coefficientDenominator: t.coefficientDenominator,
+    }));
+
+    return new Polynomial(terms);
+  }
+
+  public reciprocal(): Polynomial {
+    const terms = this.terms.map((t) => ({
+      indeterminates: t.indeterminates.reciprocal(),
+      coefficientNumerator: t.coefficientDenominator,
+      coefficientDenominator: t.coefficientNumerator,
+    }));
+    return new Polynomial(terms);
+  }
+
+  public constant(): Polynomial {
+    const terms = this.terms.filter((t) => !t.indeterminates.sortedKeys.length);
+    return new Polynomial(terms);
+  }
+
+  public add(rhs: Polynomial): Polynomial {
+    return new Polynomial(
+      Polynomial.simplifyTerms(this.terms.concat(rhs.terms)),
+    );
+  }
+
+  public subtract(rhs: Polynomial): Polynomial {
+    return this.add(rhs.negation());
+  }
+
+  public multiply(rhs: Polynomial): Polynomial {
+    const terms: Term[] = [];
+
+    for (const t1 of this.terms) {
+      for (const t2 of rhs.terms) {
+        const numerator = BI.mul(
+          t1.coefficientNumerator,
+          t2.coefficientNumerator,
+        );
+        const denominator = BI.mul(
+          t1.coefficientDenominator,
+          t2.coefficientDenominator,
+        );
+        const gcd = findGcd(numerator, denominator);
+
+        terms.push({
+          indeterminates: Indeterminates.multiply(
+            t1.indeterminates,
+            t2.indeterminates,
+          ),
+          coefficientNumerator: BI.div(numerator, gcd),
+          coefficientDenominator: BI.div(denominator, gcd),
+        });
+      }
+    }
+
+    return new Polynomial(Polynomial.simplifyTerms(terms));
+  }
+
+  public divide(rhs: Polynomial): Polynomial {
+    return this.multiply(rhs.reciprocal());
+  }
+
+  public toString(): string {
+    const add: string[] = [];
+    for (const t of this.terms) {
+      const coefficient =
+        BI.toNumber(t.coefficientNumerator) /
+        BI.toNumber(t.coefficientDenominator);
+
+      const mul: string[] = [];
+      if (t.indeterminates.sortedKeys.length) {
+        for (const k of t.indeterminates.sortedKeys) {
+          const w = t.indeterminates.map.get(k);
+          for (let i = Math.abs(w); i > 0; --i) {
+            if (w > 0) mul.push(k);
+            else mul.push(`["/",1,${k}]`);
+          }
+        }
+
+        if (coefficient !== 1) mul.push(coefficient.toString());
+
+        if (mul.length > 1) add.push(`["*",${mul.join(",")}]`);
+        else add.push(mul["0"]);
+      } else {
+        add.push(coefficient.toString());
+      }
+    }
+
+    if (!add.length) return "0";
+    else if (add.length === 1) return add[0];
+    else return `["+",${add.join(",")}]`;
+  }
+}
+
+const ADDITIVE_IDENTITY = Polynomial.fromConstant(0);
+const MULTIPLICATIVE_IDENTITY = Polynomial.fromConstant(1);
+
+const SWAPPED_OPS = {
+  "=": "=",
+  "<>": "<>",
+  ">": "<",
+  ">=": "<=",
+  "<": ">",
+  "<=": ">=",
+};
+
+function normalizeCallback(exp: Expression): Expression {
+  if (!Array.isArray(exp)) return exp;
+  const op = exp[0];
+
+  if (op === "FUNC" && exp[1] === "COALESCE") {
+    const res: Expression[] = ["CASE"];
+    for (let i = 2; i < exp.length; ++i)
+      res.push(normalizeCallback(["IS NOT NULL", exp[i]]), exp[i]);
+    return normalizeCallback(res);
+  }
+
+  if (op === "CASE") {
+    const res = [] as [Expression, Expression][];
+    for (let i = 1; i < exp.length; i += 2) {
+      let w = exp[i];
+      if (w instanceof Polynomial) w = JSON.parse(w.toString());
+      if (!Array.isArray(w) && !w) continue;
+      const t = exp[i + 1];
+      if (!Array.isArray(t) || t[0] !== "CASE") {
+        res.push([w, t]);
+        continue;
+      }
+      for (let j = 1; j < t.length; j += 2) res.push([and(w, t[j]), t[j + 1]]);
+      res.push([w, null]);
+      if (!Array.isArray(w) && w) break;
+    }
+    while (res[res.length - 1][1] == null) res.pop();
+    return ["CASE", ...res.flat()];
+  }
+
+  const permutations: Map<number, [Expression, Expression][]> = new Map();
+  for (const [i, e] of exp.entries()) {
+    if (!Array.isArray(e)) continue;
+    if (e[0] !== "CASE") continue;
+    const perms: [Expression, Expression][] = [];
+    for (let j = 1; j < e.length; j += 2) perms.push([e[j], e[j + 1]]);
+    perms.push([true, null]);
+    permutations.set(i, perms);
+  }
+
+  if (permutations.size) {
+    let res: [Expression, Expression][] = [[true, exp]];
+    for (const [i, perms] of permutations) {
+      const res2: [Expression, Expression][] = [];
+      for (const [w, t] of perms) {
+        res2.push(
+          ...res.map((r) => {
+            const e = (r[1] as Expression[]).slice();
+            e[i] = t;
+            return [and(w, r[0]), e] as [Expression, Expression];
+          }),
+        );
+        if (!Array.isArray(w) && w) break;
+      }
+      res = res2;
+    }
+
+    for (const r of res) r[1] = normalizeCallback(r[1]);
+
+    if (res[0][0] === true) return res[0][1];
+    while (res[res.length - 1][1] == null) res.pop();
+
+    return ["CASE", ...res.flat()];
+  }
+
+  function toPolynomial(e: Expression): Polynomial {
+    if (e == null) return null;
+    if (e instanceof Polynomial) return e;
+    if (typeof e === "number") return Polynomial.fromConstant(e);
+    if (typeof e === "string")
+      return Polynomial.fromConstant(parseFloat(e) || 0);
+    if (typeof e === "boolean") return Polynomial.fromConstant(+e);
+    return Polynomial.fromIndeterminate(e);
+  }
+
+  if (op === "+") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce(
+      (previousValue, currentValue) => previousValue.add(currentValue),
+      ADDITIVE_IDENTITY,
+    ) as unknown as Expression;
+  } else if (op === "*") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce(
+      (previousValue, currentValue) => previousValue.multiply(currentValue),
+      MULTIPLICATIVE_IDENTITY,
+    ) as unknown as Expression;
+  } else if (op === "-") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce((previousValue, currentValue) =>
+      previousValue.subtract(currentValue),
+    ) as unknown as Expression;
+  } else if (op === "/") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce((previousValue, currentValue) =>
+      previousValue.divide(currentValue),
+    ) as unknown as Expression;
+  } else if (["=", "<>", ">", ">=", "<", "<="].includes(op)) {
+    if (exp[1] == null || exp[2] == null) return null;
+    let lhs: Polynomial, rhs: Polynomial;
+    if (exp[1] instanceof Polynomial) lhs = exp[1];
+    else if (typeof exp[1] === "number") lhs = Polynomial.fromConstant(exp[1]);
+
+    if (exp[2] instanceof Polynomial) rhs = exp[2];
+    else if (typeof exp[2] === "number") rhs = Polynomial.fromConstant(exp[2]);
+
+    if (lhs || rhs) {
+      if (!lhs) lhs = Polynomial.fromIndeterminate(exp[1]);
+      if (!rhs) rhs = Polynomial.fromIndeterminate(exp[2]);
+
+      lhs = lhs.subtract(rhs);
+      rhs = lhs.constant().negation();
+      lhs = lhs.add(rhs);
+
+      if (!lhs.terms.length) {
+        exp = [op, JSON.parse(lhs.toString()), JSON.parse(rhs.toString())];
+
+        if (op === "=") return exp[1] === exp[2];
+        else if (op === "<>") return exp[1] !== exp[2];
+        else if (op === ">") return exp[1] > exp[2];
+        else if (op === ">=") return exp[1] >= exp[2];
+        else if (op === "<") return exp[1] < exp[2];
+        else if (op === "<=") return exp[1] <= exp[2];
+      } else {
+        let flipOp = 1;
+
+        const n = lhs.terms[0].coefficientNumerator;
+        const d = lhs.terms[0].coefficientDenominator;
+
+        if (BI.lt(n, ZERO) || BI.lt(d, ZERO)) flipOp *= -1;
+
+        const reciprocal = new Polynomial([
+          {
+            indeterminates: new Indeterminates(),
+            coefficientNumerator: d,
+            coefficientDenominator: n,
+          },
+        ]);
+
+        lhs = lhs.multiply(reciprocal);
+        rhs = rhs.multiply(reciprocal);
+
+        const keys = lhs.terms[0].indeterminates.sortedKeys;
+        let invert = lhs.terms[0].indeterminates.map.get(keys[0]) < 0 ? -1 : 0;
+
+        for (const t of lhs.terms)
+          for (const v of t.indeterminates.map.values()) invert += v;
+
+        if (invert < 0) {
+          flipOp *= -1;
+          lhs = lhs.reciprocal();
+          rhs = rhs.reciprocal();
+        }
+
+        if (flipOp < 0) exp = [SWAPPED_OPS[op], lhs, rhs];
+        else exp = [op, lhs, rhs];
+      }
+    }
+  }
+
+  // Restore polynomial expressions
+  exp = exp.map((e) =>
+    e instanceof Polynomial ? JSON.parse(e.toString()) : e,
+  );
+
+  exp = evaluateCallback(exp);
+
+  return exp;
+}
+
+export default function normalize(expr: Expression): Expression {
+  expr = map(expr, normalizeCallback);
+  if (expr instanceof Polynomial) {
+    expr = JSON.parse(expr.toString());
+  } else if (Array.isArray(expr) && expr[0] === "CASE") {
+    expr = expr.map((e) =>
+      e instanceof Polynomial ? JSON.parse(e.toString()) : e,
+    );
+  }
+  return expr;
+}

lib/common/expression/pagination.ts

@@ -0,0 +1,148 @@
+import { complement } from "espresso-iisojs";
+import { Expression } from "../../types";
+import normalize from "./normalize";
+import { Clause, SynthContext } from "./synth";
+import { and, or } from "./util";
+
+type Bookmark = Record<string, null | boolean | number | string>;
+
+type Minterm = number[];
+
+export function toBookmark(
+  sort: Record<string, number>,
+  row: unknown,
+): Bookmark {
+  const bookmark: Bookmark = {};
+  for (const param of Object.keys(sort)) {
+    let v = row[param];
+    if (v != null && typeof v === "object") v = v.value?.[0];
+    bookmark[param] = v;
+  }
+  return bookmark;
+}
+
+export function bookmarkToExpression(
+  bookmark: Bookmark,
+  sort: Record<string, number>,
+): Expression {
+  return Object.entries(sort)
+    .reverse()
+    .reduce((cur, kv) => {
+      const [param, asc] = kv;
+      if (asc < 0) {
+        if (bookmark[param] == null) {
+          return or(
+            ["IS NOT NULL", ["PARAM", param]],
+            and(["IS NULL", ["PARAM", param]], cur),
+          );
+        }
+        let f = null;
+        f = or(f, [">", ["PARAM", param], bookmark[param]]);
+        return or(f, and(["=", ["PARAM", param], bookmark[param]], cur));
+      } else {
+        let f: Expression = ["IS NULL", ["PARAM", param]];
+        if (bookmark[param] == null) return and(f, cur);
+        f = or(f, ["<", ["PARAM", param], bookmark[param]]);
+        return or(f, and(["=", ["PARAM", param], bookmark[param]], cur));
+      }
+    }, true as Expression);
+}
+
+function getCover(
+  context: SynthContext,
+  minterm: Minterm,
+  allSort: [string, number][],
+): Minterm[] {
+  if (!allSort.length) return [[]];
+  const [param, sort] = allSort[0];
+
+  const cov: Minterm = [];
+  const nextCov: Minterm = [];
+
+  if (sort > 0) {
+    const lhs = new Clause.Exp(["PARAM", param]);
+    const isNull = context.getVar(new Clause.IsNull(lhs));
+    cov.push((isNull << 2) ^ 2);
+  }
+
+  for (const m of minterm) {
+    const clause = context.getClause(m >>> 2);
+    if (clause instanceof Clause.IsNull) {
+      const lhs = clause.operand.expression();
+      if (!Array.isArray(lhs) || lhs[0] !== "PARAM") continue;
+      if (lhs[1] !== param) continue;
+      nextCov.push(m);
+      if (sort < 0 && m & 1) cov.push(m, m ^ 1);
+    } else if (clause instanceof Clause.Compare) {
+      const lhs = clause.lhs.expression();
+      if (!Array.isArray(lhs) || lhs[0] !== "PARAM") continue;
+      if (lhs[1] !== param) continue;
+      if (!(m & 1) && sort > 0) continue;
+      nextCov.push(m);
+
+      const negate = (m ^ (m >> 1)) & 1;
+
+      if (sort > 0) {
+        if (
+          (clause.op === "=" && !negate) ||
+          (clause.op === ">" && !negate) ||
+          (clause.op === "<" && negate)
+        ) {
+          const c = new Clause.Compare(clause.lhs, ">", clause.rhs);
+          const v = context.getVar(c);
+          cov.push((v << 2) ^ 3);
+        }
+      } else if (sort < 0) {
+        if (
+          (clause.op === "=" && !negate) ||
+          (clause.op === ">" && negate) ||
+          (clause.op === "<" && !negate)
+        ) {
+          const c = new Clause.Compare(clause.lhs, "<", clause.rhs);
+          const v = context.getVar(c);
+          cov.push((v << 2) ^ 0);
+        }
+      }
+    }
+  }
+
+  const next = getCover(context, minterm, allSort.slice(1));
+
+  return [cov, ...next.map((n) => [...nextCov, ...n])];
+}
+
+export function paginate(
+  fetched: Expression,
+  toFetch: Expression,
+  sort: Record<string, number>,
+): [Expression, Expression] {
+  fetched = normalize(fetched);
+  if (!fetched) return [false, toFetch];
+
+  toFetch = normalize(toFetch);
+  if (!toFetch) return [false, toFetch];
+
+  const synth1 = Clause.fromExpression(fetched);
+  const synth2 = Clause.fromExpression(toFetch);
+
+  const context = new SynthContext();
+
+  const expr1Minterms = synth1.true(context);
+  const expr2MintermsC = complement(synth2.true(context));
+
+  const gaps = context.sanitizeMinterms(
+    complement([
+      ...expr1Minterms,
+      ...expr2MintermsC,
+      ...context.getDcSet([...expr1Minterms, ...expr2MintermsC]),
+    ]),
+  );
+
+  const cover = gaps.flatMap((m) => getCover(context, m, Object.entries(sort)));
+  const minterms1 = context.minimize(complement([...cover, ...expr2MintermsC]));
+  const minterms2 = context.minimize(
+    complement([...minterms1, ...expr2MintermsC]),
+  );
+
+  return [context.toExpression(minterms1), context.toExpression(minterms2)];
+}

lib/common/expression/parser.ts

@@ -1,24 +1,5 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 import parsimmon from "parsimmon";
-import { Expression } from "../types";
+import { Expression } from "../../types.ts";
 
 // Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
 function interpretEscapes(str): string {
@@ -29,7 +10,7 @@ function interpretEscapes(str): string {
     r: "\r",
     t: "\t",
   };
-  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
+  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/g, (_, escape) => {
     const type = escape.charAt(0);
     const hex = escape.slice(1);
     if (type === "u") return String.fromCharCode(parseInt(hex, 16));
@@ -40,10 +21,7 @@ function interpretEscapes(str): string {
   });
 }
 
-export function map(
+export function map<T>(exp: Expression, callback: (e: Expression) => T): T {
-  exp: Expression,
-  callback: (e: Expression) => Expression
-): Expression {
   if (!Array.isArray(exp)) return callback(exp);
 
   let clone;
@@ -60,7 +38,7 @@ export function map(
 
 export async function mapAsync(
   exp: Expression,
-  callback: (e: Expression) => Promise<Expression>
+  callback: (e: Expression) => Promise<Expression>,
 ): Promise<Expression> {
   if (!Array.isArray(exp)) return callback(exp);
 
@@ -78,7 +56,7 @@ export async function mapAsync(
 
 function binaryLeft(
   operatorsParser: parsimmon.Parser<string>,
-  nextParser: parsimmon.Parser<Expression>
+  nextParser: parsimmon.Parser<Expression>,
 ): parsimmon.Parser<Expression> {
   return parsimmon.seqMap(
     nextParser,
@@ -87,10 +65,8 @@ function binaryLeft(
       rest.reduce((acc, ch) => {
         const [op, another] = ch;
         if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
-        if (Array.isArray(another) && op === another[0])
-          return [op, acc].concat(another.slice(1));
         return [op, acc, another];
-      }, first)
+      }, first),
   );
 }
 
@@ -103,7 +79,7 @@ const lang = parsimmon.createLanguage({
         parsimmon.string("<="),
         parsimmon.string("="),
         parsimmon.string(">"),
-        parsimmon.string("<")
+        parsimmon.string("<"),
       )
       .skip(parsimmon.optWhitespace);
   },
@@ -114,7 +90,7 @@ const lang = parsimmon.createLanguage({
         parsimmon
           .regexp(/not\s+like/i)
           .result("NOT LIKE")
-          .desc("NOT LIKE")
+          .desc("NOT LIKE"),
       )
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
@@ -129,7 +105,7 @@ const lang = parsimmon.createLanguage({
         parsimmon
           .regexp(/is\s+not\s+null/i)
           .result("IS NOT NULL")
-          .desc("IS NOT NULL")
+          .desc("IS NOT NULL"),
       )
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
@@ -164,8 +140,8 @@ const lang = parsimmon.createLanguage({
         parsimmon.regexp(/[a-zA-Z0-9_.*-]+/),
         r.Expression.wrap(
           parsimmon.string("{").skip(parsimmon.optWhitespace),
-          parsimmon.string("}")
+          parsimmon.string("}"),
-        )
+        ),
       )
       .atLeast(1)
       .map((x) => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
@@ -199,7 +175,7 @@ const lang = parsimmon.createLanguage({
     return parsimmon
       .alt(
         parsimmon.regexp(/true/i).result(true).desc("TRUE"),
-        parsimmon.regexp(/false/i).result(false).desc("FALSE")
+        parsimmon.regexp(/false/i).result(false).desc("FALSE"),
       )
       .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
       .skip(parsimmon.optWhitespace);
@@ -220,9 +196,9 @@ const lang = parsimmon.createLanguage({
         .desc("function"),
       r.ExpressionList.wrap(
         parsimmon.string("(").skip(parsimmon.optWhitespace),
-        parsimmon.string(")").skip(parsimmon.optWhitespace)
+        parsimmon.string(")").skip(parsimmon.optWhitespace),
       ),
-      (f, args) => ["FUNC", f.toUpperCase()].concat(args)
+      (f, args) => ["FUNC", f.toUpperCase()].concat(args),
     );
   },
   WhenPair: function (r) {
@@ -238,7 +214,7 @@ const lang = parsimmon.createLanguage({
         .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
         .skip(parsimmon.optWhitespace)
         .desc("THEN")
-        .then(r.Expression)
+        .then(r.Expression),
     );
   },
   CaseStatement: function (r) {
@@ -261,10 +237,10 @@ const lang = parsimmon.createLanguage({
         .skip(
           parsimmon
             .regex(/end/i)
-            .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+            .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/)),
         )
         .skip(parsimmon.optWhitespace),
-      (...arr) => arr.flat(2)
+      (...arr) => arr.flat(2),
     );
   },
   Value: function (r) {
@@ -275,7 +251,7 @@ const lang = parsimmon.createLanguage({
       r.StringValueSql,
       r.StringValueJs,
       r.FuncValue,
-      r.CaseStatement
+      r.CaseStatement,
     );
   },
   ValueExpression: function (r) {
@@ -287,18 +263,22 @@ const lang = parsimmon.createLanguage({
           .skip(parsimmon.optWhitespace),
         binaryLeft(
           parsimmon
-            .alt(parsimmon.string("*"), parsimmon.string("/"))
+            .alt(
+              parsimmon.string("*"),
+              parsimmon.string("/"),
+              parsimmon.string("%"),
+            )
             .skip(parsimmon.optWhitespace),
           parsimmon.alt(
             r.Value,
             r.Parameter,
             r.Expression.wrap(
               parsimmon.string("(").skip(parsimmon.optWhitespace),
-              parsimmon.string(")").skip(parsimmon.optWhitespace)
+              parsimmon.string(")").skip(parsimmon.optWhitespace),
-            )
+            ),
-          )
+          ),
-        )
+        ),
-      )
+      ),
     );
   },
   Comparison: function (r) {
@@ -308,7 +288,7 @@ const lang = parsimmon.createLanguage({
         r.ValueExpression,
         r.ComparisonOperator,
         r.ValueExpression,
-        (p, o, v) => [o, p, v]
+        (p, o, v) => [o, p, v],
       ),
       parsimmon.seqMap(
         r.ValueExpression,
@@ -318,28 +298,28 @@ const lang = parsimmon.createLanguage({
             .regexp(/escape/i)
             .result("ESCAPE")
             .skip(parsimmon.whitespace)
-            .desc("ESCAPE")
+            .desc("ESCAPE"),
         ),
         r.ValueExpression,
-        (a, b, c, d) => [b, a, c, d]
+        (a, b, c, d) => [b, a, c, d],
       ),
       parsimmon.seqMap(
         r.ValueExpression,
         r.LikeOperator,
         r.ValueExpression,
-        (a, b, c) => [b, a, c]
+        (a, b, c) => [b, a, c],
-      )
+      ),
     );
   },
   ExpressionList: function (r) {
     return r.Expression.sepBy(
-      parsimmon.string(",").skip(parsimmon.optWhitespace)
+      parsimmon.string(",").skip(parsimmon.optWhitespace),
     );
   },
   Expression: function (r) {
     function unary(
       operatorsParser: parsimmon.Parser<string>,
-      nextParser: parsimmon.Parser<Expression>
+      nextParser: parsimmon.Parser<Expression>,
     ): parsimmon.Parser<Expression> {
       return parsimmon.seq(operatorsParser, nextParser).or(nextParser);
     }
@@ -348,8 +328,8 @@ const lang = parsimmon.createLanguage({
       r.OrOperator,
       binaryLeft(
         r.AndOperator,
-        unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
+        unary(r.NotOperator, r.Comparison.or(r.ValueExpression)),
-      )
+      ),
     ).trim(parsimmon.optWhitespace);
   },
 });
@@ -386,6 +366,7 @@ export function stringify(exp: Expression, level = 0): string {
     "-": 31,
     "*": 32,
     "/": 32,
+    "%": 32,
   };
 
   const op = exp[0].toUpperCase();
@@ -400,7 +381,7 @@ export function stringify(exp: Expression, level = 0): string {
       `${exp[1]}(${exp
         .slice(2)
         .map((e) => stringify(e))
-        .join(", ")})`
+        .join(", ")})`,
     );
   } else if (op === "PARAM") {
     if (typeof exp[1] === "string") {
@@ -413,7 +394,7 @@ export function stringify(exp: Expression, level = 0): string {
             if (typeof p === "string") return p;
             else return `{${stringify(p)}}`;
           })
-          .join("")
+          .join(""),
       );
     } else {
       return wrap(`{${stringify(exp[1])}}`);
@@ -425,15 +406,15 @@ export function stringify(exp: Expression, level = 0): string {
       return wrap(
         `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
           exp[2],
-          opLevels[op]
+          opLevels[op],
-        )} ESCAPE ${stringify(exp[3], opLevels[op])}`
+        )} ESCAPE ${stringify(exp[3], opLevels[op])}`,
       );
     } else {
       return wrap(
         `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
           exp[2],
-          opLevels[op]
+          opLevels[op],
-        )}`
+        )}`,
       );
     }
   } else if (op === "CASE") {

lib/common/expression/synth.ts

@@ -0,0 +1,972 @@
+import { espresso, complement, tautology } from "espresso-iisojs";
+import { Expression } from "../../types.ts";
+import normalize from "./normalize.ts";
+import { map, parseLikePattern } from "./parser.ts";
+
+type Minterm = number[];
+
+export abstract class SynthContextBase<
+  T extends { toString: () => string } = unknown,
+  U = unknown,
+> {
+  public variables = new Map<string, number>();
+  protected clauses = new Map<number, U>();
+
+  public getVar(c: U): number {
+    const str = c.toString();
+    let idx = this.variables.get(str);
+    if (idx == null) {
+      idx = this.variables.size;
+      this.variables.set(str, idx);
+      this.clauses.set(idx, c);
+    }
+    return idx;
+  }
+
+  public getClause(v: number): U {
+    return this.clauses.get(v);
+  }
+
+  abstract getMinterms(exp: T, res: true | false | null): Minterm[];
+  abstract getDcSet(minterms: Minterm[]): Minterm[];
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  canRaise(idx: number, set: Set<number>): boolean {
+    return true;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  canLower(idx: number, set: Set<number>): boolean {
+    return true;
+  }
+
+  bias(a: number, b: number): number {
+    // Bias towards 1 then lower index
+    return (b ^ 1) - (a ^ 1);
+  }
+
+  sanitizeMinterms(minterms: Minterm[]): Minterm[] {
+    return minterms;
+  }
+
+  minimize(minterms: Minterm[], dcSet: Minterm[] = []): Minterm[] {
+    minterms = this.sanitizeMinterms(minterms);
+    const canRaise = this.canRaise.bind(this);
+    const canLower = this.canLower.bind(this);
+    const bias = this.bias.bind(this);
+
+    return espresso(
+      minterms,
+      [...this.getDcSet([...minterms, ...dcSet]), ...dcSet],
+      {
+        canRaise,
+        canLower,
+        bias,
+      },
+    );
+  }
+}
+
+function* findIsNullDeps(exp: Expression): IterableIterator<Expression> {
+  if (!Array.isArray(exp)) return;
+  const op = exp[0];
+  if (op === "IS NULL" || op === "IS NOT NULL") return;
+
+  if (op === "FUNC") {
+    if (exp[1] === "NOW") {
+      return;
+    } else if (exp[1] === "LOWER" || exp[1] === "UPPER") {
+      yield* findIsNullDeps(exp[2]);
+    } else if (exp[1] === "ROUND") {
+      for (const e of exp.slice(2, 4)) yield* findIsNullDeps(e);
+      return;
+    }
+  } else if (op !== "PARAM") {
+    for (const e of exp.slice(1)) yield* findIsNullDeps(e);
+    return;
+  }
+  yield exp;
+}
+
+export abstract class Clause {
+  private _isNullable: Set<string>;
+  protected _expression: Expression;
+  abstract true(context: SynthContextBase<Clause>): Minterm[];
+  abstract false(context: SynthContextBase<Clause>): Minterm[];
+  abstract null(context: SynthContextBase<Clause>): Minterm[];
+  expression(): Expression {
+    if (this._expression !== undefined) return this._expression;
+    const context = createSynthContext();
+    const minterms = this.true(context);
+    const minimized = context.minimize(minterms);
+    this._expression = context.toExpression(minimized) as Expression;
+    return this._expression;
+  }
+  isBoolean(): boolean {
+    return true;
+  }
+  isNullable(c: Clause.IsNull): boolean {
+    if (!this._isNullable) {
+      this._isNullable = new Set(
+        [...this.getNullables()].map((n) => n.toString()),
+      );
+    }
+    return this._isNullable.has(c.toString());
+  }
+  *getNullables(): IterableIterator<Clause.IsNull> {
+    const exp = this.expression();
+    for (const e of findIsNullDeps(exp)) {
+      if (e === exp) yield new Clause.IsNull(this);
+      else yield new Clause.IsNull(new Clause.Exp(e));
+    }
+  }
+  toString(): string {
+    return JSON.stringify(this.expression());
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/no-namespace
+export namespace Clause {
+  export class Not extends Clause {
+    constructor(public operand: Clause) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.false(context);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.true(context);
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.null(context);
+    }
+  }
+
+  export class Or extends Clause {
+    constructor(public operands: Clause[]) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      for (const s of this.operands) {
+        const m = s.true(context);
+        if (m.length === 1 && !m[0].length) return [[]];
+        minterms.push(...m);
+      }
+      return minterms;
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[][] = [];
+      const operands = [...this.operands];
+      for (const e of operands) {
+        if (e instanceof Or) {
+          operands.push(...e.operands);
+          continue;
+        }
+        const m = e.false(context);
+        if (!m.length) return [];
+        minterms.push(m);
+      }
+      if (!minterms.length) return [[]];
+      if (minterms.length === 1) return minterms[0];
+      return complement(minterms.map((m) => complement(m)).flat());
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      const trueMinterms: Minterm[] = [];
+      const operands = [...this.operands];
+      for (const s of operands) {
+        if (s instanceof Or) {
+          operands.push(...s.operands);
+          continue;
+        }
+        const t = s.true(context);
+        if (t.length === 1 && !t[0].length) return [];
+        trueMinterms.push(...t);
+      }
+
+      const nullMinterms: Minterm[] = [];
+      for (const s of operands) {
+        const n = s.null(context);
+        if (n.length === 1 && !n[0].length && !trueMinterms.length) return [[]];
+        nullMinterms.push(...n);
+      }
+
+      if (!trueMinterms.length) return nullMinterms;
+      return complement([...complement(nullMinterms), ...trueMinterms]);
+    }
+  }
+
+  export class And extends Clause {
+    constructor(public operands: Clause[]) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[][] = [];
+      const operands = [...this.operands];
+      for (const s of operands) {
+        if (s instanceof And) {
+          operands.push(...s.operands);
+          continue;
+        }
+        const m = s.true(context);
+        if (!m.length) return [];
+        if (m.length === 1 && !m[0].length) continue;
+        minterms.push(m);
+      }
+      if (!minterms.length) return [[]];
+      if (minterms.length === 1) return minterms[0];
+      return complement(minterms.map((m) => complement(m)).flat());
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      for (const s of this.operands) {
+        const m = s.false(context);
+        if (m.length === 1 && !m[0].length) return [[]];
+        minterms.push(...m);
+      }
+      return minterms;
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      const falseMinterms: Minterm[] = [];
+      const operands = [...this.operands];
+      for (const s of operands) {
+        if (s instanceof And) {
+          operands.push(...s.operands);
+          continue;
+        }
+        const f = s.false(context);
+        if (f.length === 1 && !f[0].length) return [];
+        falseMinterms.push(...f);
+      }
+
+      const nullMinterms: Minterm[] = [];
+      for (const s of operands) {
+        const n = s.null(context);
+        if (n.length === 1 && !n[0].length && !falseMinterms.length)
+          return [[]];
+        nullMinterms.push(...n);
+      }
+
+      if (!falseMinterms.length) return nullMinterms;
+      return complement([...complement(nullMinterms), ...falseMinterms]);
+    }
+  }
+
+  export class Case extends Clause {
+    constructor(public clauses: Clause[]) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      const cumulative: Minterm[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        const w = this.clauses[i].true(context);
+        const t = this.clauses[i + 1].true(context);
+        minterms.push(
+          ...complement([...cumulative, ...complement(w), ...complement(t)]),
+        );
+        if (i < this.clauses.length - 2) {
+          cumulative.push(
+            ...complement([
+              ...this.clauses[i].false(context),
+              ...this.clauses[i].null(context),
+            ]),
+          );
+        }
+      }
+      return minterms;
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      const cumulative: Minterm[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        const w = this.clauses[i].true(context);
+        const t = this.clauses[i + 1].false(context);
+        minterms.push(
+          ...complement([...cumulative, ...complement(w), ...complement(t)]),
+        );
+        if (i < this.clauses.length - 2) {
+          cumulative.push(
+            ...complement([
+              ...this.clauses[i].false(context),
+              ...this.clauses[i].null(context),
+            ]),
+          );
+        }
+      }
+      return minterms;
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      const cumulative: Minterm[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        const w = this.clauses[i].true(context);
+        const t = this.clauses[i + 1].null(context);
+        minterms.push(
+          ...complement([...cumulative, ...complement(w), ...complement(t)]),
+        );
+        cumulative.push(
+          ...complement([
+            ...this.clauses[i].false(context),
+            ...this.clauses[i].null(context),
+          ]),
+        );
+      }
+      minterms.push(...complement([...cumulative]));
+      return minterms;
+    }
+    expression(): Expression {
+      if (this._expression != null) return this._expression;
+      if (this.isBoolean()) {
+        this._expression = new Clause.Not(new Clause.Not(this)).expression();
+        return this._expression;
+      }
+      const context = createSynthContext();
+      const cases: { when: Minterm[]; then: Expression }[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        let minterms = this.clauses[i].true(context);
+        const then = this.clauses[i + 1].expression();
+        if (
+          cases.length &&
+          JSON.stringify(then) === JSON.stringify(cases[cases.length - 1].then)
+        )
+          minterms.push(...cases.pop().when);
+        minterms = context.minimize(
+          minterms,
+          cases.flatMap((c) => c.when),
+        );
+        if (!minterms.length) continue;
+        cases.push({ when: minterms, then });
+        if (minterms.length === 1 && !minterms[0].length) break;
+      }
+      while (cases[cases.length - 1].then == null) cases.pop();
+      if (!cases.length) return null;
+      this._expression = [
+        "CASE",
+        ...cases.flatMap((c) => [context.toExpression(c.when), c.then]),
+      ];
+      return this._expression;
+    }
+
+    isBoolean(): boolean {
+      for (let i = 1; i < this.clauses.length; i += 2)
+        if (!this.clauses[i].isBoolean()) return false;
+      return true;
+    }
+  }
+
+  export class IsNull extends Clause {
+    private _boolean: Clause;
+    constructor(public operand: Clause) {
+      super();
+      this._boolean = new Not(new Not(this));
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.null(context);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return complement(this.true(context));
+    }
+    null(): Minterm[] {
+      return [];
+    }
+    isBoolean(): boolean {
+      return true;
+    }
+    *getNullables(): IterableIterator<IsNull> {
+      // Never returns null
+    }
+    expression(): Expression {
+      const nullables = [...this.operand.getNullables()];
+      if (nullables.length === 1 && nullables[0].operand === this.operand)
+        return ["IS NULL", this.operand.expression()];
+      return super.expression();
+    }
+  }
+
+  export class Exp extends Clause {
+    constructor(public exp: Expression) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      if (Array.isArray(this.exp)) return context.getMinterms(this, true);
+      if (this.exp) return [[]];
+      return [];
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      if (Array.isArray(this.exp)) return context.getMinterms(this, false);
+      if (!this.exp && this.exp != null) return [[]];
+      return [];
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      if (Array.isArray(this.exp)) return context.getMinterms(this, null);
+      if (this.exp == null) return [[]];
+      return [];
+    }
+    expression(): Expression {
+      return this.exp;
+    }
+    isBoolean(): boolean {
+      return this.exp === true || this.exp === false || this.exp == null;
+    }
+  }
+
+  export class Compare extends Clause {
+    constructor(
+      public lhs: Clause,
+      public op: ">" | "<" | "=",
+      public rhs: boolean | number | string,
+    ) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, true);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, false);
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      return this.lhs.null(context);
+    }
+    *getNullables(): IterableIterator<IsNull> {
+      yield* this.lhs.getNullables();
+    }
+    isBoolean(): boolean {
+      return true;
+    }
+    expression(): Expression {
+      return [this.op, this.lhs.expression(), this.rhs];
+    }
+  }
+
+  export class Like extends Clause {
+    public readonly caseSensitive: boolean;
+    public readonly contradiction: boolean;
+    public readonly pattern: string[];
+    public readonly lhs: Clause;
+
+    constructor(
+      lhs: Clause,
+      public rhs: string,
+      public esc?: string,
+    ) {
+      super();
+      const exp = lhs.expression();
+      let caseSensitive = true;
+      let contradiction = false;
+      if (Array.isArray(exp)) {
+        if (exp[0] === "FUNC") {
+          const func = exp[1];
+          if (func === "UPPER" || func === "LOWER") {
+            const p = func === "UPPER" ? rhs.toUpperCase() : rhs.toLowerCase();
+            if (p === rhs) caseSensitive = false;
+            else contradiction = true;
+            lhs = new Exp(exp[2]);
+          }
+        }
+      }
+      this.lhs = lhs;
+      this.pattern = parseLikePattern(rhs, esc);
+      this.caseSensitive = caseSensitive;
+      this.contradiction = contradiction;
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, true);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, false);
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      return this.lhs.null(context);
+    }
+    isBoolean(): boolean {
+      return true;
+    }
+    isNullable(c: IsNull): boolean {
+      return this.lhs.isNullable(c);
+    }
+    getNullables(): IterableIterator<IsNull> {
+      return this.lhs.getNullables();
+    }
+    expression(): Expression {
+      let lhs = this.lhs.expression();
+      if (this.contradiction) {
+        if (this.rhs === this.rhs.toLocaleUpperCase())
+          lhs = ["FUNC", "LOWER", lhs];
+        else lhs = ["FUNC", "UPPER", lhs];
+      } else if (!this.caseSensitive) {
+        if (this.rhs === this.rhs.toLocaleUpperCase())
+          lhs = ["FUNC", "UPPER", lhs];
+        else lhs = ["FUNC", "LOWER", lhs];
+      }
+      return ["LIKE", lhs, this.rhs, ...(this.esc ? [this.esc] : [])];
+    }
+  }
+
+  export function fromExpression(exp: Expression): Clause {
+    const res = map(exp, (e) => {
+      if (!Array.isArray(e)) return new Clause.Exp(e);
+      let op = e[0];
+      let negate = true;
+      if (op === "NOT LIKE") op = "LIKE";
+      else if (op === "IS NOT NULL") op = "IS NULL";
+      else if (op === "<>") op = "=";
+      else if (op === ">=") op = "<";
+      else if (op === "<=") op = ">";
+      else negate = false;
+
+      let clause: Clause;
+      if (op === "IS NULL") {
+        clause = new Clause.IsNull(e[1]);
+      } else if (op === "NOT") {
+        clause = new Clause.Not(e[1]);
+      } else if (op === "OR") {
+        clause = new Clause.Or(e.slice(1));
+      } else if (op === "AND") {
+        clause = new Clause.And(e.slice(1));
+      } else if (op === "CASE") {
+        clause = new Clause.Case(e.slice(1));
+      } else if (op === "LIKE") {
+        const rhs = e[2] instanceof Exp ? e[2].expression() : null;
+        const esc = e[3] instanceof Exp ? e[3].expression() : null;
+        if (typeof rhs === "string" && typeof esc === "string")
+          clause = new Clause.Like(e[1], rhs, esc);
+        else if (typeof rhs === "string" && esc == null)
+          clause = new Clause.Like(e[1], rhs);
+      } else if (op === ">" || op === "<" || op === "=") {
+        const rhs = e[2] instanceof Exp ? e[2].expression() : null;
+        if (["boolean", "number", "string"].includes(typeof rhs)) {
+          clause = new Clause.Compare(
+            e[1],
+            op,
+            rhs as boolean | number | string,
+          );
+        }
+      }
+
+      if (!clause) {
+        const args = e.slice(1).map((a) => (a as Clause).expression());
+        clause = new Clause.Exp([op, ...args]);
+      }
+
+      if (negate) clause = new Clause.Not(clause);
+      return clause;
+    });
+
+    return res;
+  }
+}
+
+function groupBy<T, K>(
+  input: T[],
+  callback: (item: T) => K,
+): Iterable<[K, T[]]> {
+  const groups = new Map<K, T[]>();
+  for (const item of input) {
+    const key = callback(item);
+    let arr = groups.get(key);
+    if (!arr) groups.set(key, (arr = []));
+    arr.push(item);
+  }
+
+  return groups.entries();
+}
+
+export class SynthContext extends SynthContextBase<Clause, Clause> {
+  constructor() {
+    super();
+  }
+
+  getMinterms(clause: Clause, res: true | false | null): number[][] {
+    const v = this.getVar(clause);
+    if (res === true) return [[(v << 2) ^ 3]];
+    else if (res === false) return [[(v << 2) ^ 1]];
+    else return [[v << 2, (v << 2) ^ 2]];
+  }
+
+  getDcSet(minterms: Minterm[]): number[][] {
+    const dcSet: number[][] = [];
+
+    const whitelist = new Set([...minterms.flat()].map((v) => v >> 2));
+
+    const allClauses = [...whitelist].map((v) => this.getClause(v));
+    const comparisons = allClauses.filter(
+      (c) => c instanceof Clause.Compare,
+    ) as Clause.Compare[];
+
+    // Comparisons
+    for (const [, clauses] of groupBy(comparisons, (c) =>
+      JSON.stringify(c.lhs),
+    )) {
+      const lhs = clauses[0].lhs;
+      const values = new Set(clauses.map((c) => c.rhs));
+      const valuesSorted = [...values].sort((a, b) => {
+        const ta = typeof a;
+        const tb = typeof b;
+        if (ta === tb) return a > b ? 1 : -1;
+        else if (ta === "string") return 1;
+        else if (tb === "string") return -1;
+        return +a - +b;
+      });
+
+      for (const [i, v] of valuesSorted.entries()) {
+        const eq = this.getVar(new Clause.Compare(lhs, "=", v));
+        const gt = this.getVar(new Clause.Compare(lhs, ">", v));
+        const lt = this.getVar(new Clause.Compare(lhs, "<", v));
+
+        dcSet.push([(eq << 2) ^ 3, (gt << 2) ^ 3]);
+        dcSet.push([(lt << 2) ^ 3, (gt << 2) ^ 3]);
+        dcSet.push([(lt << 2) ^ 3, (eq << 2) ^ 3]);
+        dcSet.push([(lt << 2) ^ 1, (eq << 2) ^ 1, (gt << 2) ^ 1]);
+
+        const negEquivOp = [lt, eq, gt].filter((o) => !whitelist.has(o));
+        if (negEquivOp.length === 1) whitelist.add(negEquivOp[0]);
+
+        for (let j = 0; j < i; j++) {
+          const eq2 = this.getVar(
+            new Clause.Compare(lhs, "=", valuesSorted[j]),
+          );
+          const gt2 = this.getVar(
+            new Clause.Compare(lhs, ">", valuesSorted[j]),
+          );
+          const lt2 = this.getVar(
+            new Clause.Compare(lhs, "<", valuesSorted[j]),
+          );
+
+          // This is the minimum clauses required if all relavent vars
+          // were included in the DC set.
+          // dcSet.push([(eq << 2) ^ 3, (eq2 << 2) ^ 3]);
+          // dcSet.push([(lt << 2) ^ 1, (gt2 << 2) ^ 1]);
+
+          // But we use non-minimal set because intermediate vars
+          // between any two may not be present in the DC set.
+          dcSet.push([(gt2 << 2) ^ 1, (lt << 2) ^ 1]);
+          dcSet.push([(eq2 << 2) ^ 3, (lt << 2) ^ 1]);
+          dcSet.push([(lt2 << 2) ^ 3, (lt << 2) ^ 1]);
+          dcSet.push([(gt2 << 2) ^ 1, (gt << 2) ^ 3]);
+          dcSet.push([(gt2 << 2) ^ 1, (eq << 2) ^ 3]);
+          dcSet.push([(eq2 << 2) ^ 3, (gt << 2) ^ 3]);
+          dcSet.push([(eq2 << 2) ^ 3, (eq << 2) ^ 3]);
+          dcSet.push([(lt2 << 2) ^ 3, (gt << 2) ^ 3]);
+          dcSet.push([(lt2 << 2) ^ 3, (eq << 2) ^ 3]);
+        }
+      }
+    }
+
+    // LIKE
+    const likes = allClauses.filter(
+      (c) => c instanceof Clause.Like,
+    ) as Clause.Like[];
+
+    for (const [, clauses] of groupBy(likes, (c) => JSON.stringify(c.lhs))) {
+      for (let i1 = 0; i1 < clauses.length; ++i1) {
+        const l1 = clauses[i1];
+        if (l1.contradiction) {
+          dcSet.push([(this.getVar(l1) << 2) ^ 3]);
+          continue;
+        }
+        for (let i2 = i1 + 1; i2 < clauses.length; ++i2) {
+          const l2 = clauses[i2];
+          if (l2.contradiction) continue;
+          let p1 = l1.pattern;
+          let p2 = l2.pattern;
+          if (!l1.caseSensitive || !l2.caseSensitive) {
+            p1 = p1.map((c) => c.toLowerCase());
+            p2 = p2.map((c) => c.toLowerCase());
+          }
+          if (likeDisjoint(p1, p2)) {
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 3,
+              (this.getVar(l2) << 2) ^ 3,
+            ]);
+          } else if (
+            (!l1.caseSensitive || l2.caseSensitive) &&
+            likeImplies(p1, p2)
+          ) {
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 2,
+              (this.getVar(l2) << 2) ^ 3,
+            ]);
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 1,
+              (this.getVar(l2) << 2) ^ 0,
+            ]);
+          } else if (
+            (!l2.caseSensitive || l1.caseSensitive) &&
+            likeImplies(p2, p1)
+          ) {
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 3,
+              (this.getVar(l2) << 2) ^ 2,
+            ]);
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 0,
+              (this.getVar(l2) << 2) ^ 1,
+            ]);
+          }
+        }
+      }
+    }
+
+    for (const v of whitelist) {
+      const clause = this.getClause(v);
+      const nullables = [...clause.getNullables()].map((c) => this.getVar(c));
+      if (nullables.length) {
+        dcSet.push([
+          ...nullables.map((n) => (n << 2) ^ 2),
+          (v << 2) ^ 0,
+          (v << 2) ^ 2,
+        ]);
+        for (const n of nullables) {
+          dcSet.push([(n << 2) ^ 3, (v << 2) ^ 1]);
+          dcSet.push([(n << 2) ^ 3, (v << 2) ^ 3]);
+          whitelist.add(n);
+        }
+      }
+
+      if (clause instanceof Clause.IsNull) {
+        const str = clause.operand.toString();
+        if (str === '["PARAM","DeviceID.ID"]' || str === '["PARAM","_id"]')
+          dcSet.push([(v << 2) ^ 3]);
+      }
+    }
+    return dcSet.filter((m) => m.every((v) => whitelist.has(v >> 2)));
+  }
+
+  canRaise(idx: number, set: Set<number>): boolean {
+    const clause = this.getClause(idx >> 2);
+    if (clause instanceof Clause.IsNull) {
+      for (const i of set) {
+        if (i === idx || i & 1) continue;
+        const c = this.getClause(i >> 2);
+        if (c.isNullable(clause)) return false;
+      }
+      return true;
+    }
+    return !(idx & 1) || !set.has(idx ^ 3);
+  }
+
+  canLower(idx: number, set: Set<number>): boolean {
+    if (idx & 1) return true;
+    const clause = this.getClause(idx >> 2);
+    if (clause instanceof Clause.IsNull) return true;
+    return set.has(idx ^ 3) || set.has(idx ^ 1);
+  }
+
+  bias(a: number, b: number): number {
+    // Bias towards 1 then true
+    return ((b & 3) ^ 3) - ((a & 3) ^ 3);
+  }
+
+  sanitizeMinterms(minterms: Minterm[]): Minterm[] {
+    const res = [] as number[][];
+
+    loop: for (const m of minterms) {
+      const merged: Map<number, number> = new Map();
+      for (const i of m)
+        merged.set(i >> 2, (merged.get(i >> 2) || 0) | (1 << (i & 3)));
+      const minterm: number[] = [];
+      const perms: number[][] = [];
+      for (const [k, v] of merged) {
+        if ((v & 0b0011) === 0b0011) continue loop;
+        if ((v & 0b1100) === 0b1100) continue loop;
+        const clause = this.clauses.get(k);
+        if (!clause) throw new Error("Invalid literal");
+        if (clause instanceof Clause.IsNull) {
+          if (v === 0b0100) minterm.push((k << 2) ^ 2);
+          else if (v === 0b1000) minterm.push((k << 2) ^ 3);
+          else throw new Error("Invalid literal");
+          continue;
+        }
+        if ((v & 0b1010) === 0b1010) continue loop;
+        const isNullVars = [...clause.getNullables()].map((c) =>
+          this.getVar(c),
+        );
+        const t = k << 2;
+        if (v === 0b0101) {
+          if (isNullVars.length === 1) minterm.push((isNullVars[0] << 2) ^ 3);
+          else perms.push(isNullVars.map((n) => (n << 2) ^ 3));
+        } else if (v === 0b0001) {
+          perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 3]);
+        } else if (v === 0b0100) {
+          perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 1]);
+        } else if (v & 0b1000) {
+          minterm.push(t ^ 3);
+        } else if (v & 0b0010) {
+          minterm.push(t ^ 1);
+        }
+      }
+      let ms = [minterm];
+      while (perms.length) {
+        const newMs: number[][] = [];
+        const perm = perms.pop();
+        for (const p of perm) newMs.push(...ms.map((mm) => [...mm, p]));
+        ms = newMs;
+      }
+      res.push(...ms);
+    }
+    return res;
+  }
+
+  toExpression(sop: number[][]): Expression {
+    if (!sop.length) return false;
+    const res: Expression[] = [];
+    for (const s of sop) {
+      if (!s.length) return true;
+      const conjs: Expression[] = [];
+      for (const i of s) {
+        const clause = this.getClause(i >>> 2);
+        if (!clause) throw new Error("Invalid literal");
+        if (clause instanceof Clause.IsNull) {
+          if (!(i & 2)) throw new Error("Invalid literal");
+        } else if (!(i & 1)) {
+          // Should never be reached if minimized correctly
+          const isNullVars = [...clause.getNullables()].map((c) =>
+            this.getVar(c),
+          );
+          conjs.push(
+            this.toExpression([
+              [i ^ 3],
+              ...isNullVars.map((n) => [(n << 2) ^ 3]),
+            ]),
+          );
+          continue;
+        }
+
+        let expr = clause.expression() as Expression;
+        if (!(i & 1) !== !(i & 2)) expr = ["NOT", expr];
+        if (
+          Array.isArray(expr) &&
+          expr[0] === "NOT" &&
+          Array.isArray(expr[1])
+        ) {
+          const e: Expression[] = expr[1];
+          if (e[0] === "IS NULL") expr = ["IS NOT NULL", ...e.slice(1)];
+          else if (e[0] === "LIKE") expr = ["NOT LIKE", ...e.slice(1)];
+          else if (e[0] === "=") expr = ["<>", ...e.slice(1)];
+          else if (e[0] === "<>") expr = ["=", ...e.slice(1)];
+          else if (e[0] === ">") expr = ["<=", ...e.slice(1)];
+          else if (e[0] === ">=") expr = ["<", ...e.slice(1)];
+          else if (e[0] === "<") expr = [">=", ...e.slice(1)];
+          else if (e[0] === "<=") expr = [">", ...e.slice(1)];
+          else if (e[0] === "NOT") expr = e[1];
+        }
+        conjs.push(expr);
+      }
+      if (conjs.length > 1) res.push(["AND", ...conjs]);
+      else res.push(conjs[0]);
+    }
+    if (res.length > 1) return ["OR", ...res];
+    return res[0];
+  }
+}
+
+// Classes aren't hoisted in JS but functions are. This function is used to
+// create a new SynthContext instance from inside the Clause class.
+export function createSynthContext(): SynthContext {
+  return new SynthContext();
+}
+
+export function likeImplies(pat1: string[], pat2: string[]): boolean {
+  let backtrack: [number, number] = null;
+
+  for (let i1 = 0, i2 = 0; ; ++i1, ++i2) {
+    while (i1 < pat1.length && pat1[i1] === "\\%") backtrack = [i1++, i2];
+
+    if (i2 >= pat2.length) return i1 >= pat1.length;
+
+    const c = i1 < pat1.length ? pat1[i1] : null;
+
+    if (c !== pat2[i2] && c !== "\\_") {
+      if (!backtrack) return false;
+      [i1, i2] = backtrack;
+      ++backtrack[1];
+    }
+  }
+}
+
+export function likeDisjoint(pat1: string[], pat2: string[]): boolean {
+  const left1Idx = pat1.indexOf("\\%");
+  const left2Idx = pat2.indexOf("\\%");
+  const right1Idx = pat1.lastIndexOf("\\%");
+  const right2Idx = pat2.lastIndexOf("\\%");
+
+  const left1 = pat1.slice(0, left1Idx !== -1 ? left1Idx : pat1.length);
+  const left2 = pat2.slice(0, left2Idx !== -1 ? left2Idx : pat2.length);
+  const right1 = pat1.slice(right1Idx !== -1 ? right1Idx + 1 : 0).reverse();
+  const right2 = pat2.slice(right2Idx !== -1 ? right2Idx + 1 : 0).reverse();
+
+  for (let i = 0; i < Math.min(left1.length, left2.length); ++i) {
+    if (left1[i] !== left2[i] && left1[i] !== "\\_" && left2[i] !== "\\_")
+      return true;
+  }
+
+  for (let i = 0; i < Math.min(right1.length, right2.length); ++i) {
+    if (right1[i] !== right2[i] && right1[i] !== "\\_" && right2[i] !== "\\_")
+      return true;
+  }
+
+  if (pat1.length === left1.length)
+    return pat2.filter((c) => c !== "\\%").length > pat1.length;
+  else if (pat2.length === left2.length)
+    return pat1.filter((c) => c !== "\\%").length > pat2.length;
+
+  return false;
+}
+
+export function minimize(expr: Expression, boolean = false): Expression {
+  let synth = Clause.fromExpression(normalize(expr));
+  if (boolean) synth = new Clause.Not(new Clause.Not(synth));
+  return synth.expression();
+}
+
+export function unionDiff(
+  expr1: Expression,
+  expr2: Expression,
+): [Expression, Expression] {
+  expr2 = normalize(expr2);
+
+  if (!expr2) return [expr1, false];
+
+  const synth2 = Clause.fromExpression(expr2);
+
+  if (!expr1) {
+    const e = synth2.expression();
+    return [e, e];
+  }
+
+  expr1 = normalize(expr1);
+  const synth1 = Clause.fromExpression(expr1);
+
+  const context = new SynthContext();
+
+  const expr2Minterms = synth2.true(context);
+  const expr1Minterms = synth1.true(context);
+
+  const union = context.minimize([...expr1Minterms, ...expr2Minterms]);
+
+  const diff = context.minimize(
+    complement([...expr1Minterms, ...complement(expr2Minterms)]),
+  );
+
+  return [context.toExpression(union), context.toExpression(diff)];
+}
+
+export function covers(expr1: Expression, expr2: Expression): boolean {
+  expr2 = normalize(expr2);
+  if (!expr2) return true;
+  expr1 = normalize(expr1);
+  if (!Array.isArray(expr1)) return !!expr1;
+
+  const synt1 = Clause.fromExpression(expr1);
+  const synt2 = Clause.fromExpression(expr2);
+
+  const context = new SynthContext();
+  const expr1Minterms = synt1.true(context);
+  const expr2Minterms = synt2.true(context);
+
+  return tautology([
+    ...context.sanitizeMinterms(complement(expr2Minterms)),
+    ...context.getDcSet([...expr2Minterms, ...expr1Minterms]),
+    ...context.sanitizeMinterms(expr1Minterms),
+  ]);
+}

lib/common/expression/util.ts

@@ -1,24 +1,5 @@
-/**
+import { map, mapAsync, parseLikePattern } from "./parser.ts";
- * Copyright 2013-2019  GenieACS Inc.
+import { Expression } from "../../types.ts";
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { map, mapAsync, parseLikePattern } from "./expression-parser";
-import { Expression } from "../types";
 
 const isArray = Array.isArray;
 
@@ -74,7 +55,7 @@ export function likePatternToRegExp(pat: string, esc = "", flags = ""): RegExp {
 
 function compare(
   a: boolean | number | string,
-  b: boolean | number | string
+  b: boolean | number | string,
 ): number {
   if (typeof a === "boolean") a = +a;
   if (typeof b === "boolean") b = +b;
@@ -130,6 +111,15 @@ export function evaluateCallback(exp: Expression): Expression {
     } else if (exp[1] === "LOWER") {
       if (exp[2] == null) return null;
       if (!isArray(exp[2])) return toString(exp[2]).toLowerCase();
+    } else if (exp[1] === "ROUND") {
+      const n = exp[2];
+      const p = exp.length > 3 ? exp[3] : 0;
+      if (n == null || p == null) return null;
+      if (!isArray(n) && !isArray(p)) {
+        const d = 10 ** p;
+        const m = n * d * (1 + Number.EPSILON);
+        return Math.round(m) / d;
+      }
     }
   } else if (exp[0] === "PARAM") {
     if (exp[1] == null) return null;
@@ -169,11 +159,11 @@ export function evaluateCallback(exp: Expression): Expression {
   } else if (exp[0] === "IS NULL") {
     if (isArray(exp[1])) return exp;
     else if (exp[1] == null) return true;
-    else return null;
+    else return false;
   } else if (exp[0] === "IS NOT NULL") {
     if (isArray(exp[1])) return exp;
     else if (exp[1] != null) return true;
-    else return null;
+    else return false;
   } else if (exp[0] === "LIKE") {
     if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
     else if (
@@ -235,9 +225,12 @@ export function evaluateCallback(exp: Expression): Expression {
   } else if (exp[0] === "/") {
     return reduce(exp, (a, b, i) => {
       if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b))
+      if (isArray(a) || isArray(b)) return REDUCE_SKIP;
-        return i === 0 ? toNumber(a) / toNumber(b) : toNumber(a) * toNumber(b);
+      const n1 = toNumber(a);
-      return REDUCE_SKIP;
+      const n2 = toNumber(b);
+      if (i !== 0) return n1 * n2;
+      if (n2 === 0) return null;
+      return n1 / n2;
     });
   } else if (exp[0] === "+") {
     return reduce(exp, (a, b) => {
@@ -252,6 +245,15 @@ export function evaluateCallback(exp: Expression): Expression {
         return i === 0 ? toNumber(a) - toNumber(b) : toNumber(a) + toNumber(b);
       return REDUCE_SKIP;
     });
+  } else if (exp[0] === "%") {
+    return reduce(exp, (a, b, i) => {
+      if (a == null || b == null) return null;
+      if (isArray(a) || isArray(b) || i !== 0) return REDUCE_SKIP;
+      const n1 = toNumber(a);
+      const n2 = Math.trunc(toNumber(b));
+      if (n2 === 0) return null;
+      return n1 % n2;
+    });
   } else if (exp[0] === "||") {
     return reduce(exp, (a, b) => {
       if (a == null || b == null) return null;
@@ -266,19 +268,19 @@ export function evaluate(
   exp: Expression,
   obj: Record<string, unknown> | ((e: string) => Expression),
   now: number,
-  cb?: (e: Expression) => Expression
+  cb?: (e: Expression) => Expression,
 ): string | number | boolean | null;
 export function evaluate(
   exp: Expression,
   obj?: Record<string, unknown> | ((e: string) => Expression),
   now?: number,
-  cb?: (e: Expression) => Expression
+  cb?: (e: Expression) => Expression,
 ): Expression;
 export function evaluate(
   exp: Expression,
   obj?: Record<string, unknown> | ((e: string) => Expression),
   now?: number,
-  cb?: (e: Expression) => Expression
+  cb?: (e: Expression) => Expression,
 ): Expression {
   return map(exp, (e) => {
     if (cb) e = cb(e);
@@ -306,34 +308,26 @@ export async function evaluateAsync(
   exp: Expression,
   obj: Record<string, unknown>,
   now: number,
-  cb?: (e: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>,
 ): Promise<string | number | boolean | null>;
 export async function evaluateAsync(
   exp: Expression,
   obj?: Record<string, unknown>,
   now?: number,
-  cb?: (e: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>,
 ): Promise<Expression>;
 export async function evaluateAsync(
   exp: Expression,
   obj?: Record<string, unknown>,
   now?: number,
-  cb?: (e: Expression) => Promise<Expression>
+  cb?: (e: Expression) => Promise<Expression>,
 ): Promise<Expression> {
   return mapAsync(exp, async (e) => {
     if (cb) e = await cb(e);
     if (!isArray(e)) return e;
 
     if (e[0] === "FUNC") {
-      if (e[1] === "NOW") {
+      if (e[1] === "NOW") if (now) return now;
-        if (now) return now;
-      } else if (e[1] === "UPPER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
-      } else if (e[1] === "LOWER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
-      }
     } else if (e[0] === "PARAM") {
       if (e[1] == null) return null;
       if (obj && !isArray(e[1])) {

lib/common/memoize.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 let cache1 = new Map();
 let cache2 = new Map();
 const keys = new WeakMap();
@@ -47,9 +28,18 @@ export default function memoize<T extends (...args: any[]) => any>(func: T): T {
     if (cache1.has(key)) return cache1.get(key);
 
     let r;
-    if (cache2.has(key)) r = cache2.get(key);
+    if (cache2.has(key)) {
-    else r = func(...args);
+      cache1.set(key, (r = cache2.get(key)));
-    cache1.set(key, r);
+    } else {
+      cache1.set(key, (r = func(...args)));
+      // Evict rejected promises
+      if (r instanceof Promise) {
+        r.catch(() => {
+          cache1.delete(key);
+          cache2.delete(key);
+        });
+      }
+    }
     return r;
   }) as any;
 }

lib/common/path-set.ts

@@ -1,28 +1,9 @@
-/**
+import Path from "./path.ts";
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./path";
 
 export default class PathSet {
-  private lengthIndex: Set<Path>[];
+  private declare lengthIndex: Set<Path>[];
-  private fragmentIndex: Map<string, Set<Path>>[];
+  private declare fragmentIndex: Map<string, Set<Path>>[];
-  private stringIndex: Map<string, Path>;
+  private declare stringIndex: Map<string, Path>;
 
   public constructor() {
     this.lengthIndex = [];
@@ -75,7 +56,7 @@ export default class PathSet {
     path: Path,
     superset = false,
     subset = false,
-    depth: number = path.length
+    depth: number = path.length,
   ): Path[] {
     if (path.alias) throw new Error("PathSet does not support aliased paths");
 
@@ -115,7 +96,7 @@ export default class PathSet {
           res = [...fragmentIndexSet1, ...fragmentIndexSet2];
         } else {
           res = res.filter(
-            (r) => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
+            (r) => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r),
           );
         }
       }

lib/common/path.ts

@@ -1,66 +1,105 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 type Segments = (string | Alias)[];
 type Alias = [Path, string][];
 
 let cache1 = new Map<string, Path>();
 let cache2 = new Map<string, Path>();
 
+const CHAR_SPACE = 32;
+const CHAR_BACKSLASH = 92;
+const CHAR_COLON = 58;
+const CHAR_QUOTE = 34;
+const CHAR_OPEN_BRACKET = 91;
+const CHAR_CLOSE_BRACKET = 93;
+const CHAR_ASTERISK = 42;
+const CHAR_COMMA = 44;
+const CHAR_DOT = 46;
+
+function charCodeAt(str: string, index: number): number {
+  if (index >= str.length) return 0;
+  return str.charCodeAt(index);
+}
+
+function legalChar(c: number): boolean {
+  return (
+    (c >= 97 && c <= 122) ||
+    (c >= 65 && c <= 90) ||
+    (c >= 48 && c <= 57) ||
+    c === 95 ||
+    c === 45
+  );
+}
+
 export default class Path {
-  public readonly segments: Segments;
+  public declare readonly segments: Segments;
-  public readonly wildcard: number;
+  public declare readonly wildcard: number;
-  public readonly alias: number;
+  public declare readonly alias: number;
-  protected _string: string;
+  protected declare _string: string;
-  protected _stringIndex: number[];
+  protected declare _stringIndex: number[];
+
+  protected static parseAliasValue(
+    pattern: string,
+    index: number,
+  ): { index: number; value: string } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) === CHAR_QUOTE) {
+      for (let j = i + 1; j < pattern.length; ++j) {
+        if (
+          pattern.charCodeAt(j) === CHAR_QUOTE &&
+          pattern.charCodeAt(j - 1) !== CHAR_BACKSLASH
+        ) {
+          try {
+            ++j;
+            const v = JSON.parse(pattern.slice(i, j));
+            return { index: j, value: v };
+          } catch {
+            return { index, value: null };
+          }
+        }
+      }
+      return { index, value: null };
+    }
+
+    for (; i < pattern.length; ++i) {
+      const c = pattern.charCodeAt(i);
+      if (c === CHAR_CLOSE_BRACKET || c === CHAR_COMMA) break;
+    }
+
+    return { index: i, value: pattern.slice(index, i).trim() };
+  }
+
+  protected static parseAliasPath(
+    pattern: string,
+    index: number,
+  ): { index: number; path: Path } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    const { index: idx, segments } = Path.parsePath(pattern, i);
+    if (idx === i) return { index, path: null };
+    i = idx;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) !== CHAR_COLON) return { index, path: null };
+    return { index: i + 1, path: new Path(segments) };
+  }
 
   protected static parseAlias(
     pattern: string,
-    index: number
+    index: number,
   ): { index: number; alias: Alias } {
     const alias: Alias = [];
-    while (index < pattern.length && pattern[index] !== "]") {
+    let i = index;
-      const { index: idx, segments } = Path.parsePath(pattern, index);
+    for (;;) {
-      let j = (index = idx + 1);
+      let path: Path;
-      while (pattern[j] !== "]" && pattern[j] !== ",") {
+      let value: string;
-        if (pattern[j] === '"' && index === j) {
+      ({ index: i, path } = Path.parseAliasPath(pattern, i));
-          ++j;
+      if (!path) break;
-          while (pattern[j] !== '"' || pattern[j - 1] === "\\") {
+      ({ index: i, value } = Path.parseAliasValue(pattern, i));
-            if (++j >= pattern.length)
+      if (value == null) break;
-              throw new Error("Invalid alias expression");
+      alias.push([path, value]);
-          }
+      index = i;
-        }
-        if (++j >= pattern.length) throw new Error("Invalid alias expression");
-      }
 
-      let value = pattern.slice(index, j).trim();
+      if (charCodeAt(pattern, index) !== CHAR_COMMA) break;
-      index = j;
+      ++i;
-      if (value[0] === '"') {
-        try {
-          value = JSON.parse(value);
-        } catch (error) {
-          throw new Error("Invalid alias expression");
-        }
-      }
-
-      alias.push([new Path(segments), value]);
-      if (pattern[index] === ",") ++index;
     }
 
     // Ensure identical expressions have idential string representation
@@ -78,33 +117,29 @@ export default class Path {
 
   protected static parsePath(
     pattern: string,
-    index: number
+    index: number,
   ): { index: number; segments: Segments } {
     const segments = [];
-    // Colon separator is needed for parseAlias
-    if (index < pattern.length && pattern[index] !== ":") {
-      for (;;) {
-        if (pattern[index] === "[") {
-          const { index: idx, alias } = Path.parseAlias(pattern, index + 1);
-          index = idx + 1;
-          segments.push(alias);
-        } else {
-          const j = index;
-          while (
-            index < pattern.length &&
-            pattern[index] !== ":" &&
-            pattern[index] !== "."
-          )
-            ++index;
-          const s = pattern.slice(j, index).trim();
-          if (!s) throw new Error("Invalid parameter path");
-          segments.push(s);
-        }
 
-        if (index >= pattern.length || pattern[index] === ":") break;
+    let segStart = index;
-        else if (pattern[index] !== ".")
+    let wildcard = -1;
-          throw new Error("Invalid alias expression");
+    for (let i = index; i <= pattern.length; ++i) {
-        ++index;
+      const c = charCodeAt(pattern, i);
+      if (c === CHAR_OPEN_BRACKET && i === segStart) {
+        const { index: idx, alias } = Path.parseAlias(pattern, i + 1);
+        if (charCodeAt(pattern, idx) !== CHAR_CLOSE_BRACKET) break;
+        segments.push(alias);
+        index = i = idx + 1;
+        segStart = i + 1;
+      } else if (c === CHAR_ASTERISK && i === segStart) {
+        wildcard = i + 1;
+      } else if (!legalChar(c) || i === wildcard) {
+        if (i === segStart) break;
+        const s = pattern.slice(segStart, i);
+        segments.push(s);
+        index = i;
+        if (c !== CHAR_DOT) break;
+        segStart = i + 1;
       }
     }
 
@@ -119,7 +154,7 @@ export default class Path {
       if (Array.isArray(s)) {
         alias |= 1 << i;
         const parts = s.map(
-          (al) => `${al[0].toString()}:${JSON.stringify(al[1])}`
+          (al) => `${al[0].toString()}:${JSON.stringify(al[1])}`,
         );
         return `[${parts.join(",")}]`;
       } else if (s === "*") {
@@ -143,7 +178,8 @@ export default class Path {
     if (!path) {
       path = cache2.get(str);
       if (!path) {
-        const { segments } = Path.parsePath(str, 0);
+        const { index, segments } = Path.parsePath(str, 0);
+        if (index < str.length) throw new Error("Invalid parameter path");
         path = new Path(segments);
         if (path.toString() !== str) cache1.set(path.toString(), path);
       }

lib/common/yaml.ts

@@ -0,0 +1,218 @@
+const LINE_WIDTH = 80;
+const INDENTATION = "  ";
+
+const STRING_RESERVED = new Set([
+  "true",
+  "True",
+  "TRUE",
+  "false",
+  "False",
+  "FALSE",
+  "null",
+  "Null",
+  "NULL",
+]);
+
+function isPrintable(str: string): boolean {
+  return !/[^\t\n\x20-\x7e\x85\u{a0}-\u{d7ff}\u{e000}-\u{fffd}\u{10000}-\u{10ffff}]/u.test(
+    str,
+  );
+}
+
+function stringifyKey(str: string): string {
+  if (!str || !isPrintable(str)) return JSON.stringify(str);
+  if (/^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|[\n,[\]{}]|\s$/.test(str))
+    return JSON.stringify(str);
+  return str;
+}
+
+function foldString(str: string): string[] {
+  if (str.length <= LINE_WIDTH) return [str];
+  if (str.startsWith(" ")) return [str];
+  const lines: string[] = [];
+
+  let idx = 0;
+  let cand = 0;
+  for (let i = 1; i < str.length - 1; ++i) {
+    if (str[i] !== " ") continue;
+
+    if (str[i + 1] === " ") {
+      i += 2;
+      while (str[i] === " ") ++i;
+      continue;
+    }
+
+    if (i <= idx + LINE_WIDTH) {
+      cand = i;
+      continue;
+    }
+
+    const c = cand > idx ? cand : i;
+    lines.push(str.slice(idx, c));
+    idx = c + 1;
+    cand = i;
+  }
+
+  if (cand > idx && str.length > idx + LINE_WIDTH) {
+    lines.push(str.slice(idx, cand));
+    idx = cand + 1;
+  }
+
+  lines.push(str.slice(idx));
+
+  return lines;
+}
+
+function stringifyString(str: string, res: string[], prefix1, prefix2): void {
+  if (/^\s*$/.test(str) || STRING_RESERVED.has(str) || !isPrintable(str)) {
+    res.push(prefix1 + JSON.stringify(str));
+    return;
+  }
+
+  if (!prefix2) prefix2 = INDENTATION;
+
+  const lines = str.split("\n");
+  if (lines.length > 1) {
+    let idt = "";
+    let chmp = "-";
+    if ((lines.find((l) => l) || "").startsWith(" "))
+      idt = `${INDENTATION.length}`;
+
+    if (!lines[lines.length - 1]) {
+      lines.pop();
+      if (lines[lines.length - 1]) chmp = "";
+      else chmp = "+";
+    }
+
+    if (/^\s+$/.test(lines[lines.length - 1])) {
+      res.push(prefix1 + JSON.stringify(str));
+      return;
+    }
+
+    let isFolded = false;
+    const folded = lines.map((l) => {
+      const ls = foldString(l);
+      if (ls.length > 1) isFolded = true;
+      return ls;
+    });
+
+    if (!isFolded) {
+      res.push(
+        `${prefix1}|${idt}${chmp}`,
+        ...lines.map((l) => (l ? prefix2 + l : l)),
+      );
+      return;
+    }
+
+    res.push(`${prefix1}>${idt}${chmp}`);
+    res.push(...folded[0].map((f) => prefix2 + f));
+    for (let i = 1; i < folded.length; ++i) {
+      const prevLine = folded[i - 1][0];
+      if (prevLine && !folded[i - 1][0].startsWith(" ")) res.push("");
+      res.push(...folded[i].map((f) => prefix2 + f));
+    }
+    return;
+  }
+
+  if (
+    /^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|\s$/.test(str) ||
+    parseFloat(str) === +str
+  ) {
+    res.push(prefix1 + JSON.stringify(str));
+    return;
+  }
+
+  res.push(prefix1 + str);
+}
+
+function stringifyAny(
+  obj: unknown,
+  res: string[],
+  prefix1 = "",
+  prefix2 = "",
+): void {
+  if (obj == null) {
+    res.push(`${prefix1}null`);
+    return;
+  }
+  if (typeof obj === "number" || typeof obj === "boolean") {
+    res.push(`${prefix1}${JSON.stringify(obj)}`);
+    return;
+  }
+  if (obj instanceof Date) {
+    res.push(`${prefix1}${obj.toJSON()}`);
+    return;
+  }
+
+  if (typeof obj === "string") {
+    stringifyString(obj, res, prefix1, prefix2);
+    return;
+  }
+
+  if (Array.isArray(obj)) {
+    if (!obj.length) {
+      res.push(prefix1 + "[]");
+      return;
+    }
+
+    if (!prefix1 || prefix1.endsWith("- ")) {
+      stringifyAny(obj[0], res, prefix1 + "- ", prefix2 + INDENTATION);
+      prefix1 = prefix2 + "- ";
+      prefix2 = prefix2 + INDENTATION;
+      for (let i = 1; i < obj.length; ++i)
+        stringifyAny(obj[i], res, prefix1, prefix2);
+    } else {
+      res.push(prefix1);
+      prefix1 = prefix2 + "- ";
+      prefix2 = prefix2 + INDENTATION;
+      for (let i = 0; i < obj.length; ++i)
+        stringifyAny(obj[i], res, prefix1, prefix2);
+    }
+    return;
+  }
+
+  const entries = Object.entries(obj).filter((e) => e[1] !== undefined);
+
+  if (!entries.length) {
+    res.push(prefix1 + "{}");
+    return;
+  }
+
+  if (!prefix1 || prefix1.endsWith("- ")) {
+    stringifyAny(
+      entries[0][1],
+      res,
+      prefix1 + `${stringifyKey(entries[0][0])}: `,
+      prefix2 + INDENTATION,
+    );
+    prefix1 = prefix2;
+    prefix2 = prefix2 + INDENTATION;
+    for (let i = 1; i < entries.length; ++i) {
+      stringifyAny(
+        entries[i][1],
+        res,
+        prefix1 + `${stringifyKey(entries[i][0])}: `,
+        prefix2,
+      );
+    }
+  } else {
+    res.push(prefix1);
+    prefix1 = prefix2;
+    prefix2 = prefix2 + INDENTATION;
+    for (let i = 0; i < entries.length; ++i) {
+      stringifyAny(
+        entries[i][1],
+        res,
+        prefix1 + `${stringifyKey(entries[i][0])}: `,
+        prefix2,
+      );
+    }
+  }
+}
+
+export function stringify(obj: unknown): string {
+  if (obj === undefined) return undefined;
+  const lines: string[] = [];
+  stringifyAny(obj, lines);
+  return lines.join("\n") + "\n";
+}

lib/config.ts

@@ -1,24 +1,5 @@
-/**
+import { resolve } from "node:path";
- * Copyright 2013-2019  GenieACS Inc.
+import { readFileSync, existsSync } from "node:fs";
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { resolve } from "path";
-import { readFileSync, existsSync } from "fs";
 
 // Find project root directory
 export let ROOT_DIR = resolve(__dirname, "..");
@@ -43,7 +24,7 @@ const options = {
 
   CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
   CWMP_PORT: { type: "int", default: 7547 },
-  CWMP_INTERFACE: { type: "string", default: "0.0.0.0" },
+  CWMP_INTERFACE: { type: "string", default: "::" },
   CWMP_SSL_CERT: { type: "string", default: "" },
   CWMP_SSL_KEY: { type: "string", default: "" },
   CWMP_LOG_FILE: { type: "path", default: "" },
@@ -51,7 +32,7 @@ const options = {
 
   NBI_WORKER_PROCESSES: { type: "int", default: 0 },
   NBI_PORT: { type: "int", default: 7557 },
-  NBI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  NBI_INTERFACE: { type: "string", default: "::" },
   NBI_SSL_CERT: { type: "string", default: "" },
   NBI_SSL_KEY: { type: "string", default: "" },
   NBI_LOG_FILE: { type: "path", default: "" },
@@ -59,7 +40,7 @@ const options = {
 
   FS_WORKER_PROCESSES: { type: "int", default: 0 },
   FS_PORT: { type: "int", default: 7567 },
-  FS_INTERFACE: { type: "string", default: "0.0.0.0" },
+  FS_INTERFACE: { type: "string", default: "::" },
   FS_SSL_CERT: { type: "string", default: "" },
   FS_SSL_KEY: { type: "string", default: "" },
   FS_URL_PREFIX: { type: "string", default: "" },
@@ -68,7 +49,7 @@ const options = {
 
   UI_WORKER_PROCESSES: { type: "int", default: 0 },
   UI_PORT: { type: "int", default: 3000 },
-  UI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  UI_INTERFACE: { type: "string", default: "::" },
   UI_SSL_CERT: { type: "string", default: "" },
   UI_SSL_KEY: { type: "string", default: "" },
   UI_LOG_FILE: { type: "path", default: "" },
@@ -101,6 +82,9 @@ const options = {
 
   // Should probably never be changed
   DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
+
+  XMPP_JID: { type: "string", default: "" },
+  XMPP_PASSWORD: { type: "string", default: "" },
 };
 
 const allConfig: { [name: string]: string | number } = {};
@@ -245,7 +229,7 @@ if (fsHostname) {
   const FS_SSL = !!allConfig["FS_SSL_CERT"];
   setConfig(
     "FS_URL_PREFIX",
-    (FS_SSL ? "https" : "http") + `://${fsHostname}:${FS_PORT}/`
+    (FS_SSL ? "https" : "http") + `://${fsHostname}:${FS_PORT}/`,
   );
 }
 
@@ -255,7 +239,7 @@ for (const [k, v] of Object.entries(options))
 
 export function get(
   optionName: string,
-  deviceId?: string
+  deviceId?: string,
 ): string | number | boolean {
   if (!deviceId) return allConfig[optionName];
 

lib/connection-request.ts

@@ -1,35 +1,19 @@
-/**
+import * as crypto from "node:crypto";
- * Copyright 2013-2019  GenieACS Inc.
+import * as dgram from "node:dgram";
- *
+import * as http from "node:http";
- * This file is part of GenieACS.
+import { evaluateAsync } from "./common/expression/util.ts";
- *
+import { Expression } from "./types.ts";
- * GenieACS is free software: you can redistribute it and/or modify
+import * as auth from "./auth.ts";
- * it under the terms of the GNU Affero General Public License as
+import * as extensions from "./extensions.ts";
- * published by the Free Software Foundation, either version 3 of the
+import * as debug from "./debug.ts";
- * License, or (at your option) any later version.
+import XmppClient from "./xmpp-client.ts";
- *
+import * as config from "../lib/config.ts";
- * GenieACS is distributed in the hope that it will be useful,
+import { encodeEntities, parseAttrs, Element } from "./xml-parser.ts";
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
+import * as logger from "../lib/logger.ts";
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as crypto from "crypto";
-import * as dgram from "dgram";
-import { parse } from "url";
-import * as http from "http";
-import { evaluateAsync } from "./common/expression";
-import { Expression } from "./types";
-import * as auth from "./auth";
-import * as extensions from "./extensions";
-import * as debug from "./debug";
 
 async function extractAuth(
   exp: Expression,
-  dflt: any
+  dflt: any,
 ): Promise<[string, string, Expression]> {
   let username, password;
   const _exp = await evaluateAsync(
@@ -55,30 +39,32 @@ async function extractAuth(
         }
       }
       return e;
-    }
+    },
   );
   return [username, password, _exp];
 }
 
 function httpGet(
+  url: URL,
   options: http.RequestOptions,
   _debug: boolean,
-  deviceId: string
+  deviceId: string,
 ): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
   return new Promise((resolve, reject) => {
     const req = http
-      .get(options, (res) => {
+      .get(url, options, (res) => {
         res.resume();
         resolve({ statusCode: res.statusCode, headers: res.headers });
         if (_debug) {
-          debug.outgoingHttpRequest(req, deviceId, options, null);
+          debug.outgoingHttpRequest(req, deviceId, "GET", url, null);
           debug.incomingHttpResponse(res, deviceId, null);
         }
       })
       .on("error", (err) => {
         req.destroy();
         reject(err);
-        if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
+        if (_debug)
+          debug.outgoingHttpRequestError(req, deviceId, "GET", url, err);
       })
       .on("timeout", () => {
         req.destroy();
@@ -92,17 +78,15 @@ export async function httpConnectionRequest(
   allowBasicAuth: boolean,
   timeout: number,
   _debug: boolean,
-  deviceId: string
+  deviceId: string,
 ): Promise<string> {
-  const options: http.RequestOptions = parse(address);
+  const url = new URL(address);
-  if (options.protocol !== "http:")
+  if (url.protocol !== "http:")
     return "Invalid connection request URL or protocol";
 
-  options.agent = new http.Agent({
+  const options: http.RequestOptions = {
-    maxSockets: 1,
+    agent: new http.Agent({ maxSockets: 1, keepAlive: true, timeout: timeout }),
-    keepAlive: true,
+  };
-    timeout: timeout,
-  });
 
   let authHeader: Record<string, string>;
   let username: string;
@@ -120,7 +104,7 @@ export async function httpConnectionRequest(
               Authorization: auth.basic(username || "", password || ""),
             },
           },
-          options
+          options,
         );
       } else if (authHeader["method"] === "Digest") {
         opts = Object.assign(
@@ -129,14 +113,14 @@ export async function httpConnectionRequest(
               Authorization: auth.solveDigest(
                 username,
                 password,
-                options.path,
+                url.pathname + url.search,
                 "GET",
                 null,
-                authHeader
+                authHeader,
               ),
             },
           },
-          options
+          options,
         );
       } else {
         return "Unrecognized auth method";
@@ -145,12 +129,12 @@ export async function httpConnectionRequest(
 
     let res: { statusCode: number; headers: http.IncomingHttpHeaders };
     try {
-      res = await httpGet(opts, _debug, deviceId);
+      res = await httpGet(url, opts, _debug, deviceId);
     } catch (err) {
       // Workaround for some devices unexpectedly closing the connection
       if (authHeader) {
         try {
-          res = await httpGet(opts, _debug, deviceId);
+          res = await httpGet(url, opts, _debug, deviceId);
         } catch (err) {
           return `Connection request error: ${err.message}`;
         }
@@ -169,9 +153,13 @@ export async function httpConnectionRequest(
     if (res.statusCode === 503) return "Device is offline";
 
     if (res.statusCode === 401 && res.headers["www-authenticate"]) {
+      try {
         authHeader = auth.parseWwwAuthenticateHeader(
-        res.headers["www-authenticate"]
+          res.headers["www-authenticate"],
         );
+      } catch (err) {
+        return "Connection request error: Error parsing www-authenticate header";
+      }
       [username, password, authExp] = await extractAuth(authExp, false);
     } else {
       return `Connection request error: Unexpected status code ${res.statusCode}`;
@@ -182,14 +170,13 @@ export async function httpConnectionRequest(
 }
 
 export async function udpConnectionRequest(
-  address: string,
+  host: string,
+  port: number,
   authExp: Expression,
   sourcePort = 0,
   _debug: boolean,
-  deviceId: string
+  deviceId: string,
 ): Promise<void> {
-  const [host, portStr] = address.split(":", 2);
-  const port = portStr ? parseInt(portStr) : 80;
   const now = Date.now();
 
   const client = dgram.createSocket({ type: "udp4", reuseAddr: true });
@@ -216,12 +203,12 @@ export async function udpConnectionRequest(
       .createHmac("sha1", password)
       .update(`${ts}${id}${username}${cn}`)
       .digest("hex");
-    const uri = `http://${address}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
+    const uri = `http://${host}:${port}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
-    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${address}\r\n\r\n`;
+    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${host}:${port}\r\n\r\n`;
     const message = Buffer.from(msg);
 
     for (let i = 0; i < 3; ++i) {
-      await new Promise((resolve, reject) => {
+      await new Promise<void>((resolve, reject) => {
         client.send(message, 0, message.length, port, host, (err: Error) => {
           if (err) reject(err);
           else resolve();
@@ -234,3 +221,84 @@ export async function udpConnectionRequest(
   }
   client.close();
 }
+
+const XMPP_JID = config.get("XMPP_JID") as string;
+const XMPP_PASSWORD = config.get("XMPP_PASSWORD") as string;
+const XMPP_RESOURCE = crypto.randomBytes(8).toString("hex");
+
+let xmppClient: XmppClient;
+
+function xmppClientOnError(err: Error): void {
+  xmppClient = null;
+  logger.error({
+    message: "XMPP exception",
+    exception: err,
+    pid: process.pid,
+  });
+}
+
+function xmppClientOnClose(): void {
+  xmppClient = null;
+}
+
+export async function xmppConnectionRequest(
+  jid: string,
+  authExp: Expression,
+  timeout: number,
+  _debug: boolean,
+  deviceId: string,
+): Promise<string> {
+  if (!xmppClient) {
+    const [host, username] = XMPP_JID.split("@").reverse();
+    xmppClient = await XmppClient.connect({
+      host,
+      username,
+      resource: XMPP_RESOURCE,
+      password: XMPP_PASSWORD,
+      timeout: 120000,
+    });
+    xmppClient.on("error", xmppClientOnError);
+    xmppClient.on("close", xmppClientOnClose);
+    xmppClient.unref();
+  }
+
+  let username: string;
+  let password: string;
+
+  [username, password, authExp] = await extractAuth(authExp, null);
+  while (username != null && password != null) {
+    const msg = `<connectionRequest xmlns="urn:broadband-forum-org:cwmp:xmppConnReq-1-0"><username>${encodeEntities(
+      username,
+    )}</username><password>${encodeEntities(
+      password,
+    )}</password></connectionRequest>`;
+    let res: Element, rawRes: string, rawReq: string;
+    try {
+      ({ res, rawRes, rawReq } = await xmppClient.sendIqStanza(
+        XMPP_JID,
+        jid,
+        "get",
+        msg,
+        timeout,
+      ));
+    } catch (err) {
+      return err.message;
+    }
+    if (_debug) {
+      debug.outgoingXmppStanza(deviceId, rawReq);
+      debug.incomingXmppStanza(deviceId, rawRes);
+    }
+    const attrs = parseAttrs(res.attrs);
+    const type = attrs.find((a) => a.name === "type");
+    if (type && type.value === "result") return "";
+    const error = res.children.find((c) => c.name === "error");
+    if (!error || !error.children[0])
+      return "Unexpected XMPP connection request response";
+    if (error.children[0].name === "service-unavailable")
+      return "Device is offline";
+    if (error.children[0].name !== "not-authorized")
+      return "Unexpected XMPP connection request response";
+    [username, password, authExp] = await extractAuth(authExp, null);
+  }
+  return "Incorrect connection request credentials";
+}

lib/cwmp/db.ts

@@ -1,50 +1,18 @@
-/**
+import { ObjectId } from "mongodb";
- * Copyright 2013-2019  GenieACS Inc.
+import { decodeTag, encodeTag, escapeRegExp } from "../util.ts";
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { MongoClient, ObjectID, Collection, Db } from "mongodb";
-import { get } from "./config";
-import { encodeTag, escapeRegExp } from "./common";
-import { parse } from "./common/expression-parser";
 import {
   DeviceData,
   Attributes,
   SessionFault,
   Task,
   Operation,
-  Expression,
+} from "../types.ts";
-} from "./types";
+import Path from "../common/path.ts";
-import Path from "./common/path";
+import { collections } from "../db/db.ts";
+import { optimizeProjection } from "../db/util.ts";
+import * as MongoTypes from "../db/types.ts";
 
-export let tasksCollection: Collection,
+const INVALID_PATH_SUFFIX = "__invalid";
-  devicesCollection: Collection,
-  presetsCollection: Collection,
-  objectsCollection: Collection,
-  provisionsCollection: Collection,
-  virtualParametersCollection: Collection,
-  faultsCollection: Collection,
-  filesCollection: Collection,
-  operationsCollection: Collection,
-  permissionsCollection: Collection,
-  usersCollection: Collection,
-  configCollection: Collection;
-
-let clientPromise: Promise<MongoClient>;
 
 function compareAccessLists(list1: string[], list2: string[]): boolean {
   if (list1.length !== list2.length) return false;
@@ -52,68 +20,9 @@ function compareAccessLists(list1: string[], list2: string[]): boolean {
   return true;
 }
 
-const onConnectCallbacks: ((db: Db) => Promise<void>)[] = [];
-
-export function onConnect(callback: (db: Db) => Promise<void>): void {
-  onConnectCallbacks.push(callback);
-}
-
-export async function connect(): Promise<void> {
-  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
-    useNewUrlParser: true,
-    useUnifiedTopology: true,
-  });
-
-  const client = await clientPromise;
-  const db = client.db();
-  await Promise.all(onConnectCallbacks.map((c) => c(db)));
-}
-
-onConnect(async (db) => {
-  tasksCollection = db.collection("tasks");
-  await tasksCollection.createIndex({ device: 1, timestamp: 1 });
-
-  devicesCollection = db.collection("devices");
-  presetsCollection = db.collection("presets");
-  objectsCollection = db.collection("objects");
-  filesCollection = db.collection("fs.files");
-  provisionsCollection = db.collection("provisions");
-  virtualParametersCollection = db.collection("virtualParameters");
-  faultsCollection = db.collection("faults");
-  operationsCollection = db.collection("operations");
-  permissionsCollection = db.collection("permissions");
-  usersCollection = db.collection("users");
-  configCollection = db.collection("config");
-});
-
-export async function disconnect(): Promise<void> {
-  if (clientPromise) await (await clientPromise).close();
-}
-
-// Optimize projection by removing overlaps
-// This can modify the object
-function optimizeProjection(obj: { [path: string]: 1 }): { [path: string]: 1 } {
-  if (obj[""]) return { "": obj[""] };
-
-  const keys = Object.keys(obj).sort();
-  if (keys.length <= 1) return obj;
-
-  for (let i = 1; i < keys.length; ++i) {
-    const a = keys[i - 1];
-    const b = keys[i];
-    if (b.startsWith(a)) {
-      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
-        delete obj[b];
-        keys.splice(i--, 1);
-      }
-    }
-  }
-  return obj;
-}
-
 export async function fetchDevice(
   id: string,
-  timestamp: number
+  timestamp: number,
 ): Promise<[Path, number, Attributes?][]> {
   const res: [Path, number, Attributes?][] = [
     [
@@ -128,10 +37,15 @@ export async function fetchDevice(
     ],
   ];
 
-  const device = await devicesCollection.findOne({ _id: id });
+  const device = await collections.devices.findOne({ _id: id });
   if (!device) return null;
 
-  function storeParams(obj, path: string, pathLength: number, ts): void {
+  function storeParams(
+    obj,
+    path: string,
+    pathLength: number,
+    ts: number,
+  ): void {
     if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
     if (obj["_attributesTimestamp"])
       obj["_attributesTimestamp"] = +obj["_attributesTimestamp"];
@@ -142,7 +56,7 @@ export async function fetchDevice(
 
     if (obj["_value"] != null) {
       attrs.value = [obj["_timestamp"] || 1, [obj["_value"], obj["_type"]]];
-      if (obj["_type"] === "xsd:dateTime")
+      if (obj["_type"] === "xsd:dateTime" && obj["_value"] instanceof Date)
         attrs.value[1][0] = +attrs.value[1][0];
 
       obj["_object"] = false;
@@ -162,19 +76,34 @@ export async function fetchDevice(
     if (obj["_accessList"] != null)
       attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
 
-    res.push([Path.parse(path.slice(0, -1)), t, attrs]);
+    try {
+      res.push([Path.parse(path), t, attrs]);
+    } catch (err) {
+      // The path parser is now more strict so we might be in a situation where
+      // the database contains invalid paths from before this change So here we
+      // encode the invalid characters.
+      const splits = path.split(".");
+      splits[splits.length - 1] =
+        encodeTag(splits[splits.length - 1]) + INVALID_PATH_SUFFIX;
+      path = splits.join(".");
+      res.push([Path.parse(path), t, attrs]);
+      return;
+    }
 
     for (const [k, v] of Object.entries(obj)) {
       if (!k.startsWith("_")) {
         obj["_object"] = true;
-        storeParams(v, path + k + ".", pathLength + 1, obj["_timestamp"]);
+        storeParams(v, `${path}.${k}`, pathLength + 1, obj["_timestamp"]);
       }
     }
 
-    if ((obj["_object"] || !pathLength) && obj["_timestamp"])
+    if (obj["_object"] && obj["_timestamp"])
-      res.push([Path.parse(path + "*"), obj["_timestamp"]]);
+      res.push([Path.parse(path + ".*"), obj["_timestamp"]]);
   }
 
+  const ts: number = +device["_timestamp"] || 0;
+  if (ts) res.push([Path.parse("*"), ts]);
+
   for (const [k, v] of Object.entries(device)) {
     switch (k) {
       case "_lastInform":
@@ -302,10 +231,11 @@ export async function fetchDevice(
             },
           ]);
         }
+        break;
+      default:
+        if (!k.startsWith("_")) storeParams(v, k, 1, ts);
     }
   }
-
-  storeParams(device, "", 0, 0);
   return res;
 }
 
@@ -313,7 +243,7 @@ export async function saveDevice(
   deviceId: string,
   deviceData: DeviceData,
   isNew: boolean,
-  sessionTimestamp: number
+  sessionTimestamp: number,
 ): Promise<void> {
   const update = { $set: {}, $unset: {}, $addToSet: {}, $pull: {} };
 
@@ -426,21 +356,37 @@ export async function saveDevice(
           if (value2 != null) {
             if (!update["$addToSet"]["_tags"])
               update["$addToSet"]["_tags"] = { $each: [] };
-            update["$addToSet"]["_tags"]["$each"].push(path.segments[1]);
+            update["$addToSet"]["_tags"]["$each"].push(
+              decodeTag(path.segments[1] as string),
+            );
           } else {
             if (!update["$pull"]["_tags"]) {
               update["$pull"]["_tags"] = {
                 $in: [],
               };
             }
-            update["$pull"]["_tags"]["$in"].push(path.segments[1]);
+            update["$pull"]["_tags"]["$in"].push(
+              decodeTag(path.segments[1] as string),
+            );
           }
         }
 
         break;
       default:
         if (!diff[2]) {
-          update["$unset"][path.toString()] = 1;
+          let pathStr = path.toString();
+          // Paths with that suffix are encoded and need to be decoded
+          if (pathStr.endsWith(INVALID_PATH_SUFFIX)) {
+            const splits = pathStr.split(".");
+            splits[splits.length - 1] = decodeTag(
+              splits[splits.length - 1].slice(
+                0,
+                0 - INVALID_PATH_SUFFIX.length,
+              ),
+            );
+            pathStr = splits.join(".");
+          }
+          update["$unset"][pathStr] = 1;
           continue;
         }
 
@@ -458,7 +404,7 @@ export async function saveDevice(
                     Number.isInteger(value2 as number)
                   ) {
                     update["$set"][path.toString() + "._value"] = new Date(
-                      value2 as number
+                      value2 as number,
                     );
                   } else {
                     update["$set"][path.toString() + "._value"] = value2;
@@ -470,13 +416,13 @@ export async function saveDevice(
 
                 if (valueTimestamp2 !== valueTimestamp1) {
                   update["$set"][path.toString() + "._timestamp"] = new Date(
-                    valueTimestamp2
+                    valueTimestamp2,
                   );
                 }
 
                 break;
               case "object":
-                if (!diff[1] || !diff[1].object || object2 !== object1) {
+                if (!diff[1]?.object || object2 !== object1) {
                   update["$set"][
                     path.length ? path.toString() + "._object" : "_object"
                   ] = !!object2;
@@ -484,7 +430,7 @@ export async function saveDevice(
 
                 break;
               case "writable":
-                if (!diff[1] || !diff[1].writable || writable2 !== writable1) {
+                if (!diff[1]?.writable || writable2 !== writable1) {
                   update["$set"][
                     path.length ? path.toString() + "._writable" : "_writable"
                   ] = !!writable2;
@@ -505,9 +451,8 @@ export async function saveDevice(
                 }
 
                 if (attributesTimestamp2 !== attributesTimestamp1) {
-                  update["$set"][
+                  update["$set"][path.toString() + "._attributesTimestamp"] =
-                    path.toString() + "._attributesTimestamp"
+                    new Date(attributesTimestamp2);
-                  ] = new Date(attributesTimestamp2);
                 }
 
                 break;
@@ -525,9 +470,8 @@ export async function saveDevice(
                 }
 
                 if (attributesTimestamp2 !== attributesTimestamp1) {
-                  update["$set"][
+                  update["$set"][path.toString() + "._attributesTimestamp"] =
-                    path.toString() + "._attributesTimestamp"
+                    new Date(attributesTimestamp2);
-                  ] = new Date(attributesTimestamp2);
                 }
             }
           }
@@ -537,7 +481,7 @@ export async function saveDevice(
           for (const attrName of Object.keys(diff[1])) {
             if (
               diff[1][attrName][1] != null &&
-              (!diff[2] || !diff[2][attrName] || diff[2][attrName][1] == null)
+              diff[2]?.[attrName]?.[1] == null
             ) {
               const p = path.length ? path.toString() + "." : "";
               update["$unset"][`${p}_${attrName}`] = 1;
@@ -587,35 +531,43 @@ export async function saveDevice(
     delete update["$pull"];
   }
 
-  const result = await devicesCollection.updateOne({ _id: deviceId }, update, {
+  const result = await collections.devices.updateOne(
+    { _id: deviceId },
+    update,
+    {
       upsert: isNew,
-  });
+    },
+  );
 
-  if (result.result.n !== 1)
+  if (!result.matchedCount && !result.upsertedCount)
     throw new Error(`Device ${deviceId} not found in database`);
 
   if (update2) {
-    await devicesCollection.updateOne({ _id: deviceId }, update2);
+    await collections.devices.updateOne({ _id: deviceId }, update2);
     return;
   }
 }
 
 export async function getFaults(
-  deviceId: string
+  deviceId: string,
 ): Promise<{ [channel: string]: SessionFault }> {
-  const res = await faultsCollection
+  const res = await collections.faults
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
     .toArray();
 
   const faults: { [channel: string]: SessionFault } = {};
   for (const r of res) {
     const channel = r._id.slice(deviceId.length + 1);
-    delete r._id;
+    const fault: SessionFault = {
-    delete r.channel;
+      code: r.code,
-    delete r.device;
+      message: r.message,
-    r.timestamp = +r.timestamp;
+      ...(r.detail && { detail: r.detail }),
-    r.provisions = JSON.parse(r.provisions);
+      timestamp: +r.timestamp,
-    faults[channel] = r;
+      provisions: JSON.parse(r.provisions),
+      retries: r.retries,
+      ...(r.expiry && { expiry: +r.expiry }),
+    };
+    faults[channel] = fault;
   }
 
   return faults;
@@ -624,48 +576,83 @@ export async function getFaults(
 export async function saveFault(
   deviceId: string,
   channel: string,
-  fault: SessionFault
+  fault: SessionFault,
 ): Promise<void> {
   const id = `${deviceId}:${channel}`;
-  const f = Object.assign({}, fault) as Record<string, any>;
+  const f: MongoTypes.Fault = {
-  f["_id"] = id;
+    _id: id,
-  f["device"] = deviceId;
+    device: deviceId,
-  f["channel"] = channel;
+    channel: channel,
-  f["timestamp"] = new Date(fault.timestamp);
+    timestamp: new Date(fault.timestamp),
-  f["provisions"] = JSON.stringify(fault.provisions);
+    code: fault.code,
-  await faultsCollection.replaceOne({ _id: id }, f, { upsert: true });
+    message: fault.message,
+    ...(fault.detail && { detail: fault.detail }),
+    retries: fault.retries,
+    ...(fault.expiry && { expiry: new Date(fault.expiry) }),
+    provisions: JSON.stringify(fault.provisions),
+  };
+  await collections.faults.replaceOne({ _id: id }, f, { upsert: true });
 }
 
 export async function deleteFault(
   deviceId: string,
-  channel: string
+  channel: string,
 ): Promise<void> {
-  await faultsCollection.deleteOne({ _id: `${deviceId}:${channel}` });
+  await collections.faults.deleteOne({ _id: `${deviceId}:${channel}` });
 }
 
 export async function getDueTasks(
   deviceId: string,
-  timestamp: number
+  timestamp: number,
 ): Promise<[Task[], number]> {
-  const cur = tasksCollection.find({ device: deviceId }).sort({ timestamp: 1 });
+  const cur = collections.tasks
+    .find({ device: deviceId })
+    .sort({ timestamp: 1 });
   const tasks = [] as Task[];
 
-  for await (const task of cur) {
+  for await (const t of cur) {
-    if (task.timestamp) task.timestamp = +task.timestamp;
+    if (+t.timestamp >= timestamp) return [tasks, +t.timestamp];
-    if (task.expiry) task.expiry = +task.expiry;
+    const task: Task = {
-    if (task.timestamp >= timestamp) return [tasks, +task.timestamp];
+      _id: t._id.toString(),
-    task._id = String(task._id);
+      name: t.name,
+      ...(t.timestamp && { timestamp: +t.timestamp }),
+      ...(t.expiry && { expiry: +t.expiry }),
+      ...(t.name === "getParameterValues" && {
+        parameterNames: t.parameterNames,
+      }),
+      ...(t.name === "setParameterValues" && {
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "refreshObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "download" && {
+        fileType: t.fileType,
+        fileName: t.fileName,
+        targetFileName: t.targetFileName,
+      }),
+      ...(t.name === "addObject" && {
+        objectName: t.objectName,
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "deleteObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "provisions" && {
+        provisions: t.provisions,
+      }),
+    };
 
     tasks.push(task);
 
     // For API compatibility
-    if (task.name === "download" && task.file) {
+    if (task.name === "download" && t["file"]) {
       let q;
-      if (ObjectID.isValid(task.file))
+      if (ObjectId.isValid(t["file"]))
-        q = { _id: { $in: [task.file, new ObjectID(task.file)] } };
+        q = { _id: { $in: [t["file"], new ObjectId(t["file"])] } };
-      else q = { _id: task.file };
+      else q = { _id: t["file"] };
 
-      const res = await filesCollection.find(q).toArray();
+      const res = await collections.files.find(q).toArray();
 
       if (res[0]) {
         if (!task.fileType) task.fileType = res[0].metadata.fileType;
@@ -680,34 +667,39 @@ export async function getDueTasks(
 
 export async function clearTasks(
   deviceId: string,
-  taskIds: string[]
+  taskIds: string[],
 ): Promise<void> {
-  await tasksCollection.deleteMany({
+  await collections.tasks.deleteMany({
-    _id: { $in: taskIds.map((id) => new ObjectID(id)) },
+    _id: { $in: taskIds.map((id) => new ObjectId(id)) },
   });
 }
 
 export async function getOperations(
-  deviceId: string
+  deviceId: string,
 ): Promise<{ [commandKey: string]: Operation }> {
-  const res = await operationsCollection
+  const res = await collections.operations
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
     .toArray();
 
   const operations: { [commandKey: string]: Operation } = {};
   for (const r of res) {
     const commandKey = r._id.slice(deviceId.length + 1);
-    delete r._id;
     // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
     if (typeof r.provisions !== "string") {
-      operations[commandKey] = r;
+      delete r._id;
+      operations[commandKey] = r as unknown as Operation;
       continue;
     }
-    r.timestamp = +r.timestamp;
+    const operation: Operation = {
-    if (r.args) r.args = JSON.parse(r.args);
+      name: r.name,
-    r.provisions = JSON.parse(r.provisions);
+      timestamp: +r.timestamp,
-    r.retries = JSON.parse(r.retries);
+      channels:
-    operations[commandKey] = r;
+        typeof r.channels === "string" ? JSON.parse(r.channels) : r.channels,
+      retries: JSON.parse(r.retries),
+      provisions: JSON.parse(r.provisions),
+      ...(r.args && { args: JSON.parse(r.args) }),
+    };
+    operations[commandKey] = operation;
   }
   return operations;
 }
@@ -715,76 +707,26 @@ export async function getOperations(
 export async function saveOperation(
   deviceId: string,
   commandKey: string,
-  operation: Operation
+  operation: Operation,
 ): Promise<void> {
   const id = `${deviceId}:${commandKey}`;
-  const o = Object.assign({}, operation) as Record<string, any>;
+  const o: MongoTypes.Operation = {
-  o["_id"] = id;
+    _id: id,
-  o["timestamp"] = new Date(operation.timestamp);
+    name: operation.name,
-  o["provisions"] = JSON.stringify(operation.provisions);
+    timestamp: new Date(operation.timestamp),
-  o["retries"] = JSON.stringify(operation.retries);
+    channels: JSON.stringify(operation.channels),
-  o["args"] = JSON.stringify(operation.args);
+    provisions: JSON.stringify(operation.provisions),
-  await operationsCollection.replaceOne({ _id: id }, o, {
+    retries: JSON.stringify(operation.retries),
+    args: JSON.stringify(operation.args),
+  };
+  await collections.operations.replaceOne({ _id: id }, o, {
     upsert: true,
   });
 }
 
 export async function deleteOperation(
   deviceId: string,
-  commandKey: string
+  commandKey: string,
 ): Promise<void> {
-  await operationsCollection.deleteOne({ _id: `${deviceId}:${commandKey}` });
+  await collections.operations.deleteOne({ _id: `${deviceId}:${commandKey}` });
-}
-
-export async function getPresets(): Promise<Record<string, any>[]> {
-  return presetsCollection.find().toArray();
-}
-
-export async function getObjects(): Promise<Record<string, any>[]> {
-  return objectsCollection.find().toArray();
-}
-
-export async function getProvisions(): Promise<Record<string, any>[]> {
-  return provisionsCollection.find().toArray();
-}
-
-export async function getVirtualParameters(): Promise<Record<string, any>[]> {
-  return virtualParametersCollection.find().toArray();
-}
-
-export function getFiles(): Promise<Record<string, any>[]> {
-  return filesCollection.find().toArray();
-}
-
-export async function getConfig(): Promise<
-  { id: string; value: Expression }[]
-> {
-  const res = await configCollection.find().toArray();
-  return res.map((c) => ({
-    id: c["_id"],
-    value: parse(c["value"]),
-  }));
-}
-
-interface Permission {
-  role: string;
-  resource: string;
-  access: number;
-  filter: string;
-  validate: string;
-}
-
-export async function getPermissions(): Promise<Permission[]> {
-  return permissionsCollection.find().toArray();
-}
-
-interface User {
-  _id: string;
-  password: string;
-  salt: string;
-  roles: string;
-}
-
-export async function getUsers(): Promise<User[]> {
-  return usersCollection.find().toArray();
 }

lib/cwmp/local-cache.ts

@@ -0,0 +1,357 @@
+import * as vm from "node:vm";
+import * as crypto from "node:crypto";
+import * as config from "../config.ts";
+import { collections } from "../db/db.ts";
+import { convertOldPrecondition } from "../db/util.ts";
+import * as logger from "../logger.ts";
+import * as scheduling from "../scheduling.ts";
+import * as expression from "../common/expression/util.ts";
+import { parse } from "../common/expression/parser.ts";
+import {
+  Preset,
+  Expression,
+  Provisions,
+  VirtualParameters,
+  Files,
+  Config,
+} from "../types.ts";
+import { LocalCache } from "../local-cache.ts";
+
+interface Snapshot {
+  presets: Preset[];
+  provisions: Provisions;
+  virtualParameters: VirtualParameters;
+  files: Files;
+  config: Config;
+}
+
+function flattenObject<T extends Record<keyof T, unknown>>(
+  src: T,
+  prefix = "",
+  dst = {} as T,
+): T {
+  for (const k of Object.keys(src)) {
+    const v = src[k];
+    if (typeof v === "object" && !Array.isArray(v))
+      flattenObject(v as T, `${prefix}${k}.`, dst);
+    else dst[`${prefix}${k}`] = v;
+  }
+  return dst;
+}
+
+async function fetchPresets(): Promise<[string, Preset[]]> {
+  const res = await collections.presets.find().toArray();
+  let objects = await collections.objects.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  objects.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(res))
+    .update(JSON.stringify(objects))
+    .digest("hex");
+
+  objects = objects.map((obj) => {
+    // Flatten object
+    obj = flattenObject(obj);
+
+    // If no keys are defined, consider all parameters as keys to keep the
+    // same behavior from v1.0
+    if (!obj["_keys"]?.length)
+      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
+
+    return obj;
+  });
+
+  res.sort((a, b) => {
+    if (a["weight"] === b["weight"])
+      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
+    else return a["weight"] - b["weight"];
+  });
+
+  const presets = [] as Preset[];
+  for (const preset of res) {
+    let schedule: { md5: string; duration: number; schedule: any } = null;
+    if (preset["schedule"]) {
+      const parts = preset["schedule"].trim().split(/\s+/);
+      schedule = {
+        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
+        duration: null,
+        schedule: null,
+      };
+
+      try {
+        schedule.duration = +parts.shift() * 1000;
+        schedule.schedule = scheduling.parseCron(parts.join(" "));
+      } catch (err) {
+        logger.warn({
+          message: "Invalid preset schedule",
+          preset: preset["_id"],
+          schedule: preset["schedule"],
+        });
+        schedule.schedule = false;
+      }
+    }
+
+    const events = preset["events"] || {};
+    let precondition = true as Expression;
+    if (preset["precondition"]) {
+      try {
+        precondition = parse(preset["precondition"]);
+      } catch (error) {
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"]),
+        );
+      }
+
+      // Simplify expression
+      precondition = expression.evaluate(precondition);
+    }
+
+    const _provisions: Preset["provisions"] = [];
+
+    // Generate provisions from the old configuration format
+    for (const c of preset.configurations) {
+      switch (c.type) {
+        case "age":
+          _provisions.push(["refresh", c.name, +c.age]);
+          break;
+
+        case "value":
+          _provisions.push(["value", c.name, c.value]);
+          break;
+
+        case "add_tag":
+          _provisions.push(["tag", c.tag, true]);
+          break;
+
+        case "delete_tag":
+          _provisions.push(["tag", c.tag, false]);
+          break;
+
+        case "provision":
+          _provisions.push([c.name, ...(c.args || [])]);
+          break;
+
+        case "add_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 1]);
+
+              for (const k in obj) {
+                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
+                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
+              }
+            }
+          }
+
+          break;
+
+        case "delete_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 0]);
+            }
+          }
+
+          break;
+
+        default: {
+          const exhaustiveCheck: never = c;
+          throw new Error(
+            `Unknown configuration type ${exhaustiveCheck["type"]}`,
+          );
+        }
+      }
+    }
+
+    presets.push({
+      name: preset["_id"],
+      channel: (preset["channel"] as string) || "default",
+      schedule: schedule,
+      events: events,
+      precondition: precondition,
+      provisions: _provisions,
+    });
+  }
+
+  return [h, presets];
+}
+
+async function fetchProvisions(): Promise<[string, Provisions]> {
+  const res = await collections.provisions.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+
+  const provisions = {};
+  for (const r of res) {
+    provisions[r._id] = {};
+    provisions[r._id].md5 = crypto
+      .createHash("md5")
+      .update(r.script)
+      .digest("hex");
+    provisions[r._id].script = new vm.Script(
+      `"use strict";(function(){\n${r.script}\n})();`,
+      { filename: r._id, lineOffset: -1 },
+    );
+  }
+
+  return [h, provisions];
+}
+
+async function fetchVirtualParameters(): Promise<[string, VirtualParameters]> {
+  const res = await collections.virtualParameters.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+
+  const virtualParameters = {};
+  for (const r of res) {
+    virtualParameters[r._id] = {};
+    virtualParameters[r._id].md5 = crypto
+      .createHash("md5")
+      .update(r.script)
+      .digest("hex");
+    virtualParameters[r._id].script = new vm.Script(
+      `"use strict";(function(){\n${r.script}\n})();`,
+      { filename: r._id, lineOffset: -1 },
+    );
+  }
+
+  return [h, virtualParameters];
+}
+
+async function fetchFiles(): Promise<[string, Files]> {
+  const res = await collections.files.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+  const files = {};
+
+  for (const r of res) {
+    const id = r.filename || r._id.toString();
+    files[id] = {};
+    files[id].length = r.length;
+  }
+
+  return [h, files];
+}
+
+async function fetchConfig(): Promise<[string, Config]> {
+  const conf = await collections.config.find().toArray();
+  conf.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(conf)).digest("hex");
+
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    _config[c._id] = expression.evaluate(parse(c.value));
+  }
+
+  return [h, _config];
+}
+
+const localCache = new LocalCache<Snapshot>("cwmp-local-cache-hash", refresh);
+
+async function refresh(): Promise<[string, Snapshot]> {
+  const res = await Promise.all([
+    fetchPresets(),
+    fetchProvisions(),
+    fetchVirtualParameters(),
+    fetchFiles(),
+    fetchConfig(),
+  ]);
+
+  const h = crypto.createHash("md5");
+  for (const r of res) h.update(r[0]);
+
+  const snapshot = {
+    presets: res[0][1],
+    provisions: res[1][1],
+    virtualParameters: res[2][1],
+    files: res[3][1],
+    config: res[4][1],
+  };
+
+  return [h.digest("hex"), snapshot];
+}
+
+export async function getRevision(): Promise<string> {
+  return await localCache.getRevision();
+}
+
+export function getPresets(revision: string): Preset[] {
+  return localCache.get(revision).presets;
+}
+
+export function getProvisions(revision: string): Provisions {
+  return localCache.get(revision).provisions;
+}
+
+export function getVirtualParameters(revision: string): VirtualParameters {
+  return localCache.get(revision).virtualParameters;
+}
+
+export function getFiles(revision: string): Files {
+  return localCache.get(revision).files;
+}
+
+export function getConfig(
+  snapshotKey: string,
+  key: string,
+  context: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Expression,
+): string | number | boolean | null {
+  const snapshot = localCache.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const oldOpts = {
+    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
+    "cwmp.debug": "DEBUG",
+    "cwmp.retryDelay": "RETRY_DELAY",
+    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
+    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
+    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
+    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
+    "cwmp.cookiesPath": "COOKIES_PATH",
+    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
+    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
+    "cwmp.connectionRequestAllowBasicAuth":
+      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
+    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
+    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
+  };
+
+  if (!(key in snapshot.config)) {
+    if (key in oldOpts) {
+      let id;
+      if (context?.["id"]) {
+        id = context["id"];
+      } else if (cb) {
+        id = cb(["PARAM", "DeviceID.ID"]);
+        if (Array.isArray(id)) id = null;
+      }
+      return config.get(oldOpts[key], id);
+    }
+    return null;
+  }
+
+  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+  return Array.isArray(v) ? null : v;
+}
+
+export function getConfigExpression(
+  snapshotKey: string,
+  key: string,
+): Expression {
+  const snapshot = localCache.get(snapshotKey);
+  return snapshot.config[key];
+}

lib/db/db.ts

@@ -0,0 +1,57 @@
+import { MongoClient, Collection, GridFSBucket } from "mongodb";
+import { get } from "../config.ts";
+import * as MongoTypes from "./types.ts";
+
+export let filesBucket: GridFSBucket;
+
+export const collections = {
+  devices: null as Collection<MongoTypes.Device>,
+  presets: null as Collection<MongoTypes.Preset>,
+  objects: null as Collection<MongoTypes.Object>,
+  provisions: null as Collection<MongoTypes.Provision>,
+  virtualParameters: null as Collection<MongoTypes.VirtualParameter>,
+  faults: null as Collection<MongoTypes.Fault>,
+  tasks: null as Collection<MongoTypes.Task>,
+  files: null as Collection<MongoTypes.File>,
+  operations: null as Collection<MongoTypes.Operation>,
+  permissions: null as Collection<MongoTypes.Permission>,
+  users: null as Collection<MongoTypes.User>,
+  config: null as Collection<MongoTypes.Config>,
+  cache: null as Collection<MongoTypes.Cache>,
+  locks: null as Collection<MongoTypes.Lock>,
+};
+
+let clientPromise: Promise<MongoClient>;
+
+export async function connect(): Promise<void> {
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"));
+
+  const client = await clientPromise;
+  const db = client.db();
+
+  collections.tasks = db.collection("tasks");
+  collections.devices = db.collection("devices");
+  collections.presets = db.collection("presets");
+  collections.objects = db.collection("objects");
+  collections.files = db.collection("fs.files");
+  collections.provisions = db.collection("provisions");
+  collections.virtualParameters = db.collection("virtualParameters");
+  collections.faults = db.collection("faults");
+  collections.operations = db.collection("operations");
+  collections.permissions = db.collection("permissions");
+  collections.users = db.collection("users");
+  collections.config = db.collection("config");
+  collections.cache = db.collection("cache");
+  collections.locks = db.collection("locks");
+  filesBucket = new GridFSBucket(db);
+
+  await Promise.all([
+    collections.tasks.createIndex({ device: 1, timestamp: 1 }),
+    collections.cache.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }),
+    collections.locks.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }),
+  ]);
+}
+
+export async function disconnect(): Promise<void> {
+  if (clientPromise != null) await (await clientPromise).close();
+}

lib/db/synth.ts

@@ -0,0 +1,654 @@
+import { Filter } from "mongodb";
+import { EJSON } from "bson";
+import { parseLikePattern } from "../common/expression/parser.ts";
+import { Expression } from "../types.ts";
+import { decodeTag } from "../util.ts";
+import {
+  SynthContextBase,
+  likeDisjoint,
+  likeImplies,
+  Clause,
+} from "../common/expression/synth.ts";
+import normalize from "../common/expression/normalize.ts";
+import * as BI from "../common/expression/bigint.ts";
+
+type Minterm = number[];
+
+function getParam(exp: Expression, collection: string): string {
+  if (!Array.isArray(exp))
+    throw new Error("Left-hand operand must be a parameter");
+
+  if (exp[0] !== "PARAM")
+    throw new Error("Left-hand operand must be a parameter");
+
+  if (typeof exp[1] !== "string")
+    throw new Error("Left-hand operand must be a parameter");
+
+  const p = exp[1];
+  if (collection === "devices") {
+    if (p === "DeviceID.ID") return "_id";
+    else if (p === "DeviceID") return "_deviceId";
+    else if (p.startsWith("DeviceID.")) return "_deviceId._" + p.slice(9);
+    else if (p === "Events.Inform") return "_lastInform";
+    else if (p === "Events.Registered") return "_registered";
+    else if (p === "Events.0_BOOTSTRAP") return "_lastBootstrap";
+    else if (p === "Events.1_BOOT") return "_lastBoot";
+    else if (!p.endsWith("._value") && !p.startsWith("Tags."))
+      return `${p}._value`;
+  }
+
+  return p;
+}
+
+function getTypes(parameter: string, collection: string): string[] {
+  if (collection === "devices") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "_lastInform") return ["date"];
+    if (parameter === "_registered") return ["date"];
+    if (parameter === "_lastBootstrap") return ["date"];
+    if (parameter === "_lastBoot") return ["date"];
+    if (parameter.startsWith("_deviceId.")) return ["string"];
+    if (parameter === "Reboot._value") return ["date"];
+    if (parameter === "FactoryReset._value") return ["date"];
+    if (parameter.endsWith("_timestamp")) return ["date"];
+    if (parameter.startsWith("Downloads.")) {
+      if (parameter.endsWith("Download._value")) return ["date"];
+      if (parameter.endsWith("Time._value")) return ["date"];
+      if (parameter.endsWith("Name._value")) return ["string"];
+      if (parameter.endsWith("Type._value")) return ["string"];
+    }
+    if (parameter.endsWith("_value"))
+      return ["bool", "number", "date", "string"];
+  } else if (collection === "tasks") {
+    if (parameter === "_id") return ["oid"];
+    if (parameter === "timestamp") return ["date"];
+    if (parameter === "expiry") return ["date"];
+    if (parameter === "name") return ["string"];
+    if (parameter === "device") return ["string"];
+  } else if (collection === "faults") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "timestamp") return ["date"];
+    if (parameter === "expiry") return ["date"];
+    if (parameter === "code") return ["string"];
+    if (parameter === "retries") return ["number"];
+    if (parameter === "channel") return ["string"];
+    if (parameter === "device") return ["string"];
+    if (parameter === "message") return ["string"];
+  } else if (collection === "users") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "roles") return ["string"];
+    else if (parameter === "password")
+      throw new Error("Cannot query restricted parameters");
+    else if (parameter === "salt")
+      throw new Error("Cannot query restricted parameters");
+  } else if (collection === "config") {
+    if (parameter === "_id") return ["string"];
+    else if (parameter === "value") return ["string"];
+  } else if (collection === "files") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "metadata.fileType") return ["string"];
+    if (parameter === "metadata.oui") return ["string"];
+    if (parameter === "metadata.productClass") return ["string"];
+    if (parameter === "metadata.version") return ["string"];
+  } else if (collection === "permissions") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "role") return ["string"];
+    if (parameter === "resource") return ["devices"];
+    if (parameter === "access") return ["number"];
+    if (parameter === "filter") return ["string"];
+    if (parameter === "validate") return ["string"];
+  } else if (collection === "presets") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "weight") return ["number"];
+    if (parameter === "channel") return ["string"];
+    if (parameter === "precondition") return ["string"];
+    if (parameter.startsWith("events.")) return ["bool"];
+  } else if (collection === "provisions") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "script") return ["string"];
+  } else if (collection === "virtualParameters") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "script") return ["string"];
+  }
+
+  if (parameter === "_id") return ["oid", "string"];
+  return ["bool", "number", "date", "string"];
+}
+
+function roundOid(oid: string, roundUp: boolean): string {
+  const match = (oid.match(/^[0-9a-f]*/)?.[0] ?? "").slice(0, 24);
+  let num = BI.BigInt("0x0" + match);
+  let lastChar = 0;
+  if (oid.length > match.length) lastChar += oid.charCodeAt(match.length);
+  if (match.length < 24) lastChar -= 48;
+  if (lastChar > 0 && roundUp) num++;
+  num <<= BI.BigInt(4 * (24 - match.length));
+  if (lastChar < 0 && !roundUp) --num;
+  const str = num.toString(16);
+  if (str.length > 24 || str.startsWith("-")) return null;
+  return str.padStart(24, "0");
+}
+
+function groupBy<T, K>(
+  input: T[],
+  callback: (item: T) => K,
+): Iterable<[K, T[]]> {
+  const groups = new Map<K, T[]>();
+  for (const item of input) {
+    const key = callback(item);
+    let arr = groups.get(key);
+    if (!arr) groups.set(key, (arr = []));
+    arr.push(item);
+  }
+
+  return groups.entries();
+}
+
+abstract class MongoClause {
+  abstract readonly parameter: string;
+  abstract toQuery(truthy: boolean): Filter<unknown>;
+  toString(): string {
+    return JSON.stringify(this.toQuery(true));
+  }
+}
+
+class MongoClauseArray extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly value: string,
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    if (truthy) return { [this.parameter]: { $eq: this.value } };
+    else return { [this.parameter]: { $ne: this.value } };
+  }
+}
+
+class MongoClauseCompare<T> extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly op: "$eq" | "$gt" | "$lt" | "$gte" | "$lte",
+    public readonly value: T,
+    public readonly type: "" | "bool" | "number" | "string" | "date" | "oid",
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    let v: any = this.value;
+    if (this.type === "date" && typeof v === "number")
+      v = { $date: new Date(v).toISOString() };
+    if (this.type === "oid" && typeof v === "string") v = { $oid: v };
+
+    if (truthy) return { [this.parameter]: { [this.op]: v } };
+    else if (this.op === "$eq") return { [this.parameter]: { $ne: v } };
+    else return { [this.parameter]: { $not: { [this.op]: v } } };
+  }
+}
+
+class MongoClauseType extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly type: string,
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    if (truthy) return { [this.parameter]: { $type: this.type } };
+    else return { [this.parameter]: { $not: { $type: this.type } } };
+  }
+}
+
+class MongoClauseLike extends MongoClause {
+  readonly pattern: string[];
+  constructor(
+    public readonly parameter: string,
+    pat: string,
+    esc: string,
+    public readonly caseSensitive: boolean,
+  ) {
+    super();
+    this.pattern = parseLikePattern(pat, esc);
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    const convChars = {
+      "-": "\\-",
+      "/": "\\/",
+      "\\": "\\/",
+      "^": "\\^",
+      $: "\\$",
+      "*": "\\*",
+      "+": "\\+",
+      "?": "\\?",
+      ".": "\\.",
+      "(": "\\(",
+      ")": "\\)",
+      "|": "\\|",
+      "[": "\\[",
+      "]": "\\]",
+      "{": "\\{",
+      "}": "\\}",
+      "\\%": ".*",
+      "\\_": ".",
+    };
+
+    const chars = this.pattern.map((c) => convChars[c] ?? c);
+    chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
+    const l = chars.length - 1;
+    chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
+    const pattern = chars.join("");
+    const options = this.caseSensitive ? "s" : "is";
+    if (truthy) {
+      return { [this.parameter]: { $regularExpression: { options, pattern } } };
+    } else {
+      return {
+        [this.parameter]: {
+          $not: { $regularExpression: { options, pattern } },
+        },
+      };
+    }
+  }
+}
+
+class MongoSynthContext extends SynthContextBase<Clause, MongoClause> {
+  constructor(private readonly collection: string) {
+    super();
+  }
+
+  getMinterms(clause: Clause, res: true | false | null): number[][] {
+    const exp = clause.expression();
+    if (!Array.isArray(exp)) throw new Error("Invalid query expression");
+
+    if (res === null) {
+      const minterms: number[][] = [];
+      for (const dep of clause.getNullables()) {
+        const e = dep.operand.expression();
+        const param = getParam(e, this.collection);
+        if (param.startsWith("Tags.") && this.collection === "devices") {
+          const t = decodeTag(param.slice(5));
+          const c = new MongoClauseArray("_tags", t);
+          minterms.push([this.getVar(c) << 1]);
+          continue;
+        }
+        const c = new MongoClauseCompare(param, "$eq", null, "");
+        minterms.push([(this.getVar(c) << 1) ^ 1]);
+      }
+      return minterms;
+    }
+
+    if ([">", "<", "="].includes(exp[0])) {
+      const param = getParam(exp[1], this.collection);
+
+      let rhs = exp[2];
+      if (typeof rhs === "boolean") rhs = +rhs;
+      else if (typeof rhs !== "string" && typeof rhs !== "number")
+        throw new Error(`Right-hand operand must be a literal value`);
+
+      if (
+        param.startsWith("Tags.") &&
+        this.collection === "devices" &&
+        exp[0] === "="
+      ) {
+        const t = decodeTag(param.slice(5));
+        const c = new MongoClauseArray("_tags", t);
+        if (typeof rhs === "string") rhs = 2;
+        if (exp[0] === "=") {
+          if ((rhs === 1) !== res) return [];
+        } else if (exp[0] === ">") {
+          if ((res && rhs >= 1) || (!res && rhs < 1)) return [];
+        } else if (exp[0] === "<") {
+          if ((res && rhs <= 1) || (!res && rhs > 1)) return [];
+        }
+        return [[(this.getVar(c) << 1) ^ 1]];
+      }
+
+      let op: "$gt" | "$lt" | "$eq" | "$gte" | "$lte";
+      if (exp[0] === "=") op = "$eq";
+      else if (exp[0] === ">") op = res ? "$gt" : "$lte";
+      else if (exp[0] === "<") op = res ? "$lt" : "$gte";
+
+      const possibleTypes = new Set(getTypes(param, this.collection));
+      const clauses: MongoClause[] = [];
+
+      if (typeof rhs === "number") {
+        if (possibleTypes.has("number"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "number"));
+
+        if (possibleTypes.has("date"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "date"));
+        if (possibleTypes.has("bool") && (rhs === 0 || rhs === 1))
+          clauses.push(new MongoClauseCompare(param, op, !!rhs, "bool"));
+      } else if (typeof rhs === "string") {
+        if (possibleTypes.has("string"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "string"));
+
+        if (possibleTypes.has("oid")) {
+          const oid = roundOid(rhs, op === "$lt" || op === "$lte");
+          if (oid && (op !== "$eq" || oid === rhs))
+            clauses.push(new MongoClauseCompare(param, op, oid, "oid"));
+        }
+      }
+
+      if (op === "$eq" && !res) {
+        clauses.push(new MongoClauseCompare(param, "$eq", null, ""));
+        return [clauses.map((c) => this.getVar(c) << 1)];
+      }
+
+      // In the following clauses we could use $type operator, but we want the
+      // final query to use comparison operators to check for type because the
+      // $type operator in MongoDB doesn't use indexes
+      if (typeof rhs === "number") {
+        if (possibleTypes.has("bool")) {
+          if (
+            (rhs > 1 && (op === "$lt" || op === "$lte")) ||
+            (rhs < 0 && (op === "$gt" || op === "$gte"))
+          )
+            clauses.push(new MongoClauseCompare(param, "$gte", false, "bool"));
+        }
+
+        if (op === "$gt" || op === "$gte") {
+          if (possibleTypes.has("string"))
+            clauses.push(new MongoClauseCompare(param, "$gte", "", "string"));
+
+          if (possibleTypes.has("oid")) {
+            clauses.push(
+              new MongoClauseCompare(
+                param,
+                "$gte",
+                "000000000000000000000000",
+                "oid",
+              ),
+            );
+          }
+        }
+      } else if (typeof rhs === "string") {
+        if (op === "$lt" || op === "$lte") {
+          if (possibleTypes.has("bool"))
+            clauses.push(new MongoClauseCompare(param, "$gte", false, "bool"));
+
+          if (possibleTypes.has("number")) {
+            clauses.push(new MongoClauseCompare(param, "$gte", 0, "number"));
+            clauses.push(new MongoClauseCompare(param, "$lt", 0, "number"));
+          }
+          if (possibleTypes.has("date")) {
+            clauses.push(new MongoClauseCompare(param, "$gte", 0, "date"));
+            clauses.push(new MongoClauseCompare(param, "$lt", 0, "date"));
+          }
+        }
+      }
+
+      return clauses.map((c) => [(this.getVar(c) << 1) ^ 1]);
+    } else if (exp[0] === "LIKE") {
+      const pat = exp[2];
+      if (typeof pat !== "string")
+        throw new Error("Right-hand operand of 'LIKE' must be a string");
+      let p = exp[1];
+      let caseSensitive = true;
+      if (
+        Array.isArray(p) &&
+        p[0] === "FUNC" &&
+        ["UPPER", "LOWER"].includes(p[1])
+      ) {
+        if (p[1] === "UPPER" && pat !== pat.toUpperCase())
+          return res ? [] : [[]];
+        if (p[1] === "LOWER" && pat !== pat.toLowerCase())
+          return res ? [] : [[]];
+        caseSensitive = false;
+        p = p[2];
+      }
+
+      const param = getParam(p, this.collection);
+      const c = new MongoClauseLike(param, pat, exp[3], caseSensitive);
+      if (res) return [[(this.getVar(c) << 1) ^ 1]];
+      const typeClase = new MongoClauseType(param, "string");
+      const r = [[this.getVar(c) << 1, (this.getVar(typeClase) << 1) ^ 1]];
+      return r;
+    } else {
+      throw new Error("Invalid query expression");
+    }
+  }
+
+  getDcSet(minterms: Minterm[]): number[][] {
+    const dcSet: number[][] = [];
+
+    const vars = new Set(minterms.flat().map((l) => l >> 1));
+    const clauses = Array.from(vars).map((v) => this.getClause(v));
+
+    for (const [parameter, clauses2] of groupBy(clauses, (c) => c.parameter)) {
+      const comparisons = clauses2.filter(
+        (c) => c instanceof MongoClauseCompare && c.value !== null,
+      ) as MongoClauseCompare<unknown>[];
+
+      for (const [type, clauses3] of groupBy(comparisons, (c) => c.type)) {
+        const isType = this.getVar(new MongoClauseType(parameter, type));
+        const values = new Set(clauses3.map((c) => c.value));
+        const valuesSorted = Array.from(values).sort((a, b) =>
+          a > b ? 1 : -1,
+        );
+        for (const [i, v] of valuesSorted.entries()) {
+          const eq = this.getVar(
+            new MongoClauseCompare(parameter, "$eq", v, type),
+          );
+          const gt = this.getVar(
+            new MongoClauseCompare(parameter, "$gt", v, type),
+          );
+          const lt = this.getVar(
+            new MongoClauseCompare(parameter, "$lt", v, type),
+          );
+          const gte = this.getVar(
+            new MongoClauseCompare(parameter, "$gte", v, type),
+          );
+          const lte = this.getVar(
+            new MongoClauseCompare(parameter, "$lte", v, type),
+          );
+
+          if (type === "bool") {
+            if (v === false) dcSet.push([(lt << 1) ^ 1]);
+            else if (v === true) dcSet.push([(gt << 1) ^ 1]);
+          } else if (type === "string") {
+            if (v === "") dcSet.push([(lt << 1) ^ 1]);
+          } else if (type === "oid") {
+            if (v < "000000000000000000000000") dcSet.push([(lte << 1) ^ 1]);
+            else if (v === "000000000000000000000000")
+              dcSet.push([(lt << 1) ^ 1]);
+            else if (v > "ffffffffffffffffffffffff")
+              dcSet.push([(gte << 1) ^ 1]);
+            else if (v === "ffffffffffffffffffffffff")
+              dcSet.push([(gt << 1) ^ 1]);
+          }
+          dcSet.push([(lt << 1) ^ 1, (gte << 1) ^ 1]);
+          dcSet.push([(gt << 1) ^ 1, (gte << 1) ^ 0]);
+          dcSet.push([(eq << 1) ^ 0, (lt << 1) ^ 0, (lte << 1) ^ 1]);
+          dcSet.push([(eq << 1) ^ 1, (gte << 1) ^ 0]);
+          dcSet.push([(eq << 1) ^ 1, (gt << 1) ^ 1]);
+
+          if (i === 0) {
+            dcSet.push([(isType << 1) ^ 0, (gte << 1) ^ 1]);
+            dcSet.push([(isType << 1) ^ 0, (lt << 1) ^ 1]);
+          } else {
+            const gt2 = this.getVar(
+              new MongoClauseCompare(
+                parameter,
+                "$gt",
+                valuesSorted[i - 1],
+                type,
+              ),
+            );
+            const lte2 = this.getVar(
+              new MongoClauseCompare(
+                parameter,
+                "$lte",
+                valuesSorted[i - 1],
+                type,
+              ),
+            );
+            dcSet.push([(gt2 << 1) ^ 0, (gte << 1) ^ 1]);
+            dcSet.push([(gt2 << 1) ^ 0, (lte2 << 1) ^ 0, (lt << 1) ^ 1]);
+          }
+
+          if (i === valuesSorted.length - 1)
+            dcSet.push([(isType << 1) ^ 1, (gt << 1) ^ 0, (lte << 1) ^ 0]);
+        }
+      }
+
+      const likes = clauses2.filter(
+        (c) => c instanceof MongoClauseLike,
+      ) as MongoClauseLike[];
+      if (likes.length) {
+        const isType = this.getVar(new MongoClauseType(parameter, "string"));
+        for (let i1 = 0; i1 < likes.length; ++i1) {
+          const l1 = likes[i1];
+          let p1 = l1.pattern;
+          dcSet.push([(this.getVar(l1) << 1) ^ 1, (isType << 1) ^ 0]);
+          for (let i2 = i1 + 1; i2 < likes.length; ++i2) {
+            const l2 = likes[i2];
+            let p2 = l2.pattern;
+            if (!l1.caseSensitive || !l2.caseSensitive) {
+              p1 = p1.map((c) => c.toLowerCase());
+              p2 = p2.map((c) => c.toLowerCase());
+            }
+            if (likeDisjoint(p1, p2)) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 1,
+                (this.getVar(l2) << 1) ^ 1,
+              ]);
+            } else if (
+              (!l1.caseSensitive || l2.caseSensitive) &&
+              likeImplies(p1, p2)
+            ) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 0,
+                (this.getVar(l2) << 1) ^ 1,
+              ]);
+            } else if (
+              (!l2.caseSensitive || l1.caseSensitive) &&
+              likeImplies(p2, p1)
+            ) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 1,
+                (this.getVar(l2) << 1) ^ 0,
+              ]);
+            }
+          }
+        }
+      }
+
+      const isNull = this.getVar(
+        new MongoClauseCompare(parameter, "$eq", null, ""),
+      );
+
+      const types = getTypes(parameter, this.collection).map((t) =>
+        this.getVar(new MongoClauseType(parameter, t)),
+      );
+
+      dcSet.push([(isNull << 1) ^ 0, ...types.map((t) => (t << 1) ^ 0)]);
+      for (let i = 0; i < types.length; ++i) {
+        const t1 = types[i];
+        dcSet.push([(t1 << 1) ^ 1, (isNull << 1) ^ 1]);
+        for (let j = i + 1; j < types.length; ++j) {
+          const t2 = types[j];
+          dcSet.push([(t1 << 1) ^ 1, (t2 << 1) ^ 1]);
+        }
+      }
+
+      if (parameter === "_id") dcSet.push([(isNull << 1) ^ 1]);
+    }
+    return dcSet;
+  }
+
+  canRaise(i: number, s: Set<number>): boolean {
+    if (!(i & 1)) return true;
+    const c = this.getClause(i >> 1);
+    if (c instanceof MongoClauseCompare) {
+      for (const j of s) {
+        if (j === i) continue;
+        const c2 = this.getClause(j >> 1);
+        if (c2.parameter !== c.parameter) continue;
+        if (!(j & 1)) return false;
+        if (c2 instanceof MongoClauseType) return false;
+      }
+    }
+    return true;
+  }
+
+  toQuery(minterms: Minterm[]): Filter<unknown> {
+    const or = [];
+
+    const ejsonOps = ["$oid", "$date", "$regularExpression"];
+
+    for (const minterm of minterms) {
+      const query = {};
+      loop: for (const clause of minterm) {
+        if (!minterm.length) return {};
+        const negate = !!(clause & 1);
+        const c = this.getClause(clause >> 1);
+        const q = c.toQuery(negate);
+        if (Object.keys(q).length !== 1)
+          throw new Error("Invalid query expression");
+        const [param, value] = Object.entries(q)[0];
+        const dests = [query, ...(query["$and"] ?? [])];
+        for (const dest of dests) {
+          if (!(param in dest)) {
+            dest[param] = value;
+            continue loop;
+          }
+
+          let src = value;
+          let dst = dest[param];
+          if (Object.getPrototypeOf(src).constructor !== Object) continue;
+          if (Object.getPrototypeOf(dst).constructor !== Object) continue;
+          let srcKeys = Object.keys(src);
+          let dstKeys = Object.keys(dst);
+
+          if ([...srcKeys, ...dstKeys].every((k) => k === "$not")) {
+            src = src["$not"];
+            dst = dst["$not"];
+            if (Object.getPrototypeOf(src).constructor !== Object) continue;
+            if (Object.getPrototypeOf(dst).constructor !== Object) continue;
+            srcKeys = Object.keys(src);
+            dstKeys = Object.keys(dst);
+          }
+
+          if (srcKeys.some((k) => dstKeys.includes(k))) continue;
+
+          // Don't mix regular operators with EJSON special operators
+          if (srcKeys.some((k) => ejsonOps.includes(k))) continue;
+          if (dstKeys.some((k) => ejsonOps.includes(k))) continue;
+
+          Object.assign(dst, src);
+          continue loop;
+        }
+
+        query["$and"] ??= [];
+        query["$and"].push({ [param]: value });
+      }
+      or.push(query);
+    }
+
+    if (or.length === 1) return or[0];
+    return { $or: or };
+  }
+}
+
+export function toMongoQuery(
+  exp: Expression,
+  resource: string,
+): Filter<unknown> | false {
+  exp = normalize(exp);
+  const clause = Clause.fromExpression(normalize(exp));
+  const context = new MongoSynthContext(resource);
+  const minterms = clause.true(context);
+  const minimized = context.minimize(minterms);
+  if (!minimized.length) return false;
+  return EJSON.deserialize(context.toQuery(minimized));
+}
+
+export function validQuery(exp: Expression, resource: string): void {
+  const clause = Clause.fromExpression(normalize(exp));
+  const context = new MongoSynthContext(resource);
+  clause.true(context);
+}

lib/db/types.ts

@@ -0,0 +1,183 @@
+import { ObjectId } from "mongodb";
+import { Expression, FaultStruct } from "../types.ts";
+
+export interface Fault {
+  _id: string;
+  device: string;
+  channel: string;
+  timestamp: Date;
+  provisions: string;
+  retries: number;
+  code: string;
+  message: string;
+  detail?:
+    | FaultStruct
+    | {
+        name: string;
+        message: string;
+        stack?: string;
+      };
+  expiry?: Date;
+}
+
+interface TaskBase {
+  _id: ObjectId;
+  timestamp?: Date;
+  expiry?: Date;
+  name: string;
+  device: string;
+}
+
+interface TaskGetParameterValues extends TaskBase {
+  name: "getParameterValues";
+  parameterNames: string[];
+}
+
+interface TaskSetParameterValues extends TaskBase {
+  name: "setParameterValues";
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskRefreshObject extends TaskBase {
+  name: "refreshObject";
+  objectName: string;
+}
+
+interface TaskReboot extends TaskBase {
+  name: "reboot";
+}
+
+interface TaskFactoryReset extends TaskBase {
+  name: "factoryReset";
+}
+
+interface TaskDownload extends TaskBase {
+  name: "download";
+  fileType: string;
+  fileName: string;
+  targetFileName?: string;
+}
+
+interface TaskAddObject extends TaskBase {
+  name: "addObject";
+  objectName: string;
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskDeleteObject extends TaskBase {
+  name: "deleteObject";
+  objectName: string;
+}
+
+interface TaskProvisions extends TaskBase {
+  name: "provisions";
+  provisions?: [string, ...Expression[]][];
+}
+
+export type Task =
+  | TaskGetParameterValues
+  | TaskSetParameterValues
+  | TaskRefreshObject
+  | TaskReboot
+  | TaskFactoryReset
+  | TaskDownload
+  | TaskAddObject
+  | TaskDeleteObject
+  | TaskProvisions;
+
+export interface Operation {
+  _id: string;
+  name: string;
+  timestamp: Date;
+  channels: string;
+  retries: string;
+  provisions: string;
+  args: string;
+}
+
+export interface Config {
+  _id: string;
+  value: string;
+}
+
+export interface Cache {
+  _id: string;
+  value: string;
+  timestamp: Date;
+  expire: Date;
+}
+
+export interface Device {
+  _id: string;
+  _lastInform: Date;
+  _registered: Date;
+  _tags?: string[];
+  _timestamp?: Date;
+}
+
+type Configuration =
+  | { type: "age"; name: string; age: number }
+  | { type: "value"; name: string; value: boolean | number | string }
+  | { type: "add_tag"; tag: string }
+  | { type: "delete_tag"; tag: string }
+  | { type: "add_object"; name: string; object: string }
+  | { type: "delete_object"; name: string; object: string }
+  | { type: "provision"; name: string; args?: Expression[] };
+
+export interface Preset {
+  _id: string;
+  weight: number;
+  channel: string;
+  events: Record<string, boolean>;
+  configurations: Configuration[];
+}
+
+export interface Object {
+  _id: string;
+}
+
+export interface Provision {
+  _id: string;
+  script: string;
+}
+
+export interface VirtualParameter {
+  _id: string;
+  script: string;
+}
+
+export interface File {
+  _id: string;
+  length: number;
+  filename: string;
+  uploadDate: Date;
+  metadata?: {
+    fileType?: string;
+    oui?: string;
+    productClass?: string;
+    version?: string;
+  };
+}
+
+export interface Permission {
+  _id: string;
+  role: string;
+  resource: string;
+  access: 1 | 2 | 3;
+  filter?: string;
+  validate?: string;
+}
+
+export interface User {
+  _id: string;
+  password: string;
+  roles: string;
+  salt: string;
+}
+
+export interface Lock {
+  _id: string;
+  value: string;
+  timestamp: Date;
+  expire: Date;
+}

lib/db/util.ts

@@ -0,0 +1,123 @@
+import { Expression } from "../types.ts";
+import { encodeTag } from "../util.ts";
+
+// Optimize projection by removing overlaps
+// This can modify the object
+export function optimizeProjection(obj: { [path: string]: 1 }): {
+  [path: string]: 1;
+} {
+  if (obj[""]) return { "": obj[""] };
+
+  const keys = Object.keys(obj).sort();
+  if (keys.length <= 1) return obj;
+
+  for (let i = 1; i < keys.length; ++i) {
+    const a = keys[i - 1];
+    const b = keys[i];
+    if (b.startsWith(a)) {
+      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
+        delete obj[b];
+        keys.splice(i--, 1);
+      }
+    }
+  }
+  return obj;
+}
+
+export function convertOldPrecondition(q: Record<string, unknown>): Expression {
+  function recursive(_query): Expression {
+    const expressions: Expression[] = [];
+    for (const [k, v] of Object.entries(_query)) {
+      if (k[0] === "$") {
+        if (k === "$and") {
+          const and: Expression = ["AND"];
+          for (const vv of Object.values(v)) and.push(recursive(vv));
+          expressions.push(and);
+        } else if (k === "$or") {
+          const or: Expression = ["OR"];
+          for (const vv of Object.values(v)) or.push(recursive(vv));
+          expressions.push(or);
+        } else {
+          throw new Error(`Operator ${k} not supported`);
+        }
+      } else if (k === "_tags") {
+        if (typeof v === "object") {
+          if (Array.isArray(v)) throw new Error(`Invalid type`);
+          const conjs: Expression[] = [];
+          for (const [op, val] of Object.entries(v)) {
+            if (op === "$ne") {
+              if (typeof v["$ne"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else {
+              throw new Error(`Invalid tag query`);
+            }
+          }
+          if (conjs.length === 1) expressions.push(conjs[0]);
+          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
+        } else {
+          expressions.push([
+            "IS NOT NULL",
+            ["PARAM", `Tags.${encodeTag(v as string)}`],
+          ]);
+        }
+      } else if (k.startsWith("Tags.")) {
+        let exists: boolean;
+        if (typeof v === "boolean") exists = v;
+        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
+        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
+        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
+        else throw new Error(`Invalid tag query`);
+
+        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
+      } else if (typeof v === "object") {
+        if (Array.isArray(v)) throw new Error(`Invalid type`);
+
+        const exps: Expression[] = [];
+        for (const [kk, vv] of Object.entries(v)) {
+          if (kk === "$eq") {
+            exps.push(["=", ["PARAM", k], vv]);
+          } else if (kk === "$ne") {
+            exps.push([
+              "OR",
+              ["<>", ["PARAM", k], vv],
+              ["IS NULL", ["PARAM", k]],
+            ]);
+          } else if (kk === "$lt") {
+            exps.push(["<", ["PARAM", k], vv]);
+          } else if (kk === "$lte") {
+            exps.push(["<=", ["PARAM", k], vv]);
+          } else if (kk === "$gt") {
+            exps.push([">", ["PARAM", k], vv]);
+          } else if (kk === "$gte") {
+            exps.push([">=", ["PARAM", k], vv]);
+          } else {
+            throw new Error(`Operator ${kk} not supported`);
+          }
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
+        }
+        if (exps.length === 1) {
+          expressions.push(exps[0]);
+        } else if (exps.length > 1) {
+          const and: Expression = ["AND"];
+          expressions.push(and.concat(exps));
+        }
+      } else {
+        expressions.push(["=", ["PARAM", k], v]);
+      }
+    }
+    if (expressions.length === 1) return expressions[0];
+    if (expressions.length === 0) return true;
+    return ["AND", ...expressions];
+  }
+
+  // empty filter
+  if (!Object.keys(q).length) return true;
+
+  return recursive(q);
+}

lib/debug.ts

@@ -1,32 +1,9 @@
-/**
+import { IncomingMessage, ServerResponse, ClientRequest } from "node:http";
- * Copyright 2013-2019  GenieACS Inc.
+import { Socket } from "node:net";
- *
+import { appendFileSync } from "node:fs";
- * This file is part of GenieACS.
+import { stringify } from "./common/yaml.ts";
- *
+import * as config from "./config.ts";
- * GenieACS is free software: you can redistribute it and/or modify
+import { getSocketEndpoints } from "./server.ts";
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import {
-  IncomingMessage,
-  ServerResponse,
-  ClientRequest,
-  RequestOptions,
-} from "http";
-import { Socket } from "net";
-import { appendFileSync } from "fs";
-import * as yaml from "yaml";
-import * as config from "./config";
 
 const DEBUG_FILE = "" + config.get("DEBUG_FILE");
 const DEBUG_FORMAT = "" + config.get("DEBUG_FORMAT");
@@ -45,18 +22,19 @@ function getConnectionTimestamp(connection: Socket): Date {
 export function incomingHttpRequest(
   httpRequest: IncomingMessage,
   deviceId: string,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
   const con = httpRequest.socket;
+  const socketEndpoints = getSocketEndpoints(con);
   const msg = {
     event: "incoming HTTP request",
     timestamp: now,
-    remoteAddress: con.remoteAddress,
+    remoteAddress: socketEndpoints.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
-    localPort: con.localPort,
+    localPort: socketEndpoints.localPort,
     method: httpRequest.method,
     url: httpRequest.url,
     headers: httpRequest.headers,
@@ -64,7 +42,7 @@ export function incomingHttpRequest(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -73,15 +51,16 @@ export function incomingHttpRequest(
 export function outgoingHttpResponse(
   httpResponse: ServerResponse,
   deviceId: string,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
   const con = httpResponse.socket;
+  const socketEndpoints = getSocketEndpoints(con);
   const msg = {
     event: "outgoing HTTP response",
     timestamp: now,
-    remoteAddress: con.remoteAddress,
+    remoteAddress: socketEndpoints.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
     statusCode: httpResponse.statusCode,
@@ -90,7 +69,7 @@ export function outgoingHttpResponse(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -99,8 +78,9 @@ export function outgoingHttpResponse(
 export function outgoingHttpRequest(
   httpRequest: ClientRequest,
   deviceId: string,
-  options: RequestOptions,
+  method: "GET" | "PUT" | "POST" | "DELETE",
-  body: string
+  url: URL,
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
@@ -111,15 +91,15 @@ export function outgoingHttpRequest(
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
-    remotePort: options.port,
+    remotePort: url.port,
-    method: options.method || "GET",
+    method: method,
-    url: options.path,
+    url: url.pathname + url.search,
     headers: httpRequest.getHeaders(),
     body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -128,26 +108,27 @@ export function outgoingHttpRequest(
 export function outgoingHttpRequestError(
   httpRequest: ClientRequest,
   deviceId: string,
-  options: RequestOptions,
+  method: "GET" | "PUT" | "POST" | "DELETE",
-  err: Error
+  url: URL,
+  err: Error,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
-    remoteAddress: options.hostname,
+    remoteAddress: url.hostname,
     deviceId: deviceId,
     connection: null,
-    remotePort: options.port,
+    remotePort: url.port,
-    method: options.method,
+    method: method,
-    url: options.path,
+    url: url.pathname + url.search,
     headers: httpRequest.getHeaders(),
     error: err.message,
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -156,7 +137,7 @@ export function outgoingHttpRequestError(
 export function incomingHttpResponse(
   httpResponse: IncomingMessage,
   deviceId: string,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
@@ -173,7 +154,7 @@ export function incomingHttpResponse(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -183,7 +164,7 @@ export function outgoingUdpMessage(
   remoteAddress: string,
   deviceId: string,
   remotePort: number,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
@@ -197,7 +178,58 @@ export function outgoingUdpMessage(
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function clientError(remoteAddress: string, err: Error): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "client error",
+    timestamp: now,
+    remoteAddress: remoteAddress,
+    error: err.message,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function incomingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "incoming XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);

lib/default-provisions.ts

@@ -1,34 +1,15 @@
-/**
+import Path from "./common/path.ts";
- * Copyright 2013-2019  GenieACS Inc.
+import * as config from "./config.ts";
- *
+import * as device from "./device.ts";
- * This file is part of GenieACS.
+import * as scheduling from "./scheduling.ts";
- *
+import { SessionContext, Declaration } from "./types.ts";
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
-import * as config from "./config";
-import * as device from "./device";
-import * as scheduling from "./scheduling";
-import { SessionContext, Declaration } from "./types";
 
 const MAX_DEPTH = +config.get("MAX_DEPTH");
 
 export function refresh(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (
     (provision.length !== 2 || typeof provision[1] !== "string") &&
@@ -84,7 +65,7 @@ export function refresh(
 export function value(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (
     provision.length < 3 ||
@@ -127,7 +108,7 @@ export function value(
 export function tag(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (
     provision.length !== 3 ||
@@ -151,7 +132,7 @@ export function tag(
 export function reboot(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (provision.length !== 1) throw new Error("Invalid arguments");
 
@@ -170,7 +151,7 @@ export function reboot(
 export function reset(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (provision.length !== 1) throw new Error("Invalid arguments");
 
@@ -189,7 +170,7 @@ export function reset(
 export function download(
   sessionContext: SessionContext,
   provision: (string | number | boolean)[],
-  declarations: Declaration[]
+  declarations: Declaration[],
 ): boolean {
   if (
     (provision.length !== 3 ||
@@ -234,7 +215,7 @@ export function instances(
   provision: (string | number | boolean)[],
   declarations: Declaration[],
   startRevision: number,
-  endRevision: number
+  endRevision: number,
 ): boolean {
   if (provision.length !== 3 || typeof provision[1] !== "string")
     throw new Error("Invalid arguments");
@@ -260,7 +241,7 @@ export function instances(
     const unpacked = device.unpack(
       sessionContext.deviceData,
       path,
-      startRevision + 1
+      startRevision + 1,
     );
     count = Math.max(0, unpacked.length + count);
   }

lib/device.ts

@@ -1,30 +1,11 @@
-/**
+import Path from "./common/path.ts";
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
 import {
   DeviceData,
   Declaration,
   Clear,
   AttributeTimestamps,
   Attributes,
-} from "./types";
+} from "./types.ts";
 
 const CHANGE_FLAGS = {
   object: 2,
@@ -43,7 +24,7 @@ function parseBool(v): boolean {
 }
 
 export function sanitizeParameterValue(
-  parameterValue: [string | number | boolean, string]
+  parameterValue: [string | number | boolean, string],
 ): [string | number | boolean, string] {
   if (parameterValue[0] != null) {
     switch (parameterValue[1]) {
@@ -54,13 +35,6 @@ export function sanitizeParameterValue(
           else parameterValue[0] = b;
         }
         break;
-      case "xsd:string":
-      case "xsd:base64":
-      case "xsd:hexBinary":
-        if (typeof parameterValue[0] !== "string")
-          parameterValue[0] = "" + parameterValue[0];
-
-        break;
       case "xsd:int":
       case "xsd:unsignedInt":
         if (typeof parameterValue[0] !== "number") {
@@ -83,7 +57,7 @@ export function sanitizeParameterValue(
         }
         break;
       default:
-        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
+        parameterValue[0] = "" + parameterValue[0];
         break;
     }
   }
@@ -94,7 +68,7 @@ export function sanitizeParameterValue(
 export function getAliasDeclarations(
   path: Path,
   timestamp: number,
-  attrGet = null
+  attrGet = null,
 ): Declaration[] {
   const stripped = path.stripAlias();
   let decs: Declaration[] = [
@@ -116,7 +90,7 @@ export function getAliasDeclarations(
           decs = decs.concat(
             getAliasDeclarations(parent.concat(p), timestamp, {
               value: timestamp,
-            })
+            }),
           );
         }
       }
@@ -129,7 +103,7 @@ export function getAliasDeclarations(
 export function unpack(
   deviceData: DeviceData,
   path: Path,
-  revision?: number
+  revision?: number,
 ): Path[] {
   let allMatches = [] as Path[];
   if (!path.alias) {
@@ -201,14 +175,14 @@ export function clear(
   path: Path,
   timestamp: number,
   attributes: AttributeTimestamps,
-  changeFlags = 0
+  changeFlags = 0,
 ): void {
   const changeTrackers = {};
 
   timestamp = timestamp || 0;
 
   let descendantsTimestamp = timestamp;
-  if (attributes && attributes.object) {
+  if (attributes?.object) {
     if (attributes.object > descendantsTimestamp)
       descendantsTimestamp = attributes.object;
     if (!(attributes.object <= attributes.value))
@@ -219,7 +193,7 @@ export function clear(
     path,
     true,
     true,
-    descendantsTimestamp ? 99 : path.length
+    descendantsTimestamp ? 99 : path.length,
   )) {
     const tracker = deviceData.trackers.get(p);
     for (const k in tracker) changeTrackers[k] |= tracker[k];
@@ -281,7 +255,7 @@ export function set(
   path: Path,
   timestamp: number,
   attributes: Attributes,
-  toClear?: Clear[]
+  toClear?: Clear[],
 ): Clear[] {
   path = deviceData.paths.add(path);
 
@@ -322,7 +296,7 @@ export function set(
         else if (
           !compareEquality(
             attributes[attrName][1],
-            currentAttributes[attrName][1]
+            currentAttributes[attrName][1],
           )
         )
           changeFlags |= CHANGE_FLAGS[attrName];
@@ -341,7 +315,7 @@ export function set(
           path.slice(0, path.length - 1),
           timestamp,
           { object: [timestamp, 1] },
-          toClear
+          toClear,
         );
       }
     }
@@ -384,7 +358,7 @@ export function track(
   deviceData: DeviceData,
   path: Path,
   marker: string,
-  attributes?: string[]
+  attributes?: string[],
 ): void {
   path = deviceData.paths.add(path);
   let f = 1;
@@ -403,7 +377,7 @@ export function track(
 
 export function clearTrackers(
   deviceData: DeviceData,
-  tracker: string | string[]
+  tracker: string | string[],
 ): void {
   if (Array.isArray(tracker)) {
     for (const v of deviceData.trackers.values())

lib/extensions.ts

@@ -1,41 +1,16 @@
-/**
+import { spawn, ChildProcess } from "node:child_process";
- * Copyright 2013-2019  GenieACS Inc.
+import * as crypto from "node:crypto";
- *
+import readline from "node:readline";
- * This file is part of GenieACS.
+import * as config from "./config.ts";
- *
+import { Fault } from "./types.ts";
- * GenieACS is free software: you can redistribute it and/or modify
+import { ROOT_DIR } from "./config.ts";
- * it under the terms of the GNU Affero General Public License as
+import * as logger from "./logger.ts";
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { spawn, ChildProcess } from "child_process";
-import * as crypto from "crypto";
-import * as config from "./config";
-import { Fault } from "./types";
-import { ROOT_DIR } from "./config";
 
 const TIMEOUT = +config.get("EXT_TIMEOUT");
 
 const processes: { [script: string]: ChildProcess } = {};
 const jobs = new Map();
 
-function messageHandler(message): void {
-  const func = jobs.get(message[0]);
-  if (func) {
-    jobs.delete(message[0]);
-    func({ fault: message[1], value: message[2] });
-  }
-}
-
 export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
   return new Promise((resolve) => {
     const scriptName = args[0];
@@ -45,7 +20,7 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
 
     if (!processes[scriptName]) {
       const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
-        stdio: ["inherit", "inherit", "inherit", "ipc"],
+        stdio: ["ignore", "pipe", "pipe", "ipc"],
       });
       processes[scriptName] = p;
 
@@ -68,7 +43,26 @@ export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
         if (processes[scriptName] === p) delete processes[scriptName];
       });
 
-      p.on("message", messageHandler);
+      p.on("message", (message) => {
+        const func = jobs.get(message[0]);
+        if (func) {
+          jobs.delete(message[0]);
+          // Wait for any disconnect even to fire
+          setTimeout(() => {
+            func({ fault: message[1], value: message[2] });
+          });
+        }
+      });
+
+      const rlstdout = readline.createInterface(p.stdout);
+      rlstdout.on("line", (line) => {
+        logger.info({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
+      });
+
+      const rlstderr = readline.createInterface(p.stderr);
+      rlstderr.on("line", (line) => {
+        logger.warn({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
+      });
     }
 
     setTimeout(() => {
@@ -110,6 +104,6 @@ export async function killAll(): Promise<void> {
     Object.entries(processes).map(([k, p]) => {
       delete processes[k];
       return kill(p);
-    })
+    }),
   );
 }

lib/forwarded.ts

@@ -1,26 +1,8 @@
-/**
+import { IncomingMessage } from "node:http";
- * Copyright 2013-2019  GenieACS Inc.
+import { TLSSocket } from "node:tls";
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { IncomingMessage } from "http";
-import { TLSSocket } from "tls";
 import { parseCIDR, parse, IPv6, IPv4 } from "ipaddr.js";
-import * as config from "./config";
+import * as config from "./config.ts";
+import { getSocketEndpoints } from "./server.ts";
 
 interface RequestOrigin {
   localAddress: string;
@@ -98,19 +80,22 @@ export function getRequestOrigin(request: IncomingMessage): RequestOrigin {
   let origin = cache.get(request);
   if (!origin) {
     const soc = request.socket;
+    const socketEndpoints = getSocketEndpoints(soc);
     origin = {
-      localAddress: soc.localAddress,
+      localAddress: socketEndpoints.localAddress,
-      localPort: soc.localPort,
+      localPort: socketEndpoints.localPort,
-      remoteAddress: soc.remoteAddress,
+      remoteAddress: socketEndpoints.remoteAddress,
-      remotePort: soc.remotePort,
+      remotePort: socketEndpoints.remotePort,
       host: request.headers["host"],
       encrypted: !!(request.socket as TLSSocket).encrypted,
     };
 
     const header = request.headers["forwarded"];
     if (header) {
-      const ip = parse(soc.remoteAddress) as IPv4;
+      const ip = parse(socketEndpoints.remoteAddress);
-      if (cidrs.some((cidr) => ip.match(cidr as [IPv4, number]))) {
+      if (
+        cidrs.some((cidr) => ip.kind() === cidr[0].kind() && ip.match(cidr))
+      ) {
         const parsed = parseForwardedHeader(header);
 
         if (parsed["proto"] === "https") {

lib/fs.ts

@@ -1,54 +1,105 @@
-/**
+import * as url from "node:url";
- * Copyright 2013-2019  GenieACS Inc.
+import { IncomingMessage, ServerResponse } from "node:http";
- *
+import { PassThrough, pipeline, Readable } from "node:stream";
- * This file is part of GenieACS.
+import { createHash } from "node:crypto";
- *
+import { filesBucket, collections } from "./db/db.ts";
- * GenieACS is free software: you can redistribute it and/or modify
+import * as logger from "./logger.ts";
- * it under the terms of the GNU Affero General Public License as
+import { getRequestOrigin } from "./forwarded.ts";
- * published by the Free Software Foundation, either version 3 of the
+import memoize from "./common/memoize.ts";
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
 
-import * as url from "url";
+const getFile = memoize(
-import * as querystring from "querystring";
+  async (
-import { IncomingMessage, ServerResponse } from "http";
+    etag: string,
-import { Collection, GridFSBucket } from "mongodb";
+    size: number,
-import { onConnect } from "./db";
+    filename: string,
-import * as logger from "./logger";
+  ): Promise<Iterable<Buffer>> => {
-import { getRequestOrigin } from "./forwarded";
+    const chunks: Buffer[] = [];
+    // Using for-await over the download stream can throw ERR_STREAM_PREMATURE_CLOSE
+    // for very small files. Possibly a bug in MongoDB driver or Nodejs itself.
+    // Using a PassThrough stream to avoid this.
+    const downloadStream = pipeline(
+      filesBucket.openDownloadStreamByName(filename),
+      new PassThrough(),
+      (err) => {
+        if (err) throw err;
+      },
+    );
+    for await (const chunk of downloadStream) chunks.push(chunk);
+    // Node 12-14 don't throw error when stream is closed prematurely.
+    // However, we don't need to check for that since we're checking file size.
+    if (size !== chunks.reduce((a, b) => a + b.length, 0))
+      throw new Error("File size mismatch");
+    return chunks;
+  },
+);
 
-let filesCollection: Collection;
+async function* partialContent(
-let filesBucket: GridFSBucket;
+  chunks: Iterable<Buffer>,
+  start: number,
+  end: number,
+): AsyncIterable<Buffer> {
+  let bytesToSkip = start;
+  let bytesToRead = end - start;
 
-onConnect(async (db) => {
+  for (let chunk of chunks) {
-  filesCollection = db.collection("fs.files");
+    if (bytesToRead <= 0) return;
-  filesBucket = new GridFSBucket(db);
+    if (bytesToSkip >= chunk.length) {
-});
+      bytesToSkip -= chunk.length;
+      continue;
+    }
+    chunk = chunk.subarray(bytesToSkip, bytesToSkip + bytesToRead);
+    bytesToRead -= chunk.length;
+    bytesToSkip = 0;
+    yield chunk;
+  }
+}
 
-export function listener(
+function generateETag(file: {
+  _id: string;
+  uploadDate: Date;
+  length: number;
+}): string {
+  const hash = createHash("md5");
+  hash.update(`${file._id}-${file.uploadDate.getTime()}-${file.length}`);
+  return hash.digest("hex");
+}
+
+function matchEtag(etag: string, header: string): boolean {
+  for (let t of header.split(",")) {
+    t = t.trim();
+    if (t.startsWith("W/")) t = t.substring(2);
+    try {
+      t = JSON.parse(t);
+    } catch (e) {
+      // Ignore
+    }
+    if (t === "*") return true;
+    if (etag === t) return true;
+  }
+  return false;
+}
+
+export async function listener(
   request: IncomingMessage,
-  response: ServerResponse
+  response: ServerResponse,
-): void {
+): Promise<void> {
+  if (request.method !== "GET" && request.method !== "HEAD") {
+    response.writeHead(405, { Allow: "GET, HEAD" });
+    response.end("405 Method Not Allowed");
+    return;
+  }
+
   const urlParts = url.parse(request.url, true);
-  if (request.method === "GET") {
+  const filename = decodeURIComponent(urlParts.pathname.substring(1));
-    const filename = querystring.unescape(urlParts.pathname.substring(1));
 
   const log = {
     message: "Fetch file",
     filename: filename,
     remoteAddress: getRequestOrigin(request).remoteAddress,
+    method: request.method,
   };
 
-    filesCollection.findOne({ _id: filename }, (err, file) => {
+  const file = await collections.files.findOne({ _id: filename });
-      if (err) throw err;
 
   if (!file) {
     response.writeHead(404);
@@ -58,18 +109,95 @@ export function listener(
     return;
   }
 
-      response.writeHead(200, {
-        "Content-Type": file.contentType || "application/octet-stream",
-        "Content-Length": file.length,
-      });
-
-      const downloadStream = filesBucket.openDownloadStreamByName(filename);
-      downloadStream.pipe(response);
-
   logger.accessInfo(log);
-    });
+
-  } else {
+  const etag = generateETag(file);
-    response.writeHead(405, { Allow: "GET" });
+  const lastModified = file["uploadDate"];
-    response.end("405 Method Not Allowed");
+  lastModified.setMilliseconds(0);
+
+  let status = 200;
+  let start = 0;
+  let end = file.length;
+
+  if (request.headers["if-match"])
+    if (!matchEtag(etag, request.headers["if-match"])) status = 412;
+
+  if (request.headers["if-unmodified-since"]) {
+    const d = new Date(request.headers["if-unmodified-since"]);
+    if (lastModified > d) status = 412;
   }
+
+  if (request.headers["if-none-match"]) {
+    if (matchEtag(etag, request.headers["if-none-match"])) status = 304;
+  } else if (request.headers["if-modified-since"]) {
+    const d = new Date(request.headers["if-modified-since"]);
+    if (lastModified <= d) status = 304;
+  }
+
+  if (request.headers.range && status === 200) {
+    const match = request.headers.range.match(/^bytes=(\d*)-(\d*)$/);
+    status = 416;
+    if (match && (match[1] || match[2])) {
+      if (match[2]) end = parseInt(match[2]) + 1;
+      if (match[1]) start = parseInt(match[1]);
+      else start = file.length - parseInt(match[2]);
+      if (start < end && end <= file.length) status = 206;
+    }
+
+    if (request.headers["if-range"]) {
+      const h = request.headers["if-range"] as string;
+      const d = new Date(h);
+      if (!matchEtag(etag, h) && !(lastModified <= d)) {
+        status = 200;
+        start = 0;
+        end = file.length;
+      }
+    }
+  }
+
+  if (status === 412) {
+    response.writeHead(412);
+    response.end();
+    return;
+  }
+
+  if (status === 304) {
+    response.writeHead(304, {
+      ETag: etag,
+      "Last-Modified": lastModified.toUTCString(),
+    });
+    response.end();
+    return;
+  }
+
+  if (status === 416) {
+    response.writeHead(416, {
+      "Content-Range": `bytes */${file.length}`,
+      "Content-Length": "0",
+    });
+    response.end();
+    return;
+  }
+
+  response.writeHead(status, {
+    "Content-Type": "application/octet-stream",
+    "Content-Length": end - start,
+    "Accept-Ranges": "bytes",
+    ETag: etag,
+    "Last-Modified": lastModified.toUTCString(),
+    ...(status === 206 && {
+      "Content-Range": `bytes ${start}-${end - 1}/${file.length}`,
+    }),
+  });
+
+  if (request.method === "HEAD") {
+    response.end();
+    return;
+  }
+
+  const chunks = await getFile(etag, file.length, filename);
+
+  pipeline(Readable.from(partialContent(chunks, start, end)), response, () => {
+    // Ignore errors resulting from client disconnecting
+  });
 }

lib/gpn-heuristic.ts

@@ -1,23 +1,4 @@
-import Path from "./common/path";
+import Path from "./common/path.ts";
-
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
 
 const WILDCARD_MULTIPLIER = 2;
 const UNDISCOVERED_DEPTH = 7;
@@ -31,7 +12,7 @@ const UNDISCOVERED_DEPTH = 7;
 // point has been discovered.
 export function estimateGpnCount(
   gpnPatterns: [Path, number][],
-  depth = 0
+  depth = 0,
 ): number {
   const children: { [segment: string]: [Path, number][] } = {};
   const wildcardChildren: [Path, number][] = [];

lib/init.ts

@@ -1,26 +1,14 @@
-/**
+import { getRevision, getUiConfig, getUsers } from "./ui/local-cache.ts";
- * Copyright 2013-2019  GenieACS Inc.
+import { generateSalt, hashPassword } from "./auth.ts";
- *
+import { collections } from "./db/db.ts";
- * This file is part of GenieACS.
+import {
- *
+  putConfig,
- * GenieACS is free software: you can redistribute it and/or modify
+  putPermission,
- * it under the terms of the GNU Affero General Public License as
+  putPreset,
- * published by the Free Software Foundation, either version 3 of the
+  putProvision,
- * License, or (at your option) any later version.
+  putUser,
- *
+} from "./ui/db.ts";
- * GenieACS is distributed in the hope that it will be useful,
+import { del } from "./cache.ts";
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as localCache from "./local-cache";
-import { generateSalt, hashPassword } from "./auth";
-import * as db from "./ui/db";
-import { del } from "./cache";
 
 interface Status {
   users: boolean;
@@ -59,7 +47,7 @@ const INFORM_SCRIPT = `
 // Device ID as user name
 const username = declare("DeviceID.ID", {value: 1}).value[0]
 
-// Password will be fixed a given device because Math.random() is seeded with devcie ID by default.
+// Password will be fixed for a given device because Math.random() is seeded with device ID by default.
 const password = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER).toString(36);
 
 const informInterval = 300;
@@ -84,14 +72,16 @@ declare("Device.ManagementServer.PeriodicInformTime", {value: daily}, {value: in
 `.trim();
 
 export async function getStatus(): Promise<Status> {
-  const configSnapshot = await localCache.getCurrentSnapshot();
+  const [configSnapshot, presetCount] = await Promise.all([
-  const users = localCache.getUsers(configSnapshot);
+    getRevision(),
-  const presets = localCache.getPresets(configSnapshot);
+    collections.presets.countDocuments(),
-  const ui = localCache.getUiConfig(configSnapshot);
+  ]);
+  const users = getUsers(configSnapshot);
+  const ui = getUiConfig(configSnapshot);
 
   return {
     users: !Object.keys(users).length,
-    presets: !presets.length,
+    presets: !presetCount,
     filters: !Object.keys(ui["filters"]).length,
     device: !Object.keys(ui["device"]).length,
     index: !Object.keys(ui["index"]).length,
@@ -257,7 +247,7 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
           "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
       },
       { _id: "ui.device.3.type", value: "'parameter-table'" },
-      { _id: "ui.device.3.label", value: "'WLAN hosts'" },
+      { _id: "ui.device.3.label", value: "'LAN hosts'" },
       {
         _id: "ui.device.3.parameter",
         value: "InternetGatewayDevice.LANDevice.1.Hosts.Host",
@@ -395,7 +385,7 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
   if (resources["permissions"]) {
     for (const p of resources["permissions"]) {
       p["_id"] = `${p["role"]}:${p["resource"]}:${p["access"]}`;
-      proms.push(db.putPermission(p["_id"], p));
+      proms.push(putPermission(p["_id"], p));
     }
   }
 
@@ -406,21 +396,21 @@ export async function seed(options: Record<string, boolean>): Promise<void> {
       u["roles"] = (u["roles"] || []).join(",");
       u["_id"] = u["username"];
       delete u["username"];
-      proms.push(db.putUser(u["_id"], u));
+      proms.push(putUser(u["_id"], u));
     }
   }
 
   if (resources["provisions"]) {
     for (const p of resources["provisions"])
-      proms.push(db.putProvision(p["_id"], p));
+      proms.push(putProvision(p["_id"], p));
   }
 
   if (resources["presets"])
-    for (const p of resources["presets"]) proms.push(db.putPreset(p["_id"], p));
+    for (const p of resources["presets"]) proms.push(putPreset(p["_id"], p));
 
   if (resources["config"])
-    for (const c of resources["config"]) proms.push(db.putConfig(c["_id"], c));
+    for (const c of resources["config"]) proms.push(putConfig(c["_id"], c));
 
   await proms;
-  return del("presets_hash");
+  await Promise.all([del("ui-local-cache-hash"), del("cwmp-local-cache-hash")]);
 }

lib/instance-set.ts

@@ -1,28 +1,9 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 interface Instance {
   [name: string]: string;
 }
 
 export default class InstanceSet {
-  private set: Set<Instance>;
+  private declare set: Set<Instance>;
 
   public constructor() {
     this.set = new Set();

lib/local-cache.ts

@@ -1,548 +1,73 @@
-/**
+import { setTimeoutPromise } from "./util.ts";
- * Copyright 2013-2019  GenieACS Inc.
+import { get, set } from "./cache.ts";
- *
+import { acquireLock, releaseLock } from "./lock.ts";
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
 
-import * as vm from "vm";
+const REFRESH = 5000;
-import * as crypto from "crypto";
+const EVICT_TIMEOUT = 120000;
-import * as config from "./config";
-import * as db from "./db";
-import * as cache from "./cache";
-import { convertOldPrecondition } from "./mongodb-functions";
-import * as logger from "./logger";
-import * as scheduling from "./scheduling";
-import * as expression from "./common/expression";
-import { parse } from "./common/expression-parser";
-import {
-  Preset,
-  Expression,
-  Provisions,
-  VirtualParameters,
-  Files,
-  Users,
-  Permissions,
-  Config,
-  UiConfig,
-} from "./types";
 
-interface Snapshot {
+export class LocalCache<T> {
-  presets: Preset[];
+  private nextRefresh = 1;
-  provisions: Provisions;
+  private currentRevision: string = null;
-  virtualParameters: VirtualParameters;
+  private snapshots: Map<string, T> = new Map();
-  files: Files;
-  permissions: Permissions;
-  users: Users;
-  config: Config;
-  ui: UiConfig;
-}
 
-const REFRESH = 3000;
+  constructor(
-const EVICT_TIMEOUT = 60000;
+    private cacheKey: string,
+    private callback: () => Promise<[string, T]>,
+  ) {}
 
-const snapshots = new Map<string, Snapshot>();
+  async getRevision(): Promise<string> {
-let currentSnapshot: string = null;
+    if (Date.now() > this.nextRefresh) await this.refresh();
-let nextRefresh = 1;
+    return this.currentRevision;
-
-function computeHash(snapshot): string {
-  // MD5 hash for presets, provisions, virtual parameters for detecting changes
-  const h = crypto.createHash("md5");
-  for (const p of snapshot.presets) {
-    h.update(JSON.stringify(p.name));
-    h.update(JSON.stringify(p.channel));
-    h.update(JSON.stringify(p.schedule));
-    h.update(JSON.stringify(p.events));
-    h.update(JSON.stringify(p.precondition));
-    h.update(JSON.stringify(p.provisions));
   }
 
-  let keys;
+  hasRevision(revision: string): boolean {
-
+    return this.snapshots.has(revision);
-  keys = Object.keys(snapshot.provisions).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(snapshot.provisions[k].md5);
-
-  keys = Object.keys(snapshot.virtualParameters).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(snapshot.virtualParameters[k].md5);
-
-  keys = Object.keys(snapshot.config).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.config[k]));
-
-  keys = Object.keys(snapshot.files).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.files[k]));
-
-  keys = Object.keys(snapshot.users).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.users[k]));
-
-  const roles = Object.keys(snapshot.permissions).sort();
-  h.update(JSON.stringify(roles));
-  for (const r of roles) {
-    const levels = Object.keys(snapshot.permissions[r]).sort();
-    h.update(JSON.stringify(levels));
-    for (const l of levels) {
-      keys = Object.keys(snapshot.permissions[r][l]).sort();
-      h.update(JSON.stringify(keys));
-      for (const k of keys)
-        h.update(JSON.stringify(snapshot.permissions[r][l][k]));
-    }
   }
 
-  return h.digest("hex");
+  get(revision: string): T {
-}
+    const snapshot = this.snapshots.get(revision);
-
+    if (!snapshot) throw new Error("Cache snapshot does not exist");
-function flattenObject(
+    return snapshot;
-  src: Record<string, unknown>,
-  prefix = "",
-  dst = {}
-): Record<string, unknown> {
-  for (const k of Object.keys(src)) {
-    const v = src[k];
-    if (typeof v === "object" && !Array.isArray(v))
-      flattenObject(v as Record<string, unknown>, `${prefix}${k}.`, dst);
-    else dst[`${prefix}${k}`] = v;
-  }
-  return dst;
-}
-
-async function fetchPresets(): Promise<Preset[]> {
-  const res = await db.getPresets();
-  let objects = await db.getObjects();
-
-  objects = objects.map((obj) => {
-    // Flatten object
-    obj = flattenObject(obj);
-
-    // If no keys are defined, consider all parameters as keys to keep the
-    // same behavior from v1.0
-    if (!obj["_keys"] || !obj["_keys"].length)
-      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
-
-    return obj;
-  });
-
-  res.sort((a, b) => {
-    if (a["weight"] === b["weight"])
-      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
-    else return a["weight"] - b["weight"];
-  });
-
-  const presets = [] as Preset[];
-  for (const preset of res) {
-    let schedule: { md5: string; duration: number; schedule: any } = null;
-    if (preset["schedule"]) {
-      const parts = preset["schedule"].trim().split(/\s+/);
-      schedule = {
-        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
-        duration: null,
-        schedule: null,
-      };
-
-      try {
-        schedule.duration = +parts.shift() * 1000;
-        schedule.schedule = scheduling.parseCron(parts.join(" "));
-      } catch (err) {
-        logger.warn({
-          message: "Invalid preset schedule",
-          preset: preset["_id"],
-          schedule: preset["schedule"],
-        });
-        schedule.schedule = false;
-      }
   }
 
-    const events = preset["events"] || {};
+  async refresh(): Promise<void> {
-    let precondition = true as Expression;
+    if (!this.nextRefresh) {
-    if (preset["precondition"]) {
+      await setTimeoutPromise(20);
-      try {
+      await this.refresh();
-        precondition = parse(preset["precondition"]);
-      } catch (error) {
-        precondition = convertOldPrecondition(
-          JSON.parse(preset["precondition"])
-        );
-      }
-
-      // Simplify expression
-      precondition = expression.evaluate(precondition);
-    }
-
-    const _provisions = preset["provisions"] || [];
-
-    // Generate provisions from the old configuration format
-    for (const c of preset["configurations"]) {
-      switch (c.type) {
-        case "age":
-          _provisions.push(["refresh", c.name, +c.age]);
-          break;
-
-        case "value":
-          _provisions.push(["value", c.name, c.value]);
-          break;
-
-        case "add_tag":
-          _provisions.push(["tag", c.tag, true]);
-          break;
-
-        case "delete_tag":
-          _provisions.push(["tag", c.tag, false]);
-          break;
-
-        case "provision":
-          _provisions.push([c.name].concat(c.args || []));
-          break;
-
-        case "add_object":
-          for (const obj of objects) {
-            if (obj["_id"] === c.object) {
-              const alias = obj["_keys"]
-                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
-                .join(",");
-              const p = `${c.name}.[${alias}]`;
-              _provisions.push(["instances", p, 1]);
-
-              for (const k in obj) {
-                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
-                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
-              }
-            }
-          }
-
-          break;
-
-        case "delete_object":
-          for (const obj of objects) {
-            if (obj["_id"] === c.object) {
-              const alias = obj["_keys"]
-                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
-                .join(",");
-              const p = `${c.name}.[${alias}]`;
-              _provisions.push(["instances", p, 0]);
-            }
-          }
-
-          break;
-
-        default:
-          throw new Error(`Unknown configuration type ${c.type}`);
-      }
-    }
-
-    presets.push({
-      name: preset["_id"],
-      channel: (preset["channel"] as string) || "default",
-      schedule: schedule,
-      events: events,
-      precondition: precondition,
-      provisions: _provisions,
-    });
-  }
-
-  return presets;
-}
-
-async function fetchProvisions(): Promise<Provisions> {
-  const res = await db.getProvisions();
-
-  const provisions = {};
-  for (const r of res) {
-    provisions[r["_id"]] = {};
-    provisions[r["_id"]].md5 = crypto
-      .createHash("md5")
-      .update(r["script"])
-      .digest("hex");
-    provisions[
-      r["_id"]
-    ].script = new vm.Script(
-      `"use strict";(function(){\n${r["script"]}\n})();`,
-      { filename: r["_id"], lineOffset: -1, timeout: 50 }
-    );
-  }
-
-  return provisions;
-}
-
-async function fetchVirtualParameters(): Promise<VirtualParameters> {
-  const res = await db.getVirtualParameters();
-
-  const virtualParameters = {};
-  for (const r of res) {
-    virtualParameters[r["_id"]] = {};
-    virtualParameters[r["_id"]].md5 = crypto
-      .createHash("md5")
-      .update(r["script"])
-      .digest("hex");
-    virtualParameters[
-      r["_id"]
-    ].script = new vm.Script(
-      `"use strict";(function(){\n${r["script"]}\n})();`,
-      { filename: r["_id"], lineOffset: -1, timeout: 50 }
-    );
-  }
-
-  return virtualParameters;
-}
-
-async function fetchPermissions(): Promise<Permissions> {
-  const perms = await db.getPermissions();
-  const permissions: Permissions = {};
-
-  for (const p of perms) {
-    if (!permissions[p.role]) permissions[p.role] = {};
-    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
-
-    permissions[p.role][p.access][p.resource] = {
-      access: p.access,
-      filter: parse(p.filter || "true"),
-    };
-    if (p.validate)
-      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
-  }
-
-  return permissions;
-}
-
-async function fetchFiles(): Promise<Files> {
-  const res = await db.getFiles();
-  const files = {};
-
-  for (const r of res) {
-    const id = r["filename"] || r["_id"].toString();
-    files[id] = {};
-    files[id].length = r["length"];
-    files[id].md5 = r["md5"];
-    files[id].contentType = r["contentType"];
-  }
-
-  return files;
-}
-
-async function fetchUsers(): Promise<Users> {
-  const _users = await db.getUsers();
-  const users = {};
-
-  for (const user of _users) {
-    users[user._id] = {
-      password: user.password,
-      salt: user.salt,
-      roles: user.roles.split(",").map((s) => s.trim()),
-    };
-  }
-
-  return users;
-}
-
-async function fetchConfig(): Promise<[Config, UiConfig]> {
-  const conf = await db.getConfig();
-
-  conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
-
-  const ui = {
-    filters: {},
-    device: {},
-    index: {},
-    overview: {},
-  };
-  const _config = {};
-
-  for (const c of conf) {
-    // Evaluate expressions to simplify them
-    const val = expression.evaluate(c.value);
-    _config[c.id] = val;
-    if (c.id.startsWith("ui.")) {
-      const keys = c.id.split(".");
-      // remove the first key(ui)
-      keys.shift();
-      let ref = ui;
-      while (keys.length > 1) {
-        const k = keys.shift();
-        if (typeof ref[k] !== "object") ref[k] = {};
-        ref = ref[k];
-      }
-      ref[keys[0]] = val;
-    }
-  }
-
-  return [_config, ui];
-}
-
-async function refresh(): Promise<void> {
-  if (!nextRefresh) {
-    await new Promise((resolve) => setTimeout(resolve, 20));
-    await refresh();
       return;
     }
 
-  nextRefresh = 0;
     const now = Date.now();
+    if (now < this.nextRefresh) return;
+    this.nextRefresh = 0;
 
-  const dbHash = await cache.get("presets_hash");
+    const dbHash = await get(this.cacheKey);
 
-  if (currentSnapshot && dbHash === currentSnapshot) {
+    if (this.currentRevision && dbHash === this.currentRevision) {
-    nextRefresh = now + (REFRESH - (now % REFRESH));
+      this.nextRefresh = now + (REFRESH - (now % REFRESH));
       return;
     }
 
-  const lockToken = await cache.acquireLock("presets_hash_lock", 4000, 4000);
+    const lockToken = await acquireLock(this.cacheKey, 5000);
 
-  const res = await Promise.all([
+    const [hash, snapshot] = await this.callback();
-    fetchPresets(),
-    fetchProvisions(),
-    fetchVirtualParameters(),
-    fetchFiles(),
-    fetchPermissions(),
-    fetchUsers(),
-    fetchConfig(),
-  ]);
 
-  const snapshot = {
+    if (this.currentRevision) {
-    presets: res[0],
+      const r = this.currentRevision;
-    provisions: res[1],
+      const s = this.snapshots.get(r);
-    virtualParameters: res[2],
-    files: res[3],
-    permissions: res[4],
-    users: res[5],
-    config: res[6][0],
-    ui: res[6][1],
-  };
-
-  if (currentSnapshot) {
-    const h = currentSnapshot;
-    const s = snapshots.get(h);
       setTimeout(() => {
-      if (snapshots.get(h) === s) snapshots.delete(h);
+        if (this.snapshots.get(r) === s) this.snapshots.delete(r);
       }, EVICT_TIMEOUT).unref();
     }
 
-  currentSnapshot = computeHash(snapshot);
+    this.currentRevision = hash;
-  snapshots.set(currentSnapshot, snapshot);
+    this.snapshots.set(hash, snapshot);
-  await cache.set("presets_hash", currentSnapshot, 300);
-  await cache.releaseLock("presets_hash_lock", lockToken);
 
-  nextRefresh = now + (REFRESH - (now % REFRESH));
+    if (lockToken) {
-}
+      if (hash !== dbHash) await set(this.cacheKey, hash, 300);
-
+      await releaseLock(this.cacheKey, lockToken);
-export async function getCurrentSnapshot(): Promise<string> {
-  if (Date.now() > nextRefresh) await refresh();
-  return currentSnapshot;
-}
-
-export function hasSnapshot(hash: string): boolean {
-  return snapshots.has(hash);
-}
-
-export function getPresets(snapshotKey: string): Preset[] {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.presets;
-}
-
-export function getProvisions(snapshotKey: string): Provisions {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.provisions;
-}
-
-export function getVirtualParameters(snapshotKey: string): VirtualParameters {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.virtualParameters;
-}
-
-export function getFiles(snapshotKey: string): Files {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.files;
-}
-
-export function getConfig(
-  snapshotKey: string,
-  key: string,
-  context: Record<string, unknown>,
-  now: number,
-  cb?: (e: Expression) => Expression
-): string | number | boolean | null {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  const oldOpts = {
-    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
-    "cwmp.debug": "DEBUG",
-    "cwmp.retryDelay": "RETRY_DELAY",
-    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
-    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
-    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
-    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
-    "cwmp.cookiesPath": "COOKIES_PATH",
-    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
-    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
-    "cwmp.connectionRequestAllowBasicAuth":
-      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
-    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
-    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
-    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
-  };
-
-  if (!(key in snapshot.config)) {
-    if (key in oldOpts) {
-      let id;
-      if (context && context["id"]) {
-        id = context["id"];
-      } else if (cb) {
-        id = cb(["PARAM", "DeviceID.ID"]);
-        if (Array.isArray(id)) id = null;
-      }
-      return config.get(oldOpts[key], id);
-    }
-    return null;
     }
 
-  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+    this.nextRefresh = now + (REFRESH - (now % REFRESH));
-  return Array.isArray(v) ? null : v;
+  }
-}
-
-export function getConfigExpression(
-  snapshotKey: string,
-  key: string
-): Expression {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.config[key];
-}
-
-export function getUsers(snapshotKey: string): Users {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.users;
-}
-
-export function getPermissions(snapshotKey: string): Permissions {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.permissions;
-}
-
-export function getUiConfig(snapshotKey: string): UiConfig {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.ui;
 }

lib/lock.ts

@@ -0,0 +1,50 @@
+import { collections } from "./db/db.ts";
+
+const CLOCK_SKEW_TOLERANCE = 30000;
+
+export async function acquireLock(
+  lockName: string,
+  ttl: number,
+  timeout = 0,
+  token = Math.random().toString(36).slice(2),
+): Promise<string> {
+  try {
+    const now = Date.now();
+    const r = await collections.locks.findOneAndUpdate(
+      { _id: lockName, value: token },
+      {
+        $set: {
+          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
+        },
+        $currentDate: { timestamp: true },
+      },
+      { upsert: true, returnDocument: "after" },
+    );
+    if (Math.abs(r.value.timestamp.getTime() - now) > CLOCK_SKEW_TOLERANCE)
+      throw new Error("Database clock skew too great");
+  } catch (err) {
+    if (err.code !== 11000) throw err;
+    if (!(timeout > 0)) return null;
+    const w = 50 + Math.random() * 50;
+    await new Promise((resolve) => setTimeout(resolve, w));
+    return acquireLock(lockName, ttl, timeout - w, token);
+  }
+
+  return token;
+}
+
+export async function releaseLock(
+  lockName: string,
+  token: string,
+): Promise<void> {
+  const res = await collections.locks.deleteOne({
+    _id: lockName,
+    value: token,
+  });
+  if (res.deletedCount !== 1) throw new Error("Lock expired");
+}
+
+export async function getToken(lockName: string): Promise<string> {
+  const res = await collections.locks.findOne({ _id: lockName });
+  return res?.value;
+}

lib/logger.ts

@@ -1,27 +1,7 @@
-/**
+import * as fs from "node:fs";
- * Copyright 2013-2019  GenieACS Inc.
+import * as os from "node:os";
- *
+import * as config from "./config.ts";
- * This file is part of GenieACS.
+import { getRequestOrigin } from "./forwarded.ts";
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as fs from "fs";
-import * as os from "os";
-
-import * as config from "./config";
-import { getRequestOrigin } from "./forwarded";
 import {
   SessionContext,
   AcsRequest,
@@ -29,7 +9,7 @@ import {
   CpeFault,
   InformRequest,
   Fault,
-} from "./types";
+} from "./types.ts";
 
 const REOPEN_EVERY = 60000;
 
@@ -156,13 +136,13 @@ export function close(): void {
 }
 
 export function flatten(
-  details: Record<string, unknown>
+  details: Record<string, unknown>,
 ): Record<string, unknown> {
   if (details.sessionContext) {
     const sessionContext = details.sessionContext as SessionContext;
     details.deviceId = sessionContext.deviceId;
     details.remoteAddress = getRequestOrigin(
-      sessionContext.httpRequest
+      sessionContext.httpRequest,
     ).remoteAddress;
     delete details.sessionContext;
   }
@@ -220,7 +200,7 @@ export function flatten(
   // For genieacs-ui
   if (details.context) {
     details.remoteAddress = getRequestOrigin(
-      details.context["req"]
+      details.context["req"],
     ).remoteAddress;
     if (details.context["state"].user)
       details.user = details.context["state"].user.username;
@@ -235,7 +215,7 @@ export function flatten(
 
 function formatJson(
   details: Record<string, unknown>,
-  systemd: boolean
+  systemd: boolean,
 ): string {
   if (systemd) {
     let severity = "";
@@ -251,7 +231,7 @@ function formatJson(
 
 function formatSimple(
   details: Record<string, unknown>,
-  systemd: boolean
+  systemd: boolean,
 ): string {
   const skip = {
     user: true,
@@ -292,11 +272,9 @@ function formatSimple(
     return `${severity}${remote}${details.message}${meta}${os.EOL}`;
   }
 
-  return `${
+  return `${details.timestamp} [${(
-    details.timestamp
+    details.severity as string
-  } [${(details.severity as string).toUpperCase()}] ${remote}${
+  ).toUpperCase()}] ${remote}${details.message}${meta}${os.EOL}`;
-    details.message
-  }${meta}${os.EOL}`;
 }
 
 function log(details: Record<string, unknown>): void {

lib/mongodb-functions.ts

@@ -1,684 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ObjectID } from "mongodb";
-import { map, parse, stringify, parseList } from "./common/expression-parser";
-import { likePatternToRegExp } from "./common/expression";
-import { Expression, Fault, Task } from "./types";
-import { decodeTag, encodeTag } from "./common";
-
-const isArray = Array.isArray;
-
-export function processDeviceFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
-    if (!isArray(exp)) return exp;
-
-    if (exp[0] === "PARAM") {
-      const p = exp[1];
-      if (p === "DeviceID.ID") return ["PARAM", "_id"];
-      else if (p === "DeviceID") return ["PARAM", "_deviceId"];
-      else if (p.startsWith("DeviceID."))
-        return ["PARAM", "_deviceId._" + p.slice(9)];
-      else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
-      else if (p === "Events.Registered") return ["PARAM", "_registered"];
-      else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
-      else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"];
-    } else if (
-      isArray(exp[1]) &&
-      exp[1][0] === "PARAM" &&
-      exp[1][1].startsWith("Tags.")
-    ) {
-      const t = decodeTag(exp[1][1].slice(5));
-      if (exp[0] === "IS NULL") return ["_tags", t, false];
-      else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
-      else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
-      else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
-    } else if (
-      [
-        "=",
-        "<>",
-        ">",
-        ">=",
-        "<",
-        "<=",
-        "LIKE",
-        "NOT LIKE",
-        "IS NULL",
-        "IS NOT NULL",
-      ].includes(exp[0])
-    ) {
-      let e = map(exp, (ee) => {
-        if (
-          isArray(ee) &&
-          ee[0] === "PARAM" &&
-          typeof ee[1] === "string" &&
-          !ee[1].startsWith("_")
-        )
-          return ["PARAM", `${ee[1]}._value`];
-        return ee;
-      });
-
-      if (typeof e[2] === "number") {
-        const alt = (e as any[]).slice();
-        alt[2] = new Date(e[2]);
-        e = ["OR", e, alt];
-      }
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function processTasksFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
-    if (!isArray(exp)) return exp;
-    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
-      const e = exp.slice();
-      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectID(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
-        e[2] = new Date(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
-        e[2] = new Date(e[2]);
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function processFaultsFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
-    if (!isArray(exp)) return exp;
-    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
-      const e = exp.slice();
-      if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
-        e[2] = new Date(e[2]);
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
-  function recursive(filter, negate): Record<string, unknown> {
-    const op = filter[0];
-
-    if (op === "AND") {
-      filter = filter.filter((f) => f !== true);
-      if (filter.length === 2) return recursive(filter[1], negate);
-    }
-
-    if (op === "OR") {
-      filter = filter.filter((f) => f !== false);
-      if (filter.length === 2) return recursive(filter[1], negate);
-    }
-
-    if ((!negate && op === "AND") || (negate && op === "OR"))
-      return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
-
-    if ((!negate && op === "OR") || (negate && op === "AND"))
-      return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
-
-    if (op === "NOT") return recursive(filter[1], !negate);
-
-    if (isArray(filter[2]))
-      throw new Error(`Invalid RHS operand of ${op} clause`);
-
-    if (op === "LIKE" || op === "NOT LIKE") {
-      if (op === "NOT LIKE") negate = !negate;
-      let param;
-      let flags;
-      if (!Array.isArray(filter[1]))
-        throw new Error(`Invalid LHS operand of ${op} clause`);
-      if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
-        if (filter[2] !== filter[2].toUpperCase())
-          throw new Error(`Invalid RHS operand of ${op} clause`);
-        param = filter[1][2][1];
-        flags = "i";
-      } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
-        if (filter[2] !== filter[2].toLowerCase())
-          throw new Error(`Invalid RHS operand of ${op} clause`);
-        param = filter[1][2][1];
-        flags = "i";
-      } else if (filter[1][0] === "PARAM") {
-        param = filter[1][1];
-        flags = "";
-      } else {
-        throw new Error(`Invalid LHS operand of ${op} clause`);
-      }
-      const r = likePatternToRegExp(filter[2], filter[3], flags);
-      if (negate) return { [param]: { $nin: [r, null] } };
-      else return { [param]: r };
-    }
-
-    if (op === "_tags") {
-      let t = filter[2];
-      if (negate) t = !t;
-      if (t) return { _tags: filter[1] };
-      else return { _tags: { $ne: filter[1] } };
-    }
-
-    if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
-      throw new Error(`Invalid LHS operand of ${op} clause`);
-
-    if (op === "IS NULL") {
-      const p = filter[1][1];
-      return { [p]: null };
-    }
-
-    if (op === "IS NOT NULL") {
-      const p = filter[1][1];
-      return { [p]: { $ne: null } };
-    }
-
-    if (isArray(filter[2]) || filter[2] == null)
-      throw new Error(`Invalid RHS operand of ${op} clause`);
-
-    if (op === "=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (!negate) return { [p]: v };
-      else return { [p]: { $nin: [v, null] } };
-    }
-
-    if (op === "<>") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: v };
-      else return { [p]: { $nin: [v, null] } };
-    }
-
-    if (op === ">") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $lte: v } };
-      else return { [p]: { $gt: v } };
-    }
-
-    if (op === ">=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $lt: v } };
-      else return { [p]: { $gte: v } };
-    }
-
-    if (op === "<") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $gte: v } };
-      else return { [p]: { $lt: v } };
-    }
-
-    if (op === "<=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $gt: v } };
-      else return { [p]: { $lte: v } };
-    }
-
-    throw new Error(`Unrecognized operator ${op}`);
-  }
-
-  if (!isArray(exp)) {
-    if (exp === true) return {};
-    throw new Error("Primitives are not valid queries");
-  }
-
-  return recursive(exp, false);
-}
-
-export function processDeviceProjection(
-  projection: Record<string, 1>
-): Record<string, 1> {
-  if (!projection) return projection;
-  const p = {};
-  for (const [k, v] of Object.entries(projection)) {
-    if (k === "DeviceID.ID") {
-      p["_id"] = 1;
-    } else if (k.startsWith("DeviceID")) {
-      p["_deviceId._SerialNumber"] = v;
-      p["_deviceId._OUI"] = v;
-      p["_deviceId._ProductClass"] = v;
-      p["_deviceId._Manufacturer"] = v;
-    } else if (k.startsWith("Tags")) {
-      p["_tags"] = v;
-    } else if (k.startsWith("Events")) {
-      p["_lastInform"] = v;
-      p["_registered"] = v;
-      p["_lastBoot"] = v;
-      p["_lastBootstrap"] = v;
-    } else {
-      p[k] = v;
-    }
-  }
-
-  return p;
-}
-
-export function processDeviceSort(
-  sort: Record<string, number>
-): Record<string, number> {
-  if (!sort) return sort;
-  const s = {};
-  for (const [k, v] of Object.entries(sort)) {
-    if (k === "DeviceID.ID") s["_id"] = v;
-    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
-    else if (k === "Events.Inform") s["_lastInform"] = v;
-    else if (k === "Events.Registered") s["_registered"] = v;
-    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
-    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
-    else s[`${k}._value`] = v;
-  }
-
-  return s;
-}
-
-function parseDate(d: Date): number | string {
-  const n = +d;
-  return isNaN(n) ? "" + d : n;
-}
-
-interface FlatAttributes {
-  object?: boolean;
-  objectTimestamp?: number;
-  writable?: boolean;
-  writableTimestamp?: number;
-  value?: [string | number | boolean, string];
-  valueTimestamp?: number;
-  notification?: number;
-  notificationTimestamp?: number;
-  accessList?: string[];
-  accessListTimestamp?: number;
-}
-
-interface FlatDevice {
-  [param: string]: FlatAttributes;
-}
-
-export function flattenDevice(device: Record<string, unknown>): FlatDevice {
-  function recursive(
-    input,
-    root: string,
-    output: FlatDevice,
-    timestamp: number
-  ): void {
-    for (const [name, tree] of Object.entries(input)) {
-      if (!root) {
-        if (name === "_lastInform") {
-          output["Events.Inform"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_registered") {
-          output["Events.Registered"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_lastBoot") {
-          output["Events.1_BOOT"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_lastBootstrap") {
-          output["Events.0_BOOTSTRAP"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_id") {
-          output["DeviceID.ID"] = {
-            value: [tree as string, "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_deviceId") {
-          output["DeviceID.Manufacturer"] = {
-            value: [tree["_Manufacturer"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.OUI"] = {
-            value: [tree["_OUI"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.ProductClass"] = {
-            value: [tree["_ProductClass"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.SerialNumber"] = {
-            value: [tree["_SerialNumber"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_tags") {
-          output["Tags"] = {
-            writable: false,
-            writableTimestamp: timestamp,
-            object: true,
-            objectTimestamp: timestamp,
-          };
-
-          for (const t of tree as string[]) {
-            output[`Tags.${encodeTag(t)}`] = {
-              value: [true, "xsd:boolean"],
-              valueTimestamp: timestamp,
-              writable: true,
-              writableTimestamp: timestamp,
-              object: false,
-              objectTimestamp: timestamp,
-            };
-          }
-        }
-      }
-
-      if (name.startsWith("_")) continue;
-
-      let childrenTimestamp = timestamp;
-
-      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
-      else if (+input["_timestamp"] > timestamp)
-        childrenTimestamp = +input["_timestamp"];
-
-      const attrs: FlatAttributes = {};
-      if (tree["_value"] != null) {
-        attrs.value = [
-          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
-          tree["_type"],
-        ];
-        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
-        attrs.object = false;
-        attrs.objectTimestamp = childrenTimestamp;
-      } else if (tree["_object"] != null) {
-        attrs.object = tree["_object"];
-        attrs.objectTimestamp = childrenTimestamp;
-      }
-
-      if (tree["_writable"] != null) {
-        attrs.writable = tree["_writable"];
-        attrs.writableTimestamp = childrenTimestamp;
-      }
-
-      if (tree["_notification"] != null) {
-        attrs.notification = tree["_notification"];
-        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
-      }
-
-      if (tree["_accessList"] != null) {
-        attrs.accessList = tree["_accessList"];
-        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
-      }
-
-      const r = root ? `${root}.${name}` : name;
-      output[r] = attrs;
-
-      if (attrs.object || tree["object"] == null)
-        recursive(tree, r, output, childrenTimestamp);
-    }
-  }
-
-  const newDevice: FlatDevice = {};
-  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
-  recursive(device, "", newDevice, timestamp);
-  return newDevice;
-}
-
-export function flattenFault(fault: unknown): Fault {
-  const f = Object.assign({}, fault) as Fault;
-  if (f.timestamp) f.timestamp = +f.timestamp;
-  if (f["expiry"]) f["expiry"] = +f["expiry"];
-  return f as Fault;
-}
-
-export function flattenTask(task: unknown): Task {
-  const t = Object.assign({}, task) as Task;
-  t._id = "" + t._id;
-  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
-  if (t.expiry) t.expiry = +t.expiry;
-  return t;
-}
-
-export function convertOldPrecondition(
-  query: Record<string, unknown>
-): Expression {
-  function recursive(_query): Expression {
-    const expressions: Expression[] = [];
-    for (const [k, v] of Object.entries(_query)) {
-      if (k[0] === "$") {
-        if (k === "$and") {
-          const and: Expression = ["AND"];
-          for (const vv of Object.values(v)) and.push(recursive(vv));
-          expressions.push(and);
-        } else if (k === "$or") {
-          const or: Expression = ["OR"];
-          for (const vv of Object.values(v)) or.push(recursive(vv));
-          expressions.push(or);
-        } else {
-          throw new Error(`Operator ${k} not supported`);
-        }
-      } else if (k === "_tags") {
-        if (typeof v === "object") {
-          if (isArray(v)) throw new Error(`Invalid type`);
-          const conjs: Expression[] = [];
-          for (const [op, val] of Object.entries(v)) {
-            if (op === "$ne") {
-              if (typeof v["$ne"] !== "string")
-                throw new Error("Only string values are allowed for _tags");
-              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
-            } else if (op === "$eq") {
-              if (typeof v["$eq"] !== "string")
-                throw new Error("Only string values are allowed for _tags");
-              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
-            } else {
-              throw new Error(`Invalid tag query`);
-            }
-          }
-          if (conjs.length === 1) expressions.push(conjs[0]);
-          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
-        } else {
-          expressions.push([
-            "IS NOT NULL",
-            ["PARAM", `Tags.${encodeTag(v as string)}`],
-          ]);
-        }
-      } else if (k.startsWith("Tags.")) {
-        let exists: boolean;
-        if (typeof v === "boolean") exists = v;
-        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
-        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
-        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
-        else throw new Error(`Invalid tag query`);
-
-        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
-      } else if (typeof v === "object") {
-        if (isArray(v)) throw new Error(`Invalid type`);
-
-        const exps: Expression[] = [];
-        for (const [kk, vv] of Object.entries(v)) {
-          if (kk === "$eq") {
-            exps.push(["=", ["PARAM", k], vv]);
-          } else if (kk === "$ne") {
-            exps.push([
-              "OR",
-              ["<>", ["PARAM", k], vv],
-              ["IS NULL", ["PARAM", k]],
-            ]);
-          } else if (kk === "$lt") {
-            exps.push(["<", ["PARAM", k], vv]);
-          } else if (kk === "$lte") {
-            exps.push(["<=", ["PARAM", k], vv]);
-          } else if (kk === "$gt") {
-            exps.push([">", ["PARAM", k], vv]);
-          } else if (kk === "$gte") {
-            exps.push([">=", ["PARAM", k], vv]);
-          } else {
-            throw new Error(`Operator ${kk} not supported`);
-          }
-          if (!["string", "number", "boolean"].includes(typeof vv))
-            throw new Error(`Invalid value for ${kk} operator`);
-        }
-        if (exps.length === 1) {
-          expressions.push(exps[0]);
-        } else if (exps.length > 1) {
-          const and: Expression = ["AND"];
-          expressions.push(and.concat(exps));
-        }
-      } else {
-        expressions.push(["=", ["PARAM", k], v]);
-      }
-    }
-    if (expressions.length === 1) return expressions[0];
-    if (expressions.length === 0) return true;
-    return ["AND", ...expressions];
-  }
-
-  // empty filter
-  if (!Object.keys(query).length) return true;
-
-  return recursive(query);
-}
-
-export function flattenPreset(
-  preset: Record<string, unknown>
-): Record<string, unknown> {
-  const p = Object.assign({}, preset);
-  if (p.precondition) {
-    try {
-      // Try parse to check expression validity
-      parse(p.precondition as string);
-    } catch (error) {
-      p.precondition = convertOldPrecondition(
-        JSON.parse(p.precondition as string)
-      );
-      p.precondition = (p.precondition as string).length
-        ? stringify(p.precondition as Expression)
-        : "";
-    }
-  }
-
-  if (p.events) {
-    const e = [];
-    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
-    p.events = e.join(", ");
-  }
-
-  const provision = p.configurations[0];
-  if (
-    (p.configurations as any[]).length === 1 &&
-    provision.type === "provision" &&
-    provision.name &&
-    provision.name.length
-  ) {
-    p.provision = provision.name;
-    p.provisionArgs = provision.args
-      ? provision.args.map((a) => stringify(a)).join(", ")
-      : "";
-  }
-
-  delete p.configurations;
-  return p;
-}
-
-export function flattenFile(
-  file: Record<string, unknown>
-): Record<string, unknown> {
-  const f = {};
-  f["_id"] = file["_id"];
-  if (file.metadata) {
-    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
-    f["metadata.oui"] = file["metadata"]["oui"] || "";
-    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
-    f["metadata.version"] = file["metadata"]["version"] || "";
-  }
-  return f;
-}
-
-export function preProcessPreset(
-  data: Record<string, unknown>
-): Record<string, unknown> {
-  const preset = Object.assign({}, data);
-
-  if (!preset.precondition) preset.precondition = "";
-  // Try parse to check expression validity
-  parse(preset.precondition as string);
-
-  preset.weight = parseInt(preset.weight as string) || 0;
-
-  const events = {};
-  if (preset.events) {
-    for (let e of (preset.events as string).split(",")) {
-      let v = true;
-      e = e.trim();
-      if (e.startsWith("-")) {
-        v = false;
-        e = e.slice(1).trim();
-      }
-      if (e) events[e] = v;
-    }
-  }
-
-  preset.events = events;
-
-  if (!preset.provision) throw new Error("Invalid preset provision");
-
-  const configuration = {
-    type: "provision",
-    name: preset.provision,
-    args: null,
-  };
-
-  if (preset.provisionArgs)
-    configuration.args = parseList(preset.provisionArgs as string);
-
-  delete preset.provision;
-  delete preset.provisionArgs;
-  preset.configurations = [configuration];
-  return preset;
-}

lib/ping.ts

@@ -1,24 +1,6 @@
-/**
+import { platform } from "node:os";
- * Copyright 2013-2019  GenieACS Inc.
+import { exec } from "node:child_process";
- *
+import { domainToASCII } from "node:url";
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { platform } from "os";
-import { exec } from "child_process";
 
 export interface PingResult {
   packetsTransmitted: number;
@@ -30,33 +12,33 @@ export interface PingResult {
   mdev: number;
 }
 
-export function ping(
+function isValidHost(host: string): boolean {
-  host: string,
+  // Valid chars in IPv4, IPv6, domain names
-  callback: (err: Error, res?: PingResult, stdout?: string) => void
+  if (/^[a-zA-Z0-9\-.:[\]-]+$/.test(host)) return true;
-): void {
+
-  let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
+  // Check if input is an IDN convert to Punycode
-  switch (platform()) {
+  // Can't merge with above because domainToASCII doesn't accept IP addresses
+  return /^[a-zA-Z0-9\-.:[\]-]+$/.test(domainToASCII(host));
+}
+
+export function parsePing(osPlatform: string, stdout: string): PingResult {
+  let parseRegExp1: RegExp, parseRegExp2: RegExp, parsed: PingResult;
+  switch (osPlatform) {
     case "linux":
-      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
+      parseRegExp1 =
-      parseRegExp1 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss[^]*= ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
-      parseRegExp2 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
+      parseRegExp2 =
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss/;
       break;
 
     case "freebsd":
-      // Send a single packet because on FreeBSD only superuser can send
+      parseRegExp1 =
-      // packets that are only 200 ms apart.
+        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
-      cmd = `ping -t 1 -c 3 ${host}`;
+      parseRegExp2 =
-      parseRegExp1 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
+        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
-      parseRegExp2 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
       break;
-
-    default:
-      return callback(new Error("Platform not supported"));
   }
 
-  exec(cmd, (err, stdout) => {
-    let parsed: PingResult;
-    if (stdout) {
   const m1 = stdout.match(parseRegExp1);
   if (m1) {
     parsed = {
@@ -82,8 +64,35 @@ export function ping(
       };
     }
   }
+  return parsed;
+}
+
+export function ping(
+  host: string,
+  callback: (err: Error, res?: PingResult, stdout?: string) => void,
+): void {
+  // Validate input to prevent possible remote code execution
+  // Credit to Alex Hordijk for reporting this vulnerability
+  if (!isValidHost(host)) return callback(new Error("Invalid host"));
+  host = host.replace("[", "").replace("]", "");
+  let cmd: string;
+
+  switch (platform()) {
+    case "linux":
+      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
+      break;
+    case "freebsd":
+      // Send a single packet because on FreeBSD only superuser can send
+      // packets that are only 200 ms apart.
+      cmd = `ping -t 1 -c 3 ${host}`;
+      break;
+    default:
+      return callback(new Error("Platform not supported"));
   }
 
-    callback(err, parsed, stdout);
+  exec(cmd, (err, stdout) => {
+    if (err) return callback(err);
+    const parsed: PingResult = parsePing(platform(), stdout);
+    return callback(err, parsed, stdout);
   });
 }

lib/query.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 function isObject(obj: any): boolean {
   return Object.prototype.toString.call(obj) === "[object Object]";
 }
@@ -54,6 +35,17 @@ function normalize(input): any {
   return input;
 }
 
+const EXPAND_OPS = new Set([
+  "$eq",
+  "$gt",
+  "$gte",
+  "$in",
+  "$lt",
+  "$lte",
+  "$ne",
+  "$nin",
+]);
+
 function expandValue(value: unknown): unknown[] {
   if (Array.isArray(value)) {
     let a = [];
@@ -71,7 +63,8 @@ function expandValue(value: unknown): unknown[] {
   const values = [];
   for (const [k, v] of Object.entries(value)) {
     keys.push(k);
-    values.push(expandValue(v));
+    if (EXPAND_OPS.has(k)) values.push(expandValue(v));
+    else values.push([v]);
     indices.push(0);
   }
 
@@ -106,7 +99,7 @@ function permute(param, val): any[] {
 }
 
 export function expand(
-  query: Record<string, unknown>
+  query: Record<string, unknown>,
 ): Record<string, unknown> {
   const newQuery = {};
   for (const [k, v] of Object.entries(query)) {
@@ -120,7 +113,7 @@ export function expand(
         if (v && (v["$ne"] != null || v["$not"] != null)) {
           if (Object.keys(v).length > 1)
             throw new Error("Cannot mix $ne or $not with other operators");
-          for (const c of conditions) newQuery["$an"].push(c);
+          for (const c of conditions) newQuery["$and"].push(c);
         } else {
           newQuery["$and"].push({ $or: conditions });
         }
@@ -135,7 +128,7 @@ export function expand(
 
 export function sanitizeQueryTypes(
   query: Record<string, unknown>,
-  types: Record<string, (v: unknown) => unknown>
+  types: Record<string, (v: unknown) => unknown>,
 ): Record<string, unknown> {
   for (const [k, v] of Object.entries(query)) {
     if (k[0] === "$") {

lib/sandbox.ts

@@ -1,30 +1,11 @@
-/**
+import * as vm from "node:vm";
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as vm from "vm";
 import seedrandom from "seedrandom";
-import * as device from "./device";
+import * as device from "./device.ts";
-import * as extensions from "./extensions";
+import * as extensions from "./extensions.ts";
-import * as logger from "./logger";
+import * as logger from "./logger.ts";
-import * as scheduling from "./scheduling";
+import * as scheduling from "./scheduling.ts";
-import Path from "./common/path";
+import Path from "./common/path.ts";
-import { Fault, SessionContext, ScriptResult } from "./types";
+import { Fault, SessionContext, ScriptResult } from "./types.ts";
 
 // Used for throwing to exit user script and commit
 const COMMIT = Symbol();
@@ -34,7 +15,7 @@ const EXT = Symbol();
 
 const UNDEFINED = undefined;
 
-const context = vm.createContext();
+const context = vm.createContext(undefined, { microtaskMode: "afterEvaluate" });
 
 let state;
 
@@ -50,7 +31,7 @@ function runExtension(sessionContext, key, extCall): Promise<Fault> {
   }
 
   let prom = re.get(key);
-  if (!prom) {
+  if (prom == null) {
     re.set(
       key,
       (prom = new Promise((resolve, reject) => {
@@ -62,7 +43,7 @@ function runExtension(sessionContext, key, extCall): Promise<Fault> {
             resolve(fault);
           })
           .catch(reject);
-      }))
+      })),
     );
   }
 
@@ -78,7 +59,7 @@ class SandboxDate {
       number?,
       number?,
       number?,
-      number?
+      number?,
     ]
   ) {
     if (argumentList.length) return new Date(...argumentList);
@@ -143,7 +124,7 @@ class ParameterWrapper {
             unpacked = device.unpack(
               state.sessionContext.deviceData,
               path,
-              state.revision
+              state.revision,
             );
           }
 
@@ -151,7 +132,7 @@ class ParameterWrapper {
 
           const attr = state.sessionContext.deviceData.attributes.get(
             unpacked[0],
-            state.revision
+            state.revision,
           )[attrName];
 
           if (!attr) return UNDEFINED;
@@ -170,7 +151,7 @@ class ParameterWrapper {
           unpacked = device.unpack(
             state.sessionContext.deviceData,
             path,
-            state.revision
+            state.revision,
           );
         }
 
@@ -189,7 +170,7 @@ class ParameterWrapper {
           unpacked = device.unpack(
             state.sessionContext.deviceData,
             path,
-            state.revision
+            state.revision,
           );
         }
 
@@ -207,7 +188,7 @@ class ParameterWrapper {
         unpacked = device.unpack(
           state.sessionContext.deviceData,
           path,
-          state.revision
+          state.revision,
         );
       }
 
@@ -220,7 +201,7 @@ class ParameterWrapper {
 function declare(
   path: string,
   timestamps: { [attr: string]: number },
-  values: { [attr: string]: any }
+  values: { [attr: string]: any },
 ): ParameterWrapper {
   state.uncommitted = true;
   if (!timestamps) timestamps = {};
@@ -255,6 +236,7 @@ function declare(
   }
 
   for (const [attrName, attrTimestamp] of Object.entries(timestamps)) {
+    if (!(attrTimestamp >= 1)) continue;
     if (attrName === "path") {
       declaration.pathGet = attrTimestamp;
     } else {
@@ -285,7 +267,7 @@ function commit(): void {
     throw COMMIT;
   } else if (state.revision > state.maxRevision + 1) {
     throw new Error(
-      "Declare function should not be called from within a try/catch block"
+      "Declare function should not be called from within a try/catch block",
     );
   }
 }
@@ -350,12 +332,12 @@ function errorToFault(err: Error): Fault {
     fault.detail["stack"] = err.stack;
     // Trim the stack trace at the self-executing anonymous wrapper function
     const stackTrimIndex = fault.detail["stack"].match(
-      /\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/
+      /\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/,
     );
     if (stackTrimIndex) {
       fault.detail["stack"] = fault.detail["stack"].slice(
         0,
-        stackTrimIndex.index
+        stackTrimIndex.index,
       );
     }
   }
@@ -369,7 +351,7 @@ export async function run(
   sessionContext: SessionContext,
   startRevision: number,
   maxRevision: number,
-  extCounter = 0
+  extCounter = 0,
 ): Promise<ScriptResult> {
   state = {
     sessionContext: sessionContext,
@@ -390,7 +372,7 @@ export async function run(
   let ret, status;
 
   try {
-    ret = script.runInContext(context, { displayErrors: false });
+    ret = script.runInContext(context, { displayErrors: false, timeout: 50 });
     status = 0;
   } catch (err) {
     if (err === COMMIT) {
@@ -414,7 +396,7 @@ export async function run(
   await Promise.all(
     Object.entries(_state.extensions).map(async ([k, v]) => {
       fault = (await runExtension(_state.sessionContext, k, v)) || fault;
-    })
+    }),
   );
 
   if (fault) {
@@ -434,7 +416,7 @@ export async function run(
       sessionContext,
       startRevision,
       maxRevision,
-      extCounter - _state.extCounter
+      extCounter - _state.extCounter,
     );
   }
 

lib/scheduling.ts

@@ -1,24 +1,5 @@
-/**
+import * as crypto from "node:crypto";
- * Copyright 2013-2019  GenieACS Inc.
+import * as later from "@breejs/later";
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as crypto from "crypto";
-import * as later from "later";
 
 function md532(str): number {
   const digest = crypto.createHash("md5").update(str).digest();
@@ -37,7 +18,7 @@ export function variance(deviceId: string, vrnc: number): number {
 export function interval(
   timestamp: number,
   intrvl: number,
-  offset = 0
+  offset = 0,
 ): number {
   return Math.trunc((timestamp + offset) / intrvl) * intrvl - offset;
 }
@@ -52,7 +33,7 @@ export function parseCron(cronExp: string): any {
 export function cron(
   timestamp: number,
   schedule: unknown,
-  offset = 0
+  offset = 0,
 ): number[] {
   // TODO later.js doesn't throw erorr if expression is invalid!
   const ret = [0, 0];

lib/server.ts

@@ -1,31 +1,13 @@
-/**
+import { readFileSync } from "node:fs";
- * Copyright 2013-2019  GenieACS Inc.
+import * as http from "node:http";
- *
+import * as https from "node:https";
- * This file is part of GenieACS.
+import { Socket } from "node:net";
- *
+import * as path from "node:path";
- * GenieACS is free software: you can redistribute it and/or modify
+import { ROOT_DIR } from "./config.ts";
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as fs from "fs";
-import * as http from "http";
-import * as https from "https";
-import { Socket } from "net";
-import * as path from "path";
-import { ROOT_DIR } from "./config";
 
 let server: http.Server | https.Server;
-let listener: (...args) => void;
+let listener: http.RequestListener;
+let stopping = false;
 
 function closeServer(timeout, callback): void {
   if (!server) return void callback();
@@ -58,23 +40,70 @@ interface ServerOptions {
   ssl?: { key: string; cert: string };
   timeout?: number;
   keepAliveTimeout?: number;
+  requestTimeout?: number;
   onConnection?: (socket: Socket) => void;
+  onClientError?: (err: Error, socket: Socket) => void;
+}
+
+interface SocketEndpoint {
+  localAddress: string;
+  localPort: number;
+  remoteAddress: string;
+  remotePort: number;
+  remoteFamily: "IPv4" | "IPv6";
+}
+
+// Save this info as they're not accessible after a socket has been closed
+const socketEndpoints: WeakMap<Socket, SocketEndpoint> = new WeakMap();
+
+type Promisify<T extends (...args: any) => any> = (
+  ...args: Parameters<T>
+) => Promise<ReturnType<T>>;
+
+function getValidPrivKeys(value: string): Buffer[] {
+  return value.split(":").map((str) => {
+    str = str.trim();
+    const buf = str.startsWith("-----BEGIN ")
+      ? Buffer.from(str)
+      : readFileSync(path.resolve(ROOT_DIR, str));
+    return buf;
+  });
+}
+
+function getValidCerts(value: string): Buffer[] {
+  return value.split(":").map((str) => {
+    str = str.trim();
+    const buf = str.startsWith("-----BEGIN ")
+      ? Buffer.from(str)
+      : readFileSync(path.resolve(ROOT_DIR, str));
+    return buf;
+  });
 }
 
 export function start(
   options: ServerOptions,
-  _listener: http.RequestListener
+  _listener: Promisify<http.RequestListener>,
 ): void {
-  listener = _listener;
+  listener = (req, res) => {
+    if (stopping) res.setHeader("Connection", "close");
+    _listener(req, res).catch((err) => {
+      try {
+        res.socket.unref();
+        if (res.headersSent) {
+          res.writeHead(500, { Connection: "close" });
+          res.end(`${err.name}: ${err.message}`);
+        }
+      } catch (err) {
+        // Ignore
+      }
+      throw err;
+    });
+  };
 
   if (options.ssl) {
     const opts = {
-      key: options.ssl.key
+      key: getValidPrivKeys(options.ssl.key),
-        .split(":")
+      cert: getValidCerts(options.ssl.cert),
-        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
-      cert: options.ssl.cert
-        .split(":")
-        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
     };
 
     server = https.createServer(opts, listener);
@@ -85,13 +114,38 @@ export function start(
     if (options.onConnection) server.on("connection", options.onConnection);
   }
 
+  server.on("connection", (socket: Socket) => {
+    socketEndpoints.set(socket, {
+      localAddress: socket.localAddress,
+      localPort: socket.localPort,
+      remoteAddress: socket.remoteAddress,
+      remotePort: socket.remotePort,
+      remoteFamily: socket.remoteFamily as "IPv4" | "IPv6",
+    });
+  });
+
+  if (options.onClientError) {
+    server.on("clientError", (err, socket: Socket) => {
+      if (err["code"] !== "ECONNRESET" && socket.writable)
+        socket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n");
+
+      // As per Node docs: This event is guaranteed to be passed an instance
+      // of the <net.Socket> class
+      options.onClientError(err, socket as Socket);
+    });
+  }
+
   server.timeout = options.timeout || 0;
   if (options.keepAliveTimeout != null)
     server.keepAliveTimeout = options.keepAliveTimeout;
+  if (options.requestTimeout != null)
+    server.requestTimeout = options.requestTimeout;
+
   server.listen({ port: options.port, host: options.host });
 }
 
-export function stop(): Promise<void> {
+export function stop(terminateConnections = true): Promise<void> {
+  stopping = terminateConnections;
   return new Promise((resolve, reject) => {
     setTimeout(() => {
       reject(new Error("Could not close server in a timely manner"));
@@ -99,3 +153,8 @@ export function stop(): Promise<void> {
     closeServer(20000, resolve);
   });
 }
+
+export function getSocketEndpoints(socket: Socket): SocketEndpoint {
+  // TLSSocket keeps a reference to the raw TCP socket in _parent
+  return socketEndpoints.get(socket["_parent"] ?? socket);
+}

lib/soap.ts

@@ -1,30 +1,11 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 import {
   parseXml,
   Element,
   parseAttrs,
   encodeEntities,
   decodeEntities,
-} from "./xml-parser";
+} from "./xml-parser.ts";
-import memoize from "./common/memoize";
+import memoize from "./common/memoize.ts";
 import { version as VERSION } from "../package.json";
 import {
   InformRequest,
@@ -34,22 +15,22 @@ import {
   SoapMessage,
   TransferCompleteRequest,
   AcsRequest,
-  GetParameterNamesResponse,
+  type GetParameterNamesResponse,
-  GetParameterValuesResponse,
+  type GetParameterValuesResponse,
-  GetParameterAttributesResponse,
+  type GetParameterAttributesResponse,
-  SetParameterValuesResponse,
+  type SetParameterValuesResponse,
-  SetParameterAttributesResponse,
+  type SetParameterAttributesResponse,
-  AddObjectResponse,
+  type AddObjectResponse,
-  DeleteObject,
+  type DeleteObject,
-  DeleteObjectResponse,
+  type DeleteObjectResponse,
-  RebootResponse,
+  type RebootResponse,
-  FactoryResetResponse,
+  type FactoryResetResponse,
-  DownloadResponse,
+  type DownloadResponse,
   GetRPCMethodsRequest,
   RequestDownloadRequest,
   AcsResponse,
-} from "./types";
+} from "./types.ts";
-import Path from "./common/path";
+import Path from "./common/path.ts";
 
 const SERVER_NAME = `GenieACS/${VERSION}`;
 
@@ -109,8 +90,8 @@ function event(xml: Element): string[] {
 
 function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterInfoStruct")
     .map<[Path, boolean, boolean]>((e) => {
+      if (e.localName !== "ParameterInfoStruct") return null;
       let param: string, value: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -134,10 +115,20 @@ function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
         parsed = false;
       }
 
+      try {
         if (param && !param.endsWith("."))
           return [Path.parse(param), false, parsed];
         else return [Path.parse(param.slice(0, -1)), true, parsed];
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
         });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 const getValueType = memoize((str: string) => {
@@ -148,11 +139,11 @@ const getValueType = memoize((str: string) => {
 });
 
 function parameterValueList(
-  xml: Element
+  xml: Element,
 ): [Path, string | number | boolean, string][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterValueStruct")
     .map<[Path, string | number | boolean, string]>((e) => {
+      if (e.localName !== "ParameterValueStruct") return null;
       let valueElement: Element, param: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -208,15 +199,24 @@ function parameterValueList(
           parsed = value;
         }
       }
-
+      try {
         return [Path.parse(param), parsed, valueType];
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
         });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 function parameterAttributeList(xml: Element): [Path, number, string[]][] {
   return xml.children
-    .filter((e) => e.localName === "ParameterAttributeStruct")
     .map<[Path, number, string[]]>((e) => {
+      if (e.localName !== "ParameterAttributeStruct") return null;
       let notificationElement: Element,
         accessListElement: Element,
         param: string;
@@ -248,8 +248,18 @@ function parameterAttributeList(xml: Element): [Path, number, string[]][] {
         .filter((c) => c.localName === "string")
         .map((c) => decodeEntities(c.text));
 
+      try {
         return [Path.parse(param), notification, accessList];
+      } catch (err) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
         });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 function GetParameterNames(methodRequest): string {
@@ -262,7 +272,7 @@ function GetParameterNamesResponse(xml): GetParameterNamesResponse {
   return {
     name: "GetParameterNamesResponse",
     parameterList: parameterInfoList(
-      xml.children.find((n) => n.localName === "ParameterList")
+      xml.children.find((n) => n.localName === "ParameterList"),
     ),
   };
 }
@@ -279,7 +289,7 @@ function GetParameterValuesResponse(xml: Element): GetParameterValuesResponse {
   return {
     name: "GetParameterValuesResponse",
     parameterList: parameterValueList(
-      xml.children.find((n) => n.localName === "ParameterList")
+      xml.children.find((n) => n.localName === "ParameterList"),
     ),
   };
 }
@@ -293,12 +303,12 @@ function GetParameterAttributes(methodRequest): string {
 }
 
 function GetParameterAttributesResponse(
-  xml: Element
+  xml: Element,
 ): GetParameterAttributesResponse {
   return {
     name: "GetParameterAttributesResponse",
     parameterList: parameterAttributeList(
-      xml.children.find((n) => n.localName === "ParameterList")
+      xml.children.find((n) => n.localName === "ParameterList"),
     ),
   };
 }
@@ -480,19 +490,19 @@ function Download(methodRequest): string {
   }</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
     methodRequest.url
   }</URL><Username>${encodeEntities(
-    methodRequest.username || ""
+    methodRequest.username || "",
   )}</Username><Password>${encodeEntities(
-    methodRequest.password || ""
+    methodRequest.password || "",
   )}</Password><FileSize>${
     methodRequest.fileSize || "0"
   }</FileSize><TargetFileName>${encodeEntities(
-    methodRequest.targetFileName || ""
+    methodRequest.targetFileName || "",
   )}</TargetFileName><DelaySeconds>${
     methodRequest.delaySeconds || "0"
   }</DelaySeconds><SuccessURL>${encodeEntities(
-    methodRequest.successUrl || ""
+    methodRequest.successUrl || "",
   )}</SuccessURL><FailureURL>${encodeEntities(
-    methodRequest.failureUrl || ""
+    methodRequest.failureUrl || "",
   )}</FailureURL></cwmp:Download>`;
 }
 
@@ -701,6 +711,18 @@ function RequestDownloadResponse(): string {
   return "<cwmp:RequestDownloadResponse></cwmp:RequestDownloadResponse>";
 }
 
+function AcsFault(f: CpeFault): string {
+  return `<soap-env:Body:Fault><faultcode>${encodeEntities(
+    f.faultCode,
+  )}</faultcode><faultstring>${encodeEntities(
+    f.faultString,
+  )}</faultstring><detail><cwmp:Fault><FaultCode>${encodeEntities(
+    f.detail.faultCode,
+  )}</FaultCode><FaultString>${encodeEntities(
+    f.detail.faultString,
+  )}</FaultString></cwmp:Fault></detail></soap-env:Body:Fault>`;
+}
+
 function faultStruct(xml: Element): FaultStruct {
   let faultCode: string,
     faultString: string,
@@ -798,18 +820,18 @@ function fault(xml: Element): CpeFault {
 
 export function request(
   body: string,
-  cwmpVersion: string,
+  warn: Record<string, unknown>[],
-  warn: Record<string, unknown>[]
 ): SoapMessage {
   warnings = warn;
 
   const rpc = {
     id: null,
-    cwmpVersion: cwmpVersion,
+    cwmpVersion: null,
     sessionTimeout: null,
     cpeRequest: null,
     cpeFault: null,
     cpeResponse: null,
+    unknownMethod: null,
   };
 
   if (!body.length) return rpc;
@@ -848,7 +870,7 @@ export function request(
 
   const methodElement = bodyElement.children[0];
 
-  if (!rpc.cwmpVersion && methodElement.localName !== "Fault") {
+  if (methodElement.localName === "Inform") {
     let namespace, namespaceHref;
     for (const e of [methodElement, bodyElement, envelope]) {
       namespace = namespace || e.namespace;
@@ -856,7 +878,7 @@ export function request(
         const attrs = memoizedParseAttrs(e.attrs);
         const attr = namespace
           ? attrs.find(
-              (s) => s.namespace === "xmlns" && s.localName === namespace
+              (s) => s.namespace === "xmlns" && s.localName === namespace,
             )
           : attrs.find((s) => s.name === "xmlns");
 
@@ -931,7 +953,8 @@ export function request(
       rpc.cpeFault = fault(methodElement);
       break;
     default:
-      throw new Error(`8000 Method not supported ${methodElement.localName}`);
+      rpc.unknownMethod = methodElement.localName;
+      break;
   }
 
   return rpc;
@@ -959,6 +982,7 @@ export function response(rpc: {
   id: string;
   acsRequest?: AcsRequest;
   acsResponse?: AcsResponse;
+  acsFault?: CpeFault;
   cwmpVersion?: string;
 }): { code: number; headers: Record<string, string>; data: string } {
   const headers = {
@@ -987,7 +1011,7 @@ export function response(rpc: {
         throw new Error(
           `Unknown method response type ${
             (rpc.acsResponse as AcsResponse).name
-          }`
+          }`,
         );
     }
   } else if (rpc.acsRequest) {
@@ -1024,9 +1048,11 @@ export function response(rpc: {
         break;
       default:
         throw new Error(
-          `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`
+          `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`,
         );
     }
+  } else if (rpc.acsFault) {
+    body = AcsFault(rpc.acsFault);
   }
 
   headers["Content-Type"] = 'text/xml; charset="utf-8"';

lib/types.ts

@@ -1,28 +1,9 @@
-/**
+import { IncomingMessage, ServerResponse } from "node:http";
- * Copyright 2013-2019  GenieACS Inc.
+import { Script } from "node:vm";
- *
+import Path from "./common/path.ts";
- * This file is part of GenieACS.
+import PathSet from "./common/path-set.ts";
- *
+import VersionedMap from "./versioned-map.ts";
- * GenieACS is free software: you can redistribute it and/or modify
+import InstanceSet from "./instance-set.ts";
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
-import PathSet from "./common/path-set";
-import VersionedMap from "./versioned-map";
-import InstanceSet from "./instance-set";
-import { IncomingMessage, ServerResponse } from "http";
-import { Script } from "vm";
 
 export type Expression = string | number | boolean | null | any[];
 
@@ -97,7 +78,7 @@ export type VirtualParameterDeclaration = [
     value?: [string | number | boolean, string?];
     notification?: number;
     accessList?: string[];
-  }?
+  }?,
 ];
 
 export interface SyncState {
@@ -139,7 +120,7 @@ export interface SessionContext {
     AttributeTimestamps,
     AttributeValues,
     AttributeTimestamps,
-    AttributeValues
+    AttributeValues,
   ][][];
   revisions: number[];
   rpcCount: number;
@@ -160,7 +141,6 @@ export interface SessionContext {
   authState: number;
   tasks?: Task[];
   operations?: { [commandKey: string]: Operation };
-  cacheUntil?: number;
   syncState?: SyncState;
   lastActivity?: number;
   extendLock?: number;
@@ -180,7 +160,7 @@ export interface Task {
   fileName?: string;
   targetFileName?: string;
   expiry?: number;
-  provisions?: (string | number | boolean)[][];
+  provisions?: [string, ...Expression[]][];
 }
 
 export interface Operation {
@@ -456,7 +436,7 @@ export type Clear = [
     notification?: number;
     accessList?: number;
   }?,
-  number?
+  number?,
 ];
 
 export interface Preset {
@@ -477,7 +457,7 @@ export interface VirtualParameters {
 }
 
 export interface Files {
-  [name: string]: { length: number; md5: string; contentType: string };
+  [name: string]: { length: number };
 }
 
 export interface Users {
@@ -526,6 +506,7 @@ export interface SoapMessage {
   cpeRequest?: CpeRequest;
   cpeFault?: CpeFault;
   cpeResponse?: CpeResponse;
+  unknownMethod?: string;
 }
 
 export interface ScriptResult {

lib/ui.ts

@@ -1,46 +1,22 @@
-/**
+import { constants } from "node:zlib";
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { constants } from "zlib";
 import Koa from "koa";
-import Router from "koa-router";
+import Router from "@koa/router";
 import * as jwt from "jsonwebtoken";
-import koaStatic from "koa-static";
+import koaSend from "koa-send";
 import koaCompress from "koa-compress";
-import koaBodyParser from "koa-bodyparser";
+import koaBodyParser from "@koa/bodyparser";
 import koaJwt from "koa-jwt";
-import * as config from "./config";
+import * as config from "./config.ts";
-import api from "./ui/api";
+import api from "./ui/api.ts";
-import Authorizer from "./common/authorizer";
+import Authorizer from "./common/authorizer.ts";
-import * as logger from "./logger";
+import * as logger from "./logger.ts";
-import * as localCache from "./local-cache";
+import * as localCache from "./ui/local-cache.ts";
-import { PermissionSet } from "./types";
+import { PermissionSet } from "./types.ts";
-import { authLocal } from "./ui/api-functions";
+import { authLocal } from "./api-functions.ts";
-import * as init from "./init";
+import * as init from "./init.ts";
 import { version as VERSION } from "../package.json";
-import memoize from "./common/memoize";
+import memoize from "./common/memoize.ts";
-
+import { APP_JS, APP_CSS, FAVICON_PNG } from "../build/assets.ts";
-declare module "koa" {
-  interface Request {
-    body: any;
-  }
-}
 
 const koa = new Koa();
 const router = new Router();
@@ -53,20 +29,26 @@ const getAuthorizer = memoize(
     const roles: string[] = JSON.parse(rolesStr);
     const allPermissions = localCache.getPermissions(snapshot);
     const permissionSets: PermissionSet[] = roles.map((r) =>
-      Object.values(allPermissions[r] || {})
+      Object.values(allPermissions[r] || {}),
     );
     return new Authorizer(permissionSets);
-  }
+  },
 );
 
-koa.on("error", async (err) => {
+koa.on("error", (err, ctx) => {
   setTimeout(() => {
+    // Ignored errors resulting from aborted requests
+    if (ctx?.req.aborted) return;
+
+    // Ignore client errors (e.g. malicious path)
+    if (err.status === 400) return;
+
     throw err;
   });
 });
 
 koa.use(async (ctx, next) => {
-  const configSnapshot = await localCache.getCurrentSnapshot();
+  const configSnapshot = await localCache.getRevision();
   ctx.state.configSnapshot = configSnapshot;
   ctx.set("X-Config-Snapshot", configSnapshot);
   ctx.set("GenieACS-Version", VERSION);
@@ -87,13 +69,13 @@ koa.use(
 
       return true;
     },
-  })
+  }),
 );
 
 koa.use(async (ctx, next) => {
   let roles: string[] = [];
 
-  if (ctx.state.user && ctx.state.user.username) {
+  if (ctx.state.user?.username) {
     let user;
     if (ctx.state.user.authMethod === "local") {
       user = localCache.getUsers(ctx.state.configSnapshot)[
@@ -107,7 +89,7 @@ koa.use(async (ctx, next) => {
 
   ctx.state.authorizer = getAuthorizer(
     ctx.state.configSnapshot,
-    JSON.stringify(roles)
+    JSON.stringify(roles),
   );
 
   return next();
@@ -219,18 +201,23 @@ router.post("/init", async (ctx) => {
 });
 
 router.get("/", async (ctx) => {
-  const permissionSets: PermissionSet[] = ctx.state.authorizer.getPermissionSets();
+  // koa-router seems to tolerate double slashes in the URL but that can
+  // be problematic when using relatives asset paths in HTML
+  if (ctx.path.endsWith("//")) return;
+
+  const permissionSets: PermissionSet[] =
+    ctx.state.authorizer.getPermissionSets();
 
   let wizard = "";
   if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
     wizard = '<script>window.location.hash = "#!/wizard";</script>';
 
-  ctx.body = `
+  ctx.body = `<!DOCTYPE html>
   <html>
     <head>
       <title>GenieACS</title>
-      <link rel="shortcut icon" type="image/png" href="favicon.png" />
+      <link rel="shortcut icon" type="image/png" href="${FAVICON_PNG}" />
-      <link rel="stylesheet" href="app.css">
+      <link rel="stylesheet" href="${APP_CSS}">
     </head>
     <body>
     <noscript>GenieACS UI requires JavaScript to work. Please enable JavaScript in your browser.</noscript>
@@ -241,11 +228,11 @@ router.get("/", async (ctx) => {
         window.configSnapshot = ${JSON.stringify(ctx.state.configSnapshot)};
         window.genieacsVersion = ${JSON.stringify(VERSION)};
         window.username = ${JSON.stringify(
-          ctx.state.user ? ctx.state.user.username : ""
+          ctx.state.user ? ctx.state.user.username : "",
         )};
         window.permissionSets = ${JSON.stringify(permissionSets)};
       </script>
-      <script src="app.js"></script>${wizard} 
+      <script type="module" src="${APP_JS}"></script>${wizard} 
     </body>
   </html>
   `;
@@ -259,10 +246,27 @@ koa.use(
     deflate: {
       flush: constants.Z_SYNC_FLUSH,
     },
-  })
+    br: {
+      flush: constants.BROTLI_OPERATION_FLUSH,
+      params: {
+        [constants.BROTLI_PARAM_QUALITY]: 5,
+      },
+    },
+  }),
 );
 
 koa.use(router.routes());
-koa.use(koaStatic(config.ROOT_DIR + "/public"));
+koa.use(async (ctx, next) => {
+  await next();
+  if (ctx.method !== "HEAD" && ctx.method !== "GET") return;
+  if (ctx.body != null || ctx.status !== 404) return;
+  if (/(?:^|[\\/])\.\.(?:[\\/]|$)/.test(ctx.path)) return;
+
+  try {
+    await koaSend(ctx, ctx.path, { root: config.ROOT_DIR + "/public" });
+  } catch (err) {
+    if (err.status !== 404) throw err;
+  }
+});
 
 export const listener = koa.callback();

lib/ui/api-functions.ts

@@ -1,211 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ObjectID } from "mongodb";
-import * as db from "./db";
-import { del } from "../cache";
-import { getUsers } from "../local-cache";
-import { hashPassword } from "../auth";
-import {
-  insertTasks,
-  awaitSessionEnd,
-  awaitSessionStart,
-  connectionRequest,
-  deleteDevice,
-} from "../api-functions";
-import { Task } from "../types";
-
-async function deleteFault(id): Promise<void> {
-  const deviceId = id.split(":", 1)[0];
-  const channel = id.slice(deviceId.length + 1);
-
-  await Promise.all([
-    db.deleteFault(id),
-    channel.startsWith("task_")
-      ? db.deleteTask(new ObjectID(channel.slice(5)))
-      : null,
-  ]);
-
-  await del(`${deviceId}_tasks_faults_operations`);
-}
-
-export async function deleteResource(
-  resource: string,
-  id: string
-): Promise<void> {
-  switch (resource) {
-    case "devices":
-      await deleteDevice(id);
-      break;
-
-    case "files":
-      await db.deleteFile(id);
-      break;
-
-    case "faults":
-      await deleteFault(id);
-      break;
-
-    case "provisions":
-      await db.deleteProvision(id);
-      break;
-
-    case "presets":
-      await db.deletePreset(id);
-      break;
-
-    case "virtualParameters":
-      await db.deleteVirtualParameter(id);
-      break;
-
-    case "config":
-      await db.deleteConfig(id);
-      break;
-
-    case "permissions":
-      await db.deletePermission(id);
-      break;
-
-    case "users":
-      await db.deleteUser(id);
-      break;
-  }
-
-  await del("presets_hash");
-}
-
-export async function postTasks(
-  deviceId: string,
-  tasks: Task[],
-  timeout: number,
-  device: Record<string, { value?: [boolean | number | string, string] }>
-): Promise<{ connectionRequest: string; tasks: any[] }> {
-  for (const task of tasks) {
-    delete task._id;
-    task["device"] = deviceId;
-  }
-
-  tasks = await insertTasks(tasks);
-  const statuses = tasks.map((t) => {
-    return { _id: t._id, status: "pending" };
-  });
-  const lastInform = Date.now();
-  const notInSession = await awaitSessionEnd(deviceId, 30000);
-  if (!notInSession) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  await del(`${deviceId}_tasks_faults_operations`);
-
-  const status = await connectionRequest(deviceId, device);
-
-  if (status) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    return {
-      connectionRequest: status,
-      tasks: statuses,
-    };
-  }
-
-  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
-  if (!sessionStarted) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    return {
-      connectionRequest: "No contact from CPE",
-      tasks: statuses,
-    };
-  }
-
-  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
-  if (!sessionEnded) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  const promises = [];
-  for (const s of statuses) {
-    promises.push(db.query("tasks", ["=", ["PARAM", "_id"], s._id]));
-    promises.push(
-      db.query("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
-    );
-  }
-
-  const res = await Promise.all(promises);
-  for (const [i, r] of statuses.entries()) {
-    if (res[i * 2].length === 0) {
-      r.status = "done";
-    } else if (res[i * 2 + 1].length === 1) {
-      r.status = "fault";
-      r["fault"] = res[i * 2 + 1][0];
-    }
-  }
-
-  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-
-  return { connectionRequest: "OK", tasks: statuses };
-}
-
-export async function putResource(
-  resource: string,
-  id: string,
-  data: Record<string, unknown>
-): Promise<void> {
-  if (resource === "presets") {
-    await db.putPreset(id, data);
-  } else if (resource === "provisions") {
-    await db.putProvision(id, data);
-  } else if (resource === "virtualParameters") {
-    await db.putVirtualParameter(id, data);
-  } else if (resource === "config") {
-    await db.putConfig(id, data);
-  } else if (resource === "permissions") {
-    await db.putPermission(id, data);
-  } else if (resource === "users") {
-    delete data["password"];
-    delete data["salt"];
-    await db.putUser(id, data);
-  }
-
-  await del("presets_hash");
-}
-
-export function authLocal(
-  snapshot: string,
-  username: string,
-  password: string
-): Promise<boolean> {
-  return new Promise((resolve, reject) => {
-    const users = getUsers(snapshot);
-    const user = users[username];
-    if (!user || !user.password) return void resolve(null);
-    hashPassword(password, user.salt)
-      .then((hash) => {
-        if (hash === user.password) resolve(true);
-        else resolve(false);
-      })
-      .catch(reject);
-  });
-}

lib/ui/api.ts

@@ -1,37 +1,22 @@
-/**
+import { Readable } from "node:stream";
- * Copyright 2013-2019  GenieACS Inc.
+import Router from "@koa/router";
- *
+import { ObjectId } from "mongodb";
- * This file is part of GenieACS.
+import * as db from "./db.ts";
- *
+import * as apiFunctions from "../api-functions.ts";
- * GenieACS is free software: you can redistribute it and/or modify
+import { evaluate, and, extractParams } from "../common/expression/util.ts";
- * it under the terms of the GNU Affero General Public License as
+import { parse } from "../common/expression/parser.ts";
- * published by the Free Software Foundation, either version 3 of the
+import * as logger from "../logger.ts";
- * License, or (at your option) any later version.
+import { getConfig } from "../ui/local-cache.ts";
- *
+import { Expression, Task } from "../types.ts";
- * GenieACS is distributed in the hope that it will be useful,
+import { generateSalt, hashPassword } from "../auth.ts";
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
+import { del } from "../cache.ts";
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+import Authorizer from "../common/authorizer.ts";
- * GNU Affero General Public License for more details.
+import { ping } from "../ping.ts";
- *
+import { decodeTag } from "../util.ts";
- * You should have received a copy of the GNU Affero General Public License
+import { stringify as yamlStringify } from "../common/yaml.ts";
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+import { ResourceLockedError } from "../common/errors.ts";
- */
+import { acquireLock, releaseLock } from "../lock.ts";
-
+import { collections } from "../db/db.ts";
-import * as stream from "stream";
-import Router from "koa-router";
-import * as db from "./db";
-import * as apiFunctions from "./api-functions";
-import { evaluate, and, extractParams } from "../common/expression";
-import { parse } from "../common/expression-parser";
-import * as logger from "../logger";
-import { getConfig } from "../local-cache";
-import { QueryOptions, Expression } from "../types";
-import { generateSalt, hashPassword } from "../auth";
-import { del } from "../cache";
-import Authorizer from "../common/authorizer";
-import { ping } from "../ping";
-import * as url from "url";
-import { decodeTag } from "../common";
 
 const router = new Router();
 export default router;
@@ -70,6 +55,10 @@ const resources = {
   tasks: 0,
 };
 
+function singleParam(p: string | string[]): string {
+  return Array.isArray(p) ? p[p.length - 1] : p;
+}
+
 router.get(`/devices/:id.csv`, async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
@@ -89,40 +78,48 @@ router.get(`/devices/:id.csv`, async (ctx) => {
     return void (ctx.status = 403);
   }
 
-  const res = await db.query("devices", filter);
+  const { value: device } = await db.query("devices", filter).next();
-  if (!res[0]) return void (ctx.status = 404);
+  if (!device) return void (ctx.status = 404);
 
   ctx.type = "text/csv";
   ctx.attachment(
     `device-${ctx.params.id}-${new Date()
       .toISOString()
-      .replace(/[:.]/g, "")}.csv`
+      .replace(/[:.]/g, "")}.csv`,
   );
 
-  ctx.body = new stream.PassThrough();
+  const lines: string[] = [
-  ctx.body.write(
+    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp,Notification,Notification timestamp,Access list,Access list timestamp",
-    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp,Notification,Notification timestamp,Access list,Access list timestamp\n"
+  ];
-  );
+
+  for (const k of Object.keys(device).sort()) {
+    const p = device[k];
+    let value = "";
+    let type = "";
+    if (p.value) {
+      value = p.value[0];
+      type = p.value[1];
+      if (type === "xsd:dateTime" && typeof value === "number")
+        value = new Date(value).toJSON();
+    }
 
-  for (const k of Object.keys(res[0]).sort()) {
-    const p = res[0][k];
     const row = [
       k,
       p.object,
-      p.objectTimestamp,
+      new Date(p.objectTimestamp).toJSON(),
       p.writable,
-      p.writableTimestamp,
+      new Date(p.writableTimestamp).toJSON(),
-      p.value != null ? `"${p.value[0].toString().replace(/"/g, '""')}"` : "",
+      `"${String(value).replace(/"/g, '""')}"`,
-      p.value != null ? p.value[1] : "",
+      type,
-      p.valueTimestamp,
+      new Date(p.valueTimestamp).toJSON(),
       p.notification,
-      p.notificationTimestamp,
+      new Date(p.notificationTimestamp).toJSON(),
       p.accessList ? p.accessList.join(", ") : "",
-      p.accessListTimestamp,
+      new Date(p.accessListTimestamp).toJSON(),
     ];
-    ctx.body.write(row.map((r) => (r != null ? r : "")).join(",") + "\n");
+    lines.push(row.map((r) => (r != null ? r : "")).join(","));
   }
-  ctx.body.end();
+  ctx.body = lines.join("\n");
   logger.accessInfo(log);
 });
 
@@ -131,7 +128,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     const authorizer: Authorizer = ctx.state.authorizer;
     let filter: Expression = authorizer.getFilter(resource, 1);
     if (ctx.request.query.filter)
-      filter = and(filter, parse(ctx.request.query.filter));
+      filter = and(filter, parse(singleParam(ctx.request.query.filter)));
 
     const log = {
       message: `Count ${resource}`,
@@ -165,16 +162,16 @@ for (const [resource, flags] of Object.entries(resources)) {
 
   router.get(`/${resource}`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
-    const options: QueryOptions = {};
+    const options: Parameters<typeof db.query>[2] = {};
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
-      filter = and(filter, parse(ctx.request.query.filter));
+      filter = and(filter, parse(singleParam(ctx.request.query.filter)));
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
-      options.sort = JSON.parse(ctx.request.query.sort);
+      options.sort = JSON.parse(singleParam(ctx.request.query.sort));
     if (ctx.request.query.projection) {
-      options.projection = ctx.request.query.projection
+      options.projection = singleParam(ctx.request.query.projection)
         .split(",")
         .reduce((obj, k) => Object.assign(obj, { [k]: 1 }), {});
     }
@@ -203,30 +200,30 @@ for (const [resource, flags] of Object.entries(resources)) {
       ]);
     }
 
-    ctx.body = new stream.PassThrough();
-    ctx.type = "application/json";
-
-    let c = 0;
-    ctx.body.write("[\n");
-    await db.query(resource, filter, options, (obj) => {
-      ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
-    });
-    ctx.body.end("]");
-
     logger.accessInfo(log);
+    ctx.type = "application/json";
+    ctx.body = Readable.from(
+      (async function* () {
+        let c = 0;
+        yield "[\n";
+        for await (const obj of db.query(resource, filter, options))
+          yield (c++ ? "," : "") + JSON.stringify(obj) + "\n";
+        yield "]";
+      })(),
+    );
   });
 
   // CSV download
   router.get(`/${resource}.csv`, async (ctx) => {
     const authorizer: Authorizer = ctx.state.authorizer;
-    const options: QueryOptions = { projection: {} };
+    const options: Parameters<typeof db.query>[2] = { projection: {} };
     let filter: Expression = authorizer.getFilter(resource, 2);
     if (ctx.request.query.filter)
-      filter = and(filter, parse(ctx.request.query.filter));
+      filter = and(filter, parse(singleParam(ctx.request.query.filter)));
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
-      options.sort = JSON.parse(ctx.request.query.sort);
+      options.sort = JSON.parse(singleParam(ctx.request.query.sort));
 
     const log = {
       message: `Query ${resource} (CSV)`,
@@ -243,7 +240,7 @@ for (const [resource, flags] of Object.entries(resources)) {
     }
 
     const columns: Record<string, Expression> = JSON.parse(
-      ctx.request.query.columns
+      singleParam(ctx.request.query.columns),
     );
     const now = Date.now();
 
@@ -263,16 +260,17 @@ for (const [resource, flags] of Object.entries(resources)) {
       ]);
     }
 
-    ctx.body = new stream.PassThrough();
+    logger.accessInfo(log);
     ctx.type = "text/csv";
     ctx.attachment(
-      `${resource}-${new Date(now).toISOString().replace(/[:.]/g, "")}.csv`
+      `${resource}-${new Date(now).toISOString().replace(/[:.]/g, "")}.csv`,
     );
 
-    ctx.body.write(
+    ctx.body = Readable.from(
-      Object.keys(columns).map((k) => `"${k.replace(/"/, '""')}"`) + "\n"
+      (async function* () {
-    );
+        yield Object.keys(columns).map((k) => `"${k.replace(/"/, '""')}"`) +
-    await db.query(resource, filter, options, (obj) => {
+          "\n";
+        for await (const obj of db.query(resource, filter, options)) {
           const arr = Object.values(columns).map((exp) => {
             const v = evaluate(exp, obj, null, (e) => {
               if (Array.isArray(e)) {
@@ -281,15 +279,28 @@ for (const [resource, flags] of Object.entries(resources)) {
                     if (e[1] === "Tags") {
                       const tags = [];
                       for (const p in obj)
-                    if (p.startsWith("Tags.")) tags.push(decodeTag(p.slice(5)));
+                        if (p.startsWith("Tags."))
+                          tags.push(decodeTag(p.slice(5)));
 
                       return tags.join(", ");
                     }
+                    if (e === exp) {
+                      const p = obj[e[1]];
+                      if (
+                        p &&
+                        p.value &&
+                        p.value[1] === "xsd:dateTime" &&
+                        typeof p.value[0] === "number"
+                      )
+                        return new Date(p.value[0]).toJSON();
+                    }
+                  } else if (resource === "faults") {
+                    if (e[1] === "detail") return yamlStringify(obj["detail"]);
                   }
                 } else if (e[0] === "FUNC") {
                   if (e[1] === "DATE_STRING") {
                     if (e[2] && !Array.isArray(e[2]))
-                  return new Date(e[2]).toISOString();
+                      return new Date(e[2]).toJSON();
                   }
                 }
               }
@@ -301,11 +312,10 @@ for (const [resource, flags] of Object.entries(resources)) {
             if (typeof v === "string") return `"${v.replace(/"/g, '""')}"`;
             return v;
           });
-      ctx.body.write(arr.join(",") + "\n");
+          yield arr.join(",") + "\n";
-    });
+        }
-    ctx.body.end();
+      })(),
-
+    );
-    logger.accessInfo(log);
   });
 
   router.head(`/${resource}/:id`, async (ctx) => {
@@ -326,9 +336,9 @@ for (const [resource, flags] of Object.entries(resources)) {
       return void (ctx.status = 403);
     }
 
-    const res = await db.query(resource, filter);
+    const count = await db.count(resource, filter);
 
-    if (!res.length) return void (ctx.status = 404);
+    if (!count) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = "";
@@ -352,12 +362,12 @@ for (const [resource, flags] of Object.entries(resources)) {
       return void (ctx.status = 403);
     }
 
-    const res = await db.query(resource, filter);
+    const { value: res } = await db.query(resource, filter).next();
 
-    if (!res.length) return void (ctx.status = 404);
+    if (!res) return void (ctx.status = 404);
 
     logger.accessInfo(log);
-    ctx.body = res[0];
+    ctx.body = res;
   });
 
   if (flags & RESOURCE_DELETE) {
@@ -378,16 +388,27 @@ for (const [resource, flags] of Object.entries(resources)) {
         logUnauthorizedWarning(log);
         return void (ctx.status = 403);
       }
-      const res = await db.query(resource, filter);
+      const { value: res } = await db.query(resource, filter).next();
-      if (!res.length) return void (ctx.status = 404);
+      if (!res) return void (ctx.status = 404);
 
-      const validate = authorizer.getValidator(resource, res[0]);
+      const validate = authorizer.getValidator(resource, res);
       if (!validate("delete")) {
         logUnauthorizedWarning(log);
         return void (ctx.status = 403);
       }
 
+      try {
         await apiFunctions.deleteResource(resource, ctx.params.id);
+      } catch (err) {
+        if (err instanceof ResourceLockedError) {
+          log.message += " failed";
+          logger.accessWarn(log);
+          ctx.status = 503;
+          ctx.body = err.message;
+          return;
+        }
+        throw err;
+      }
 
       logger.accessInfo(log);
 
@@ -435,6 +456,36 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 }
 
+router.get("/blob/files/:id", async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
+  const resource = "files";
+  const id = ctx.params.id;
+
+  const log = {
+    message: `Download ${resource}`,
+    context: ctx,
+    id: id,
+  };
+
+  const filter = and(authorizer.getFilter(resource, 2), [
+    "=",
+    ["PARAM", RESOURCE_IDS[resource]],
+    ctx.params.id,
+  ]);
+
+  if (!authorizer.hasAccess(resource, 2)) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 403);
+  }
+
+  const count = await db.count(resource, filter);
+  if (!count) return void (ctx.status = 404);
+
+  logger.accessInfo(log);
+  ctx.body = db.downloadFile(id);
+  ctx.attachment(id);
+});
+
 router.put("/files/:id", async (ctx) => {
   const authorizer: Authorizer = ctx.state.authorizer;
   const resource = "files";
@@ -453,10 +504,11 @@ router.put("/files/:id", async (ctx) => {
   }
 
   const metadata = {
-    fileType: ctx.request.headers["metadata.filetype"] || "",
+    fileType: singleParam(ctx.request.headers["metadata-filetype"]) || "",
-    oui: ctx.request.headers["metadata.oui"] || "",
+    oui: singleParam(ctx.request.headers["metadata-oui"]) || "",
-    productClass: ctx.request.headers["metadata.productclass"] || "",
+    productClass:
-    version: ctx.request.headers["metadata.version"] || "",
+      singleParam(ctx.request.headers["metadata-productclass"]) || "",
+    version: singleParam(ctx.request.headers["metadata-version"]) || "",
   };
 
   const validate = authorizer.getValidator(resource, metadata);
@@ -479,26 +531,48 @@ router.put("/files/:id", async (ctx) => {
 });
 
 router.post("/devices/:id/tasks", async (ctx) => {
+  const deviceId = ctx.params.id;
   const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Commit tasks",
     context: ctx,
-    deviceId: ctx.params.id,
+    deviceId: deviceId,
     tasks: null,
   };
 
-  const filter = and(authorizer.getFilter("devices", 3), [
-    "=",
-    ["PARAM", "DeviceID.ID"],
-    ctx.params.id,
-  ]);
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
     return void (ctx.status = 403);
   }
-  const devices = await db.query("devices", filter);
+
-  if (!devices.length) return void (ctx.status = 404);
+  const socketTimeout: number = ctx.socket.timeout;
-  const device = devices[0];
+
+  // Extend socket timeout while waiting for session
+  if (socketTimeout) ctx.socket.setTimeout(300000);
+
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000, 30000);
+  if (!token) {
+    log.message += " failed";
+    logger.accessWarn(log);
+    ctx.body = "Device is in session";
+    ctx.status = 503;
+    // Restore socket timeout
+    if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
+    return;
+  }
+
+  let device;
+
+  let statuses: { _id: string; status: string }[];
+
+  try {
+    const filter = and(authorizer.getFilter("devices", 3), [
+      "=",
+      ["PARAM", "DeviceID.ID"],
+      deviceId,
+    ]);
+    device = (await db.query("devices", filter).next()).value;
+    if (!device) return void (ctx.status = 404);
 
     const validate = authorizer.getValidator("devices", device);
     for (const t of ctx.request.body) {
@@ -508,6 +582,20 @@ router.post("/devices/:id/tasks", async (ctx) => {
       }
     }
 
+    let tasks = ctx.request.body as Task[];
+
+    for (const task of tasks) {
+      delete task._id;
+      task["device"] = deviceId;
+    }
+
+    tasks = await apiFunctions.insertTasks(tasks);
+
+    statuses = tasks.map((t) => ({ _id: t._id, status: "pending" }));
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
+
   const onlineThreshold = getConfig(
     ctx.state.configSnapshot,
     "cwmp.deviceOnlineThreshold",
@@ -517,41 +605,57 @@ router.post("/devices/:id/tasks", async (ctx) => {
       if (!Array.isArray(exp)) return exp;
       if (exp[0] === "PARAM") {
         const p = device[exp[1]];
-        if (p && p.value) return p.value[0];
+        if (p?.value) return p.value[0];
       } else if (exp[0] === "FUNC") {
         if (exp[1] === "REMOTE_ADDRESS") {
           for (const root of ["InternetGatewayDevice", "Device"]) {
             const p = device[`${root}.ManagementServer.ConnectionRequestURL`];
-            if (p && p.value) return url.parse(p.value[0]).host;
+            if (p?.value) return new URL(p.value[0]).hostname;
           }
           return null;
         }
       }
       return exp;
-    }
+    },
   ) as number;
 
-  const socketTimeout: number = ctx.socket["timeout"];
+  const lastInform = device["Events.Inform"].value[0] as number;
 
-  // Disable socket timeout while waiting for postTasks()
+  let status = await apiFunctions.connectionRequest(deviceId, device);
-  if (socketTimeout) ctx.socket.setTimeout(0);
+  if (!status) {
-
+    const sessionStarted = await apiFunctions.awaitSessionStart(
-  const res = await apiFunctions.postTasks(
+      deviceId,
-    ctx.params.id,
+      lastInform,
-    ctx.request.body,
       onlineThreshold,
-    device
     );
+    if (!sessionStarted) {
+      status = "No contact from CPE";
+    } else {
+      const sessionEnded = await apiFunctions.awaitSessionEnd(deviceId, 120000);
+      if (!sessionEnded) status = "Session took too long to complete";
+    }
+  }
+
+  if (!status) {
+    const promises = statuses.map((t) =>
+      collections.faults.count({ _id: `${deviceId}:task_${t._id}` }),
+    );
+
+    const res = await Promise.all(promises);
+    for (const [i, r] of statuses.entries())
+      r.status = res[i] ? "fault" : "done";
+  }
+
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
 
   // Restore socket timeout
   if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
 
-  log.tasks = res.tasks.map((t) => t._id).join(",");
+  log.tasks = statuses.map((t) => t._id).join(",");
-
   logger.accessInfo(log);
 
-  ctx.set("Connection-Request", res.connectionRequest);
+  ctx.set("Connection-Request", status || "OK");
-  ctx.body = res.tasks;
+  ctx.body = statuses;
 });
 
 router.post("/devices/:id/tags", async (ctx) => {
@@ -572,10 +676,10 @@ router.post("/devices/:id/tags", async (ctx) => {
     logUnauthorizedWarning(log);
     return void (ctx.status = 403);
   }
-  const res = await db.query("devices", filter);
+  const { value: res } = await db.query("devices", filter).next();
-  if (!res.length) return void (ctx.status = 404);
+  if (!res) return void (ctx.status = 404);
 
-  const validate = authorizer.getValidator("devices", res[0]);
+  const validate = authorizer.getValidator("devices", res);
   if (!validate("tags", ctx.request.body)) {
     logUnauthorizedWarning(log);
     return void (ctx.status = 403);
@@ -596,13 +700,14 @@ router.post("/devices/:id/tags", async (ctx) => {
 });
 
 router.get("/ping/:host", async (ctx) => {
-  return new Promise((resolve) => {
+  if (!ctx.state.user) return void (ctx.status = 401);
+  return new Promise<void>((resolve) => {
     ping(ctx.params.host, (err, parsed) => {
       if (parsed) {
         ctx.body = parsed;
       } else {
         ctx.status = 500;
-        ctx.body = `${err.name}: ${err.message}`;
+        ctx.body = err ? `${err.name}: ${err.message}` : "Unknown error";
       }
       resolve();
     });
@@ -624,7 +729,7 @@ router.put("/users/:id/password", async (ctx) => {
       !(await apiFunctions.authLocal(
         ctx.state.configSnapshot,
         username,
-        ctx.request.body.authPassword
+        ctx.request.body.authPassword,
       ))
     ) {
       logUnauthorizedWarning(log);
@@ -637,17 +742,17 @@ router.put("/users/:id/password", async (ctx) => {
     return void (ctx.status = 403);
   }
 
+  const newPassword = ctx.request.body.newPassword;
+
+  if (ctx.state.user) {
     const filter = and(authorizer.getFilter("users", 3), [
       "=",
       ["PARAM", RESOURCE_IDS.users],
       username,
     ]);
-  const res = await db.query("users", filter);
+    const { value: res } = await db.query("users", filter).next();
-  if (!res.length) return void (ctx.status = 404);
+    if (!res) return void (ctx.status = 404);
-
+    const validate = authorizer.getValidator("users", res);
-  const newPassword = ctx.request.body.newPassword;
-  if (ctx.state.user) {
-    const validate = authorizer.getValidator("users", res[0]);
     if (!validate("password", { password: newPassword })) {
       logUnauthorizedWarning(log);
       return void (ctx.status = 403);
@@ -658,7 +763,7 @@ router.put("/users/:id/password", async (ctx) => {
   const password = await hashPassword(newPassword, salt);
   await db.putUser(username, { password, salt });
 
-  await del("presets_hash");
+  await del("ui-local-cache-hash");
 
   logger.accessInfo(log);
   ctx.body = "";

lib/ui/db.ts

@@ -1,83 +1,379 @@
-/**
+import { Script } from "node:vm";
- * Copyright 2013-2019  GenieACS Inc.
+import { Readable } from "node:stream";
- *
+import { Collection, ObjectId, WithoutId } from "mongodb";
- * This file is part of GenieACS.
+import { encodeTag } from "../util.ts";
- *
+import { evaluate } from "../common/expression/util.ts";
- * GenieACS is free software: you can redistribute it and/or modify
+import { Expression, Fault, Task } from "../types.ts";
- * it under the terms of the GNU Affero General Public License as
+import { collections, filesBucket } from "../db/db.ts";
- * published by the Free Software Foundation, either version 3 of the
+import { convertOldPrecondition, optimizeProjection } from "../db/util.ts";
- * License, or (at your option) any later version.
+import * as MongoTypes from "../db/types.ts";
- *
+import { parse, parseList, stringify } from "../common/expression/parser.ts";
- * GenieACS is distributed in the hope that it will be useful,
+import { toMongoQuery } from "../db/synth.ts";
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
 
-import { Db, GridFSBucket, ObjectID } from "mongodb";
+function processDeviceProjection(
-import { Script } from "vm";
+  projection: Record<string, 1>,
-import { onConnect } from "../db";
+): Record<string, 1> {
-import * as mongodbFunctions from "../mongodb-functions";
+  if (!projection) return projection;
-import * as expression from "../common/expression";
+  const p = {};
-import { QueryOptions, Expression } from "../types";
+  for (const [k, v] of Object.entries(projection)) {
-import { Readable } from "stream";
+    if (k === "DeviceID.ID") {
-import { minimize } from "../common/boolean-expression";
+      p["_id"] = 1;
+    } else if (k.startsWith("DeviceID")) {
+      p["_deviceId._SerialNumber"] = v;
+      p["_deviceId._OUI"] = v;
+      p["_deviceId._ProductClass"] = v;
+      p["_deviceId._Manufacturer"] = v;
+    } else if (k.startsWith("Tags")) {
+      p["_tags"] = v;
+    } else if (k.startsWith("Events")) {
+      p["_lastInform"] = v;
+      p["_registered"] = v;
+      p["_lastBoot"] = v;
+      p["_lastBootstrap"] = v;
+    } else {
+      p[k] = v;
+    }
+  }
 
-const RESOURCE_COLLECTION = {
+  return p;
-  files: "fs.files",
+}
-};
 
-let db: Db;
+function processDeviceSort(
+  sort: Record<string, number>,
+): Record<string, number> {
+  if (!sort) return sort;
+  const s = {};
+  for (const [k, v] of Object.entries(sort)) {
+    if (k === "DeviceID.ID") s["_id"] = v;
+    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
+    else if (k === "Events.Inform") s["_lastInform"] = v;
+    else if (k === "Events.Registered") s["_registered"] = v;
+    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
+    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
+    else s[`${k}._value`] = v;
+  }
 
-onConnect(async (_db) => {
+  return s;
-  db = _db;
+}
-});
 
-export function query(
+function parseDate(d: Date): number | string {
+  const n = +d;
+  return isNaN(n) ? "" + d : n;
+}
+
+interface FlatAttributes {
+  object?: boolean;
+  objectTimestamp?: number;
+  writable?: boolean;
+  writableTimestamp?: number;
+  value?: [string | number | boolean, string];
+  valueTimestamp?: number;
+  notification?: number;
+  notificationTimestamp?: number;
+  accessList?: string[];
+  accessListTimestamp?: number;
+}
+
+export interface FlatDevice {
+  [param: string]: FlatAttributes;
+}
+
+export function flattenDevice(device: Record<string, unknown>): FlatDevice {
+  function recursive(
+    input,
+    root: string,
+    output: FlatDevice,
+    timestamp: number,
+  ): void {
+    for (const [name, tree] of Object.entries(input)) {
+      if (!root) {
+        if (name === "_lastInform") {
+          output["Events.Inform"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_registered") {
+          output["Events.Registered"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_lastBoot") {
+          output["Events.1_BOOT"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_lastBootstrap") {
+          output["Events.0_BOOTSTRAP"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_id") {
+          output["DeviceID.ID"] = {
+            value: [tree as string, "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_deviceId") {
+          output["DeviceID.Manufacturer"] = {
+            value: [tree["_Manufacturer"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.OUI"] = {
+            value: [tree["_OUI"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.ProductClass"] = {
+            value: [tree["_ProductClass"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.SerialNumber"] = {
+            value: [tree["_SerialNumber"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_tags") {
+          output["Tags"] = {
+            writable: false,
+            writableTimestamp: timestamp,
+            object: true,
+            objectTimestamp: timestamp,
+          };
+
+          for (const t of tree as string[]) {
+            output[`Tags.${encodeTag(t)}`] = {
+              value: [true, "xsd:boolean"],
+              valueTimestamp: timestamp,
+              writable: true,
+              writableTimestamp: timestamp,
+              object: false,
+              objectTimestamp: timestamp,
+            };
+          }
+        }
+      }
+
+      if (name.startsWith("_")) continue;
+
+      let childrenTimestamp = timestamp;
+
+      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
+      else if (+input["_timestamp"] > timestamp)
+        childrenTimestamp = +input["_timestamp"];
+
+      const attrs: FlatAttributes = {};
+      if (tree["_value"] != null) {
+        attrs.value = [
+          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
+          tree["_type"],
+        ];
+        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
+        attrs.object = false;
+        attrs.objectTimestamp = childrenTimestamp;
+      } else if (tree["_object"] != null) {
+        attrs.object = tree["_object"];
+        attrs.objectTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_writable"] != null) {
+        attrs.writable = tree["_writable"];
+        attrs.writableTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_notification"] != null) {
+        attrs.notification = tree["_notification"];
+        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      if (tree["_accessList"] != null) {
+        attrs.accessList = tree["_accessList"];
+        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      const r = root ? `${root}.${name}` : name;
+      output[r] = attrs;
+
+      if (attrs.object || tree["object"] == null)
+        recursive(tree, r, output, childrenTimestamp);
+    }
+  }
+
+  const newDevice: FlatDevice = {};
+  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
+  recursive(device, "", newDevice, timestamp);
+  return newDevice;
+}
+
+function flattenFault(fault: unknown): Fault {
+  const f = Object.assign({}, fault) as Fault;
+  if (f.timestamp) f.timestamp = +f.timestamp;
+  if (f["expiry"]) f["expiry"] = +f["expiry"];
+  return f as Fault;
+}
+
+function flattenTask(task: unknown): Task {
+  const t = Object.assign({}, task) as Task;
+  t._id = "" + t._id;
+  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
+  if (t.expiry) t.expiry = +t.expiry;
+  return t;
+}
+
+function flattenPreset(
+  preset: Record<string, unknown>,
+): Record<string, unknown> {
+  const p = Object.assign({}, preset);
+  if (p.precondition) {
+    try {
+      // Try parse to check expression validity
+      parse(p.precondition as string);
+    } catch (error) {
+      p.precondition = convertOldPrecondition(
+        JSON.parse(p.precondition as string),
+      );
+      p.precondition = (p.precondition as string).length
+        ? stringify(p.precondition as Expression)
+        : "";
+    }
+  }
+
+  if (p.events) {
+    const e = [];
+    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
+    p.events = e.join(", ");
+  }
+
+  const provision = p.configurations[0];
+  if (
+    (p.configurations as any[]).length === 1 &&
+    provision.type === "provision" &&
+    provision.name &&
+    provision.name.length
+  ) {
+    p.provision = provision.name;
+    p.provisionArgs = provision.args
+      ? provision.args.map((a) => stringify(a)).join(", ")
+      : "";
+  }
+
+  delete p.configurations;
+  return p;
+}
+
+function flattenFile(file: Record<string, unknown>): Record<string, unknown> {
+  const f = {};
+  f["_id"] = file["_id"];
+  if (file.metadata) {
+    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
+    f["metadata.oui"] = file["metadata"]["oui"] || "";
+    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
+    f["metadata.version"] = file["metadata"]["version"] || "";
+  }
+  return f;
+}
+
+function preProcessPreset(data: Record<string, unknown>): MongoTypes.Preset {
+  const preset = Object.assign({}, data);
+
+  if (!preset.precondition) preset.precondition = "";
+  // Try parse to check expression validity
+  parse(preset.precondition as string);
+
+  preset.weight = parseInt(preset.weight as string) || 0;
+
+  const events = {};
+  if (preset.events) {
+    for (let e of (preset.events as string).split(",")) {
+      let v = true;
+      e = e.trim();
+      if (e.startsWith("-")) {
+        v = false;
+        e = e.slice(1).trim();
+      }
+      if (e) events[e] = v;
+    }
+  }
+
+  preset.events = events;
+
+  if (!preset.provision) throw new Error("Invalid preset provision");
+
+  const configuration = {
+    type: "provision",
+    name: preset.provision,
+    args: null,
+  };
+
+  if (preset.provisionArgs)
+    configuration.args = parseList(preset.provisionArgs as string);
+
+  delete preset.provision;
+  delete preset.provisionArgs;
+  preset.configurations = [configuration];
+  return preset as unknown as MongoTypes.Preset;
+}
+
+interface QueryOptions {
+  projection?: any;
+  skip?: number;
+  limit?: number;
+  sort?: {
+    [param: string]: number;
+  };
+}
+
+export async function* query(
   resource: string,
   filter: Expression,
   options?: QueryOptions,
-  callback?: (doc: any) => void
+): AsyncGenerator<any, void, undefined> {
-): Promise<any[]> {
   options = options || {};
-  let q;
+  filter = evaluate(filter, null, Date.now());
-  filter = expression.evaluate(filter, null, Date.now());
+  const q = toMongoQuery(filter, resource);
-  filter = minimize(filter, true);
+  if (!q) return;
 
-  if (Array.isArray(filter)) {
+  const collection = collections[resource] as Collection<any>;
-    if (resource === "devices") {
-      filter = mongodbFunctions.processDeviceFilter(filter);
-    } else if (resource === "tasks") {
-      filter = mongodbFunctions.processTasksFilter(filter);
-    } else if (resource === "faults") {
-      filter = mongodbFunctions.processFaultsFilter(filter);
-    } else if (resource === "users") {
-      // Protect against brute force, and dictionary attacks
-      const params = expression.extractParams(filter);
-      if (params.includes("password") || params.includes("salt"))
-        return Promise.reject(new Error("Invalid users filter"));
-    }
-
-    q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (!filter) {
-    return Promise.resolve([]);
-  }
-
-  return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
   const cursor = collection.find(q);
   if (options.projection) {
     let projection = options.projection;
-      if (resource === "devices") {
+    if (resource === "devices")
-        projection = mongodbFunctions.processDeviceProjection(
+      projection = processDeviceProjection(options.projection);
-          options.projection
-        );
-      }
 
     if (resource === "presets") projection.configurations = 1;
+    projection = optimizeProjection(projection);
     cursor.project(projection);
   }
 
@@ -92,80 +388,35 @@ export function query(
       .reduce(
         (obj, [k, v]) =>
           Object.assign(obj, { [k]: Math.min(Math.max(v, -1), 1) }),
-          {}
+        {},
       );
 
-      if (resource === "devices") s = mongodbFunctions.processDeviceSort(s);
+    if (resource === "devices") s = processDeviceSort(s);
     cursor.sort(s);
   }
 
-    if (!callback) {
+  for await (let doc of cursor) {
-      cursor.toArray((err, docs) => {
+    if (resource === "devices") doc = flattenDevice(doc);
-        if (err) return reject(err);
+    else if (resource === "faults") doc = flattenFault(doc);
-        if (resource === "devices")
+    else if (resource === "tasks") doc = flattenTask(doc);
-          docs = docs.map((d) => mongodbFunctions.flattenDevice(d));
+    else if (resource === "presets") doc = flattenPreset(doc);
-        else if (resource === "faults")
+    else if (resource === "files") doc = flattenFile(doc);
-          docs = docs.map((d) => mongodbFunctions.flattenFault(d));
+
-        else if (resource === "tasks")
+    yield doc;
-          docs = docs.map((d) => mongodbFunctions.flattenTask(d));
-        else if (resource === "presets")
-          docs = docs.map((d) => mongodbFunctions.flattenPreset(d));
-        else if (resource === "files")
-          docs = docs.map((d) => mongodbFunctions.flattenFile(d));
-        return resolve(docs);
-      });
-    } else {
-      cursor.forEach(
-        (doc) => {
-          if (resource === "devices") doc = mongodbFunctions.flattenDevice(doc);
-          else if (resource === "faults")
-            doc = mongodbFunctions.flattenFault(doc);
-          else if (resource === "tasks")
-            doc = mongodbFunctions.flattenTask(doc);
-          else if (resource === "presets")
-            doc = mongodbFunctions.flattenPreset(doc);
-          else if (resource === "files")
-            doc = mongodbFunctions.flattenFile(doc);
-          callback(doc);
-        },
-        (err) => {
-          if (err) reject(err);
-          else resolve();
   }
-      );
-    }
-  });
 }
 
 export function count(resource: string, filter: Expression): Promise<number> {
-  let q;
+  const collection = collections[resource] as Collection<any>;
-  filter = expression.evaluate(filter, null, Date.now());
+  filter = evaluate(filter, null, Date.now());
-  filter = minimize(filter, true);
+  const q = toMongoQuery(filter, resource);
-
+  if (!q) return Promise.resolve(0);
-  if (Array.isArray(filter)) {
+  return collection.countDocuments(q);
-    if (resource === "devices")
-      filter = mongodbFunctions.processDeviceFilter(filter);
-    else if (resource === "tasks")
-      filter = mongodbFunctions.processTasksFilter(filter);
-    else if (resource === "faults")
-      filter = mongodbFunctions.processFaultsFilter(filter);
-    q = mongodbFunctions.filterToMongoQuery(filter);
-  } else if (!filter) {
-    return Promise.resolve(0);
-  }
-
-  return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-    collection.find(q).count((err, c) => {
-      if (err) reject(err);
-      else resolve(c);
-    });
-  });
 }
 
 export async function updateDeviceTags(
   deviceId: string,
-  tags: Record<string, boolean>
+  tags: Record<string, boolean>,
 ): Promise<void> {
   const add = [];
   const pull = [];
@@ -175,161 +426,168 @@ export async function updateDeviceTags(
     if (onOff) add.push(tag);
     else pull.push(tag);
   }
-
-  const collection = db.collection("devices");
   const object = {};
 
-  if (add && add.length) object["$addToSet"] = { _tags: { $each: add } };
+  if (add?.length) object["$addToSet"] = { _tags: { $each: add } };
-  if (pull && pull.length) object["$pullAll"] = { _tags: pull };
+  if (pull?.length) object["$pullAll"] = { _tags: pull };
 
-  await collection.updateOne({ _id: deviceId }, object);
+  await collections.devices.updateOne({ _id: deviceId }, object);
 }
 
-function putResource(resource, id, object): Promise<void> {
+export async function putPreset(
-  return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-    collection.replaceOne({ _id: id }, object, { upsert: true }, (err) => {
-      if (err) return void reject(err);
-      resolve();
-    });
-  });
-}
-
-function deleteResource(
-  resource: string,
-  id: string | ObjectID
-): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const collection = db.collection(RESOURCE_COLLECTION[resource] || resource);
-    collection.deleteOne({ _id: id }, (err) => {
-      if (err) return void reject(err);
-      resolve();
-    });
-  });
-}
-
-export function putPreset(
   id: string,
-  object: Record<string, unknown>
+  object: Record<string, unknown>,
 ): Promise<void> {
-  object = mongodbFunctions.preProcessPreset(object);
+  const p = preProcessPreset(object);
-  return putResource("presets", id, object);
+  await collections.presets.replaceOne({ _id: id }, p, { upsert: true });
 }
 
-export function deletePreset(id: string): Promise<void> {
+export async function deletePreset(id: string): Promise<void> {
-  return deleteResource("presets", id);
+  await collections.presets.deleteOne({ _id: id });
 }
 
-export function putProvision(
+export async function putProvision(
   id: string,
-  object: Record<string, unknown>
+  object: { script: string },
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
-    new Script(`"use strict";(function(){\n${object.script}\n})();`);
+    new Script(`"use strict";(function(){\n${object.script}\n})();`, {
-  } catch (error) {
+      filename: id,
-    return Promise.reject(error);
+      lineOffset: -1,
+    });
+  } catch (err) {
+    if (err.stack?.startsWith(`${id}:`)) {
+      return Promise.reject(
+        new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`),
+      );
     }
-  return putResource("provisions", id, object);
+    return Promise.reject(err);
+  }
+  await collections.provisions.replaceOne({ _id: id }, object, {
+    upsert: true,
+  });
 }
 
-export function deleteProvision(id: string): Promise<void> {
+export async function deleteProvision(id: string): Promise<void> {
-  return deleteResource("provisions", id);
+  await collections.provisions.deleteOne({ _id: id });
 }
 
-export function putVirtualParameter(
+export async function putVirtualParameter(
   id: string,
-  object: Record<string, unknown>
+  object: { script: string },
 ): Promise<void> {
   if (!object.script) object.script = "";
   try {
-    new Script(`"use strict";(function(){\n${object.script}\n})();`);
+    new Script(`"use strict";(function(){\n${object.script}\n})();`, {
-  } catch (error) {
+      filename: id,
-    return Promise.reject(error);
+      lineOffset: -1,
+    });
+  } catch (err) {
+    if (err.stack?.startsWith(`${id}:`)) {
+      return Promise.reject(
+        new Error(`${err.name} at ${err.stack.split("\n", 1)[0]}`),
+      );
     }
-  return putResource("virtualParameters", id, object);
+    return Promise.reject(err);
+  }
+  await collections.virtualParameters.replaceOne({ _id: id }, object, {
+    upsert: true,
+  });
 }
 
-export function deleteVirtualParameter(id: string): Promise<void> {
+export async function deleteVirtualParameter(id: string): Promise<void> {
-  return deleteResource("virtualParameters", id);
+  await collections.virtualParameters.deleteOne({ _id: id });
 }
 
-export function putConfig(
+export async function putConfig(
   id: string,
-  object: Record<string, unknown>
+  object: WithoutId<MongoTypes.Config>,
 ): Promise<void> {
-  return putResource("config", id, object);
+  await collections.config.replaceOne({ _id: id }, object, { upsert: true });
 }
 
-export function deleteConfig(id: string): Promise<void> {
+export async function deleteConfig(id: string): Promise<void> {
-  return deleteResource("config", id);
+  await collections.config.deleteOne({ _id: id });
 }
 
-export function putPermission(
+export async function putPermission(
   id: string,
-  object: Record<string, unknown>
+  object: WithoutId<MongoTypes.Permission>,
 ): Promise<void> {
-  return putResource("permissions", id, object);
+  await collections.permissions.replaceOne({ _id: id }, object, {
+    upsert: true,
+  });
 }
 
-export function deletePermission(id: string): Promise<void> {
+export async function deletePermission(id: string): Promise<void> {
-  return deleteResource("permissions", id);
+  await collections.permissions.deleteOne({ _id: id });
 }
 
-export function putUser(
+export async function putUser(
   id: string,
-  object: Record<string, unknown>
+  object: Partial<WithoutId<MongoTypes.User>>,
 ): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const collection = db.collection("users");
   // update instead of replace to keep the password if not set by user
-    collection.updateOne(
+  await collections.users.updateOne(
     { _id: id },
     { $set: object },
     { upsert: true },
-      (err) => {
-        if (err) return void reject(err);
-        resolve();
-      }
   );
-  });
 }
 
-export function deleteUser(id: string): Promise<void> {
+export async function deleteUser(id: string): Promise<void> {
-  return deleteResource("users", id);
+  await collections.users.deleteOne({ _id: id });
+}
+
+export function downloadFile(filename: string): Readable {
+  return filesBucket.openDownloadStreamByName(filename);
 }
 
 export function putFile(
   filename: string,
   metadata: Record<string, string>,
-  contentStream: Readable
+  contentStream: Readable,
 ): Promise<void> {
   return new Promise((resolve, reject) => {
-    const bucket = new GridFSBucket(db);
+    const uploadStream = filesBucket.openUploadStreamWithId(
-    const uploadStream = bucket.openUploadStreamWithId(filename, filename, {
+      filename as unknown as ObjectId,
+      filename,
+      {
         metadata: metadata,
+      },
+    );
+
+    let readableEnded = false;
+    contentStream.on("end", () => {
+      readableEnded = true;
     });
+    contentStream.on("close", () => {
+      // In Node versions prior to 15, the stream will not emit an error if the
+      // connection is closed before the stream is finished.
+      // For Node 12.9+ we can just use stream.readableEnded
+      if (!readableEnded)
+        uploadStream.destroy(new Error("Stream closed prematurely"));
+    });
+
+    contentStream.on("error", (err) => {
+      uploadStream.destroy(err);
+    });
+
     uploadStream.on("error", reject);
-    contentStream.on("error", reject);
     uploadStream.on("finish", resolve);
     contentStream.pipe(uploadStream);
   });
 }
 
-export function deleteFile(filename: string): Promise<void> {
+export async function deleteFile(filename: string): Promise<void> {
-  return new Promise((resolve, reject) => {
+  await filesBucket.delete(filename as any);
-    const bucket = new GridFSBucket(db);
-    bucket.delete(filename as any, (err) => {
-      if (err) return void reject(err);
-      resolve();
-    });
-  });
 }
 
-export function deleteFault(id: string): Promise<void> {
+export async function deleteFault(id: string): Promise<void> {
-  return deleteResource("faults", id);
+  await collections.faults.deleteOne({ _id: id });
 }
 
-export function deleteTask(id: ObjectID): Promise<void> {
+export async function deleteTask(id: ObjectId): Promise<void> {
-  return deleteResource("tasks", id);
+  await collections.tasks.deleteOne({ _id: id });
 }

lib/ui/local-cache.ts

@@ -0,0 +1,187 @@
+import * as crypto from "node:crypto";
+import * as config from "../config.ts";
+import { collections } from "../db/db.ts";
+import * as expression from "../common/expression/util.ts";
+import { parse } from "../common/expression/parser.ts";
+import { Expression, Users, Permissions, Config, UiConfig } from "../types.ts";
+import { LocalCache } from "../local-cache.ts";
+
+interface Snapshot {
+  permissions: Permissions;
+  users: Users;
+  config: Config;
+  ui: UiConfig;
+}
+
+async function fetchPermissions(): Promise<[string, Permissions]> {
+  const perms = await collections.permissions.find().toArray();
+  perms.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(perms))
+    .digest("hex");
+  const permissions: Permissions = {};
+
+  for (const p of perms) {
+    if (!permissions[p.role]) permissions[p.role] = {};
+    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
+
+    permissions[p.role][p.access][p.resource] = {
+      access: p.access,
+      filter: parse(p.filter || "true"),
+    };
+    if (p.validate)
+      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
+  }
+
+  return [h, permissions];
+}
+
+async function fetchUsers(): Promise<[string, Users]> {
+  const _users = await collections.users.find().toArray();
+  _users.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(_users))
+    .digest("hex");
+  const users = {};
+
+  for (const user of _users) {
+    users[user._id] = {
+      password: user.password,
+      salt: user.salt,
+      roles: user.roles.split(",").map((s) => s.trim()),
+    };
+  }
+
+  return [h, users];
+}
+
+async function fetchConfig(): Promise<[string, Config, UiConfig]> {
+  const conf = await collections.config.find().toArray();
+  conf.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(conf)).digest("hex");
+
+  const ui = {
+    filters: {},
+    device: {},
+    index: {},
+    overview: {},
+    pageSize: null,
+  };
+
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    const val = expression.evaluate(parse(c.value));
+    _config[c._id] = val;
+    if (c._id.startsWith("ui.")) {
+      const keys = c._id.split(".");
+      if (!(keys[1] in ui)) continue;
+      // remove the first key(ui)
+      keys.shift();
+      let ref = ui;
+      while (keys.length > 1) {
+        const k = keys.shift();
+        if (ref[k] == null || typeof ref[k] !== "object") ref[k] = {};
+        ref = ref[k];
+      }
+      ref[keys[0]] = val;
+    }
+  }
+
+  return [h, _config, ui];
+}
+
+const localCache = new LocalCache<Snapshot>("ui-local-cache-hash", refresh);
+
+async function refresh(): Promise<[string, Snapshot]> {
+  const res = await Promise.all([
+    fetchPermissions(),
+    fetchUsers(),
+    fetchConfig(),
+  ]);
+
+  const h = crypto.createHash("md5");
+  for (const r of res) h.update(r[0]);
+
+  const snapshot = {
+    permissions: res[0][1],
+    users: res[1][1],
+    config: res[2][1],
+    ui: res[2][2],
+  };
+
+  return [h.digest("hex"), snapshot];
+}
+
+export async function getRevision(): Promise<string> {
+  return await localCache.getRevision();
+}
+
+export function getConfig(
+  revision: string,
+  key: string,
+  context: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Expression,
+): string | number | boolean | null {
+  const snapshot = localCache.get(revision);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const oldOpts = {
+    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
+    "cwmp.debug": "DEBUG",
+    "cwmp.retryDelay": "RETRY_DELAY",
+    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
+    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
+    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
+    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
+    "cwmp.cookiesPath": "COOKIES_PATH",
+    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
+    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
+    "cwmp.connectionRequestAllowBasicAuth":
+      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
+    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
+    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
+  };
+
+  if (!(key in snapshot.config)) {
+    if (key in oldOpts) {
+      let id;
+      if (context?.["id"]) {
+        id = context["id"];
+      } else if (cb) {
+        id = cb(["PARAM", "DeviceID.ID"]);
+        if (Array.isArray(id)) id = null;
+      }
+      return config.get(oldOpts[key], id);
+    }
+    return null;
+  }
+
+  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+  return Array.isArray(v) ? null : v;
+}
+
+export function getConfigExpression(revision: string, key: string): Expression {
+  const snapshot = localCache.get(revision);
+  return snapshot.config[key];
+}
+
+export function getUsers(revision: string): Users {
+  const snapshot = localCache.get(revision);
+  return snapshot.users;
+}
+
+export function getPermissions(revision: string): Permissions {
+  const snapshot = localCache.get(revision);
+  return snapshot.permissions;
+}
+
+export function getUiConfig(revision: string): UiConfig {
+  const snapshot = localCache.get(revision);
+  return snapshot.ui;
+}

lib/util.ts

@@ -1,24 +1,7 @@
-/**
+import { EventEmitter } from "node:events";
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
 
 export function generateDeviceId(
-  deviceIdStruct: Record<string, string>
+  deviceIdStruct: Record<string, string>,
 ): string {
   // Escapes everything except alphanumerics and underscore
   function esc(str): string {
@@ -51,8 +34,8 @@ export function escapeRegExp(str: string): string {
 export function encodeTag(tag: string): string {
   return encodeURIComponent(tag)
     .replace(
-      /[!~*'()]/g,
+      /[!~*'().]/g,
-      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
+      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase(),
     )
     .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")
     .replace(/%/g, "0x");
@@ -61,3 +44,27 @@ export function encodeTag(tag: string): string {
 export function decodeTag(tag: string): string {
   return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%"));
 }
+
+export function once(
+  emitter: EventEmitter,
+  event: string,
+  timeout: number,
+): Promise<unknown[]> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(new Error(`Event ${event} timed out after ${timeout} ms`));
+    }, timeout);
+
+    emitter.once(event, (...args: unknown[]) => {
+      clearTimeout(timer);
+      resolve(args);
+    });
+  });
+}
+
+export function setTimeoutPromise(delay: number, ref = true): Promise<void> {
+  return new Promise((resolve) => {
+    const timerId = setTimeout(resolve, delay);
+    if (!ref) timerId.unref();
+  });
+}

lib/versioned-map.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 const NONEXISTENT = Symbol();
 const UNDEFINED = undefined;
 
@@ -26,10 +7,10 @@ interface Revisions<V> {
 }
 
 export default class VersionedMap<K, V> {
-  private _sizeDiff: number[];
+  private declare _sizeDiff: number[];
-  private _revision: number;
+  private declare _revision: number;
-  private map: Map<K, (V | symbol)[]>;
+  private declare map: Map<K, (V | symbol)[]>;
-  public dirty: number;
+  public declare dirty: number;
 
   public constructor() {
     this._sizeDiff = [0];
@@ -138,7 +119,7 @@ export default class VersionedMap<K, V> {
     const del = revisionsObj.delete || 0;
     const mutations = Object.keys(revisionsObj).reduce(
       (acc, cur) => (cur === "delete" ? acc : acc | (1 << +cur)),
-      del
+      del,
     );
 
     const revisions = [];
@@ -168,6 +149,7 @@ export default class VersionedMap<K, V> {
 
   public *diff(): IterableIterator<[K, V, V]> {
     for (const [key, revisions] of this.map) {
+      if (revisions.length === 1) continue;
       let first = revisions[0];
       let last = revisions[revisions.length - 1];
       if (first === NONEXISTENT && last === NONEXISTENT) continue;

lib/xml-parser.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 const CHAR_SINGLE_QUOTE = 39;
 const CHAR_DOUBLE_QUOTE = 34;
 const CHAR_LESS_THAN = 60;

lib/xmpp-client.ts

@@ -0,0 +1,460 @@
+import * as net from "node:net";
+import * as tls from "node:tls";
+import { EventEmitter } from "node:events";
+import { createHash, createHmac, randomBytes } from "node:crypto";
+import { parseXml, Element, parseAttrs } from "./xml-parser.ts";
+
+function encodeBase64(str: string): string {
+  return Buffer.from(str).toString("base64");
+}
+
+function decodeBase64(str: string): string {
+  return Buffer.from(str, "base64").toString();
+}
+
+function detectStreamTag(data: string): number {
+  const i1 = data.indexOf("<stream:stream");
+  if (i1 < 0) throw new Error("Cannot detect opening stream tag");
+  const i2 = data.indexOf(">", i1);
+  if (i2 < 0) throw new Error("Cannot detect opening stream tag");
+  return i2 + 1;
+}
+
+function xmppStream<T>(
+  socket: net.Socket,
+  callback: Generator<void, T, Element>,
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const onError = (err: Error): void => {
+      socket.removeListener("error", onError);
+      reject(err);
+    };
+    socket.on("error", onError);
+
+    const onData = (chunk: Buffer): void => {
+      try {
+        const str = chunk.toString("utf8");
+        const xml = parseXml(str);
+        const { value, done } = callback.next(xml.children[0]);
+        if (done) {
+          socket.removeListener("error", onError);
+          socket.removeListener("data", onData);
+          resolve(value as T);
+        }
+      } catch (err) {
+        socket.removeListener("error", onError);
+        socket.removeListener("data", onData);
+        reject(err);
+      }
+    };
+
+    socket.once("data", (chunk: Buffer) => {
+      try {
+        const str = chunk.toString("utf8");
+        const i = detectStreamTag(str);
+        const streamTagStr = str.slice(0, i);
+        const xml = parseXml(streamTagStr + "</stream:stream>");
+        callback.next(xml.children[0]);
+        chunk = chunk.slice(Buffer.byteLength(streamTagStr));
+        if (chunk.length) onData(chunk);
+        socket.on("data", onData);
+      } catch (err) {
+        socket.removeListener("error", onError);
+        socket.removeListener("data", onData);
+        reject(err);
+      }
+    });
+
+    try {
+      callback.next();
+    } catch (err) {
+      socket.removeListener("error", onError);
+      socket.removeListener("data", onData);
+      reject(err);
+    }
+  });
+}
+
+const INT_1 = Buffer.from([0, 0, 0, 1]);
+const saltedPasswordCache = {
+  password: "",
+  iterationCount: 0,
+  saltBase64: "",
+  salted: Buffer.allocUnsafe(0),
+};
+
+function saltPassword(
+  password: string,
+  saltBase64: string,
+  iteractionCount: number,
+): Buffer {
+  if (
+    password === saltedPasswordCache.password &&
+    saltBase64 === saltedPasswordCache.saltBase64 &&
+    iteractionCount === saltedPasswordCache.iterationCount
+  )
+    return saltedPasswordCache.salted;
+
+  const hi = createHmac("sha1", password)
+    .update(Buffer.concat([Buffer.from(saltBase64, "base64"), INT_1]))
+    .digest();
+  let hi2: Buffer = hi;
+  for (let i = 1; i < iteractionCount; ++i) {
+    hi2 = createHmac("sha1", password).update(hi2).digest();
+    for (const [j, b] of hi2.entries()) hi[j] ^= b;
+  }
+
+  saltedPasswordCache.saltBase64 = saltBase64;
+  saltedPasswordCache.password = password;
+  saltedPasswordCache.iterationCount = iteractionCount;
+  saltedPasswordCache.salted = hi;
+
+  return hi;
+}
+
+function* loginPlain(
+  socket: net.Socket,
+  username: string,
+  password: string,
+): Generator<void, void, Element> {
+  socket.write(
+    `<auth xmlns="urn:ietf:params:xml:ns:xmpp-sasl" mechanism="PLAIN">${encodeBase64(
+      `\x00${username}\x00${password}`,
+    )}</auth>`,
+  );
+  const res1 = yield;
+
+  if (
+    res1.name === "failure" &&
+    res1.children.some((c) => c.name === "not-authorized")
+  )
+    throw new Error("Not authorized");
+
+  if (res1.name !== "success")
+    throw new Error(`Unexpected response ${res1.name}`);
+}
+
+function* loginScram(
+  socket: net.Socket,
+  username: string,
+  password: string,
+): Generator<void, void, Element> {
+  const cnonce = randomBytes(8).toString("base64");
+  const gs2Header = "n,,";
+  const clientFirstMessageBare = `n=${username},r=${cnonce}`;
+  const clientFirstMessage = gs2Header + clientFirstMessageBare;
+  socket.write(
+    `<auth xmlns="urn:ietf:params:xml:ns:xmpp-sasl" mechanism="SCRAM-SHA-1">${encodeBase64(
+      clientFirstMessage,
+    )}</auth>`,
+  );
+  const res1 = yield;
+  if (res1.name !== "challenge")
+    throw new Error(`Unexpected element ${res1.name}`);
+  const serverFirstMessage = decodeBase64(res1.text);
+
+  let iterationCount: number;
+  let saltBase64: string;
+  let nonce: string;
+  for (const s of serverFirstMessage.split(",")) {
+    if (s.startsWith("i=")) iterationCount = parseInt(s.slice(2));
+    else if (s.startsWith("s=")) saltBase64 = s.slice(2);
+    else if (s.startsWith("r=")) nonce = s.slice(2);
+  }
+  if (iterationCount == null || isNaN(iterationCount))
+    throw new Error("Invalid iteration count");
+  if (saltBase64 == null) throw new Error("Missing salt");
+  if (nonce == null) throw new Error("Missing nonce");
+
+  const saltedPassword = saltPassword(password, saltBase64, iterationCount);
+  const clientKey = createHmac("sha1", saltedPassword)
+    .update("Client Key")
+    .digest();
+  const storedKey = createHash("sha1").update(clientKey).digest();
+
+  const clientFinalMessageWithoutProof = `c=${encodeBase64(
+    gs2Header,
+  )},r=${nonce}`;
+  const authMessage = `${clientFirstMessageBare},${serverFirstMessage},${clientFinalMessageWithoutProof}`;
+  const clientSignature = createHmac("sha1", storedKey)
+    .update(authMessage)
+    .digest();
+
+  const clientProof = Buffer.from(clientKey);
+  for (const [i, b] of clientSignature.entries()) clientProof[i] ^= b;
+
+  const clientFinalMessage = `${clientFinalMessageWithoutProof},p=${clientProof.toString(
+    "base64",
+  )}`;
+  socket.write(
+    `<response xmlns="urn:ietf:params:xml:ns:xmpp-sasl">${encodeBase64(
+      clientFinalMessage,
+    )}</response>`,
+  );
+  const res2 = yield;
+
+  if (
+    res2.name === "failure" &&
+    res2.children.some((c) => c.name === "not-authorized")
+  )
+    throw new Error("Not authorized");
+
+  if (res2.name !== "success")
+    throw new Error(`Unexpected response ${res2.name}`);
+
+  const serverKey = createHmac("sha1", saltedPassword)
+    .update("Server Key")
+    .digest();
+  const serverSignature = createHmac("sha1", serverKey)
+    .update(authMessage)
+    .digest("base64");
+
+  if (!decodeBase64(res2.text).endsWith(serverSignature))
+    throw new Error("Invalid server signature");
+}
+
+function* starttls(socket: net.Socket): Generator<void, void, Element> {
+  socket.write("<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>");
+  const res1 = yield;
+  if (res1.name !== "proceed") throw new Error("Failed to initiate STARTTLS");
+}
+
+function* bind(
+  socket: net.Socket,
+  resource: string,
+): Generator<void, void, Element> {
+  const id = randomBytes(8).toString("base64");
+  socket.write(
+    `<iq id='${id}' type='set'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'><resource>${resource}</resource></bind></iq>`,
+  );
+  const res1 = yield;
+  if (res1.name !== "iq") throw new Error(`Unexpected element ${res1.name}`);
+  const attrs1 = parseAttrs(res1.attrs);
+  const idAttr = attrs1.find((a) => a.name === "id");
+  if (!idAttr || idAttr.value !== id) throw new Error("Invalid ID");
+  const typeAttr = attrs1.find((a) => a.name === "type");
+  if (!typeAttr) throw new Error("Missing type attribute");
+  if (typeAttr.value !== "result") throw new Error("Cannot bind to resource");
+}
+
+const STATUS_RESTART_STREAM = 1;
+const STATUS_STARTTLS = 2;
+
+function* init(
+  socket: net.Socket,
+  host: string,
+  username: string,
+  password: string,
+  resource: string,
+): Generator<void, number, Element> {
+  socket.write(
+    `<?xml version='1.0'?><stream:stream from='${username}@${host}' to='${host}' version='1.0' xml:lang='en' xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams'>`,
+  );
+  const open = yield;
+  if (open.name !== "stream:stream")
+    throw new Error(`Unexpected element ${open.name}`);
+  const features = yield;
+  if (features.name !== "stream:features")
+    throw new Error(`Unexpected element ${features.name}`);
+  for (const feature of features.children) {
+    if (feature.name === "starttls") {
+      if (feature.children.some((c) => c.name === "required")) {
+        yield* starttls(socket);
+        return STATUS_STARTTLS;
+      }
+    } else if (feature.name === "mechanisms") {
+      const mechanisms: Set<string> = new Set(
+        feature.children.map((c) => c.text),
+      );
+      if (mechanisms.has("PLAIN")) {
+        yield* loginPlain(socket, username, password);
+        return STATUS_RESTART_STREAM;
+      } else if (mechanisms.has("SCRAM-SHA-1")) {
+        yield* loginScram(socket, username, password);
+        return STATUS_RESTART_STREAM;
+      } else {
+        throw new Error("No supported SASL method");
+      }
+    } else if (feature.name === "bind") {
+      yield* bind(socket, resource);
+      return 0;
+    }
+  }
+  return 0;
+}
+
+function upgradeTls(socket: net.Socket, host: string): Promise<tls.TLSSocket> {
+  return new Promise((resolve, reject) => {
+    socket.on("error", reject);
+    const newSocket = tls.connect({ socket, host }, () => {
+      socket.removeListener("error", reject);
+      resolve(newSocket);
+    });
+  });
+}
+
+interface XmppClientOptions {
+  host: string;
+  port?: number;
+  username?: string;
+  password?: string;
+  resource?: string;
+  timeout?: number;
+}
+
+export default class XmppClient extends EventEmitter {
+  private _socket: net.Socket;
+  private _host: string;
+  private _username: string;
+  private _resource: string;
+  private _iqStanzaCallbacks: Map<
+    string,
+    (err: Error, r?: { rawRes: string; res: Element }) => void
+  >;
+
+  private constructor() {
+    super();
+    this._socket = null;
+    this._host = null;
+    this._username = null;
+    this._resource = null;
+    this._iqStanzaCallbacks = new Map();
+  }
+
+  static async connect(opts: XmppClientOptions): Promise<XmppClient> {
+    function connectSocket(host: string, port: number): Promise<net.Socket> {
+      return new Promise((resolve, reject) => {
+        const socket = new net.Socket();
+        socket.on("error", reject);
+        socket.connect(port, host, () => {
+          socket.removeListener("error", reject);
+          resolve(socket);
+        });
+      });
+    }
+
+    let socket = await connectSocket(opts.host, opts.port || 5222);
+    try {
+      let status = 1;
+      while (status) {
+        if (status === STATUS_STARTTLS)
+          socket = await upgradeTls(socket, opts.host);
+        status = await xmppStream(
+          socket,
+          init(socket, opts.host, opts.username, opts.password, opts.resource),
+        );
+      }
+    } catch (err) {
+      socket.destroy();
+      throw err;
+    }
+
+    const client = new XmppClient();
+    client._socket = socket;
+    client._host = opts.host;
+    client._username = opts.username;
+    client._resource = opts.resource;
+    socket.on("data", client._onData.bind(client));
+    socket.on("error", client._onError.bind(client));
+    if (opts.timeout)
+      socket.setTimeout(opts.timeout, client.close.bind(client));
+    return client;
+  }
+
+  close(): void {
+    this._socket.end("</stream:stream>");
+    this._socket.removeAllListeners("data");
+    this._socket.removeAllListeners("error");
+    this.emit("close");
+  }
+
+  ref(): void {
+    this._socket.ref();
+  }
+
+  unref(): void {
+    this._socket.unref();
+  }
+
+  get host(): string {
+    return this._host;
+  }
+
+  get username(): string {
+    return this._username;
+  }
+
+  get resource(): string {
+    return this._resource;
+  }
+
+  private _onData(chunk: Buffer): void {
+    try {
+      let close = false;
+      let str = chunk.toString("utf8");
+      if (str.endsWith("</stream:stream>")) {
+        str = str.slice(0, -16);
+        close = true;
+      }
+      const xml = parseXml(str);
+      const idx = xml.children.map((c) => c.bodyIndex);
+      for (const [i, c] of xml.children.entries()) {
+        const s = str.slice(idx[i], idx[i + 1]);
+        if (c.name === "iq") {
+          const attrs = parseAttrs(c.attrs);
+          const id = attrs.find((a) => a.name === "id");
+          if (id) {
+            const cb = this._iqStanzaCallbacks.get(id.value);
+            if (cb) cb(null, { rawRes: s, res: c });
+          }
+        }
+        this.emit("stanza", c, s);
+      }
+      if (close) {
+        this._socket.removeAllListeners("data");
+        this._socket.removeAllListeners("error");
+        this.emit("close");
+      }
+    } catch (err) {
+      this._socket.removeAllListeners("data");
+      this._socket.removeAllListeners("error");
+      this.emit("error", err);
+    }
+  }
+
+  private _onError(err: Error): void {
+    this._socket.end();
+    this.emit("error", err);
+    for (const cb of this._iqStanzaCallbacks.values()) cb(err);
+  }
+
+  send(msg: string): void {
+    this._socket.write(msg);
+  }
+
+  sendIqStanza(
+    from: string,
+    to: string,
+    type: string,
+    body: string,
+    timeout = 3000,
+  ): Promise<{ rawReq: string; rawRes: string; res: Element }> {
+    return new Promise((resolve, reject) => {
+      const id = randomBytes(8).toString("base64");
+      const rawReq = `<iq from="${from}" to="${to}" id="${id}" type="${type}">${body}</iq>`;
+      this.send(rawReq);
+      const t = setTimeout(() => {
+        this._iqStanzaCallbacks.delete(id);
+        reject(
+          new Error("Did not receive IQ stanza response in a timely manner"),
+        );
+      }, timeout);
+      this._iqStanzaCallbacks.set(id, (err, r) => {
+        this._iqStanzaCallbacks.delete(id);
+        clearTimeout(t);
+        if (err) reject(err);
+        else resolve(Object.assign(r, { rawReq }));
+      });
+    });
+  }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "genieacs",
-  "version": "1.2.3",
+  "version": "1.2.13",
   "description": "A TR-069 Auto Configuration Server (ACS)",
   "repository": {
     "type": "git",
@@ -12,77 +12,63 @@
     "CWMP",
     "ACS"
   ],
-  "author": "Zaid Abdulla <zaid@genieacs.com>",
+  "author": {
+    "name": "GenieACS Inc.",
+    "url": "https://genieacs.com"
+  },
   "license": "AGPL-3.0",
-  "preferGlobal": true,
+  "private": true,
-  "directories": {
+  "bin": {
-    "bin": "bin",
+    "genieacs-cwmp": "bin/genieacs-cwmp",
-    "tools": "tools"
+    "genieacs-fs": "bin/genieacs-fs",
+    "genieacs-nbi": "bin/genieacs-nbi",
+    "genieacs-ui": "bin/genieacs-ui"
   },
   "dependencies": {
-    "espresso-iisojs": "^1.0.3",
+    "@breejs/later": "^4.2.0",
-    "iconv-lite": "^0.6.2",
+    "@koa/bodyparser": "^5.1.1",
-    "ipaddr.js": "^2.0.0",
+    "@koa/router": "^13.1.0",
-    "jsonwebtoken": "^8.5.1",
+    "bson": "^4.7.2",
-    "koa": "^2.13.0",
+    "espresso-iisojs": "^1.0.8",
-    "koa-bodyparser": "^4.3.0",
+    "iconv-lite": "^0.6.3",
-    "koa-compress": "^5.0.1",
+    "ipaddr.js": "^2.2.0",
-    "koa-jwt": "^4.0.0",
+    "jsonwebtoken": "^9.0.2",
-    "koa-router": "^9.4.0",
+    "koa": "^2.15.3",
-    "koa-static": "^5.0.0",
+    "koa-compress": "^5.1.0",
-    "later": "^1.2.0",
+    "koa-jwt": "^4.0.3",
-    "mongodb": "^3.6.2",
+    "koa-send": "^5.0.1",
-    "parsimmon": "^1.16.0",
+    "mongodb": "^4.16.0",
-    "seedrandom": "^3.0.5",
+    "parsimmon": "^1.18.1",
-    "yaml": "^1.10.0"
+    "seedrandom": "^3.0.5"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^16.0.0",
+    "@types/codemirror": "^5.60.15",
-    "@rollup/plugin-json": "^4.1.0",
+    "@types/jsonwebtoken": "^9.0.6",
-    "@rollup/plugin-node-resolve": "^10.0.0",
+    "@types/koa": "^2.15.0",
-    "@rollup/plugin-typescript": "^6.0.0",
+    "@types/koa-compress": "^4.0.6",
-    "@types/jsonwebtoken": "^8.5.0",
+    "@types/mithril": "^2.2.6",
-    "@types/koa": "^2.11.6",
+    "@types/node": "^20.14.2",
-    "@types/koa-compress": "^4.0.1",
+    "@types/parsimmon": "^1.10.9",
-    "@types/koa-router": "^7.4.1",
+    "@types/seedrandom": "^3.0.8",
-    "@types/koa-static": "^4.0.1",
+    "@typescript-eslint/eslint-plugin": "^7.12.0",
-    "@types/mithril": "^2.0.5",
+    "@typescript-eslint/parser": "^7.12.0",
-    "@types/mongodb": "^3.5.31",
+    "codemirror": "^5.65.16",
-    "@types/node": "^14.14.5",
+    "esbuild": "^0.25.0",
-    "@types/parsimmon": "^1.10.3",
+    "eslint": "^8.57.0",
-    "@types/seedrandom": "^2.4.28",
+    "eslint-config-prettier": "^9.1.0",
-    "@types/svgo": "^1.3.3",
+    "mithril": "^2.2.2",
-    "@typescript-eslint/eslint-plugin": "^4.6.0",
+    "prettier": "^3.3.1",
-    "@typescript-eslint/parser": "^4.6.0",
+    "sql.js": "^1.10.3",
-    "ava": "^3.13.0",
+    "svgo": "^3.3.2",
-    "codemirror": "^5.58.2",
+    "typescript": "^5.4.5",
-    "cssnano": "^4.1.10",
+    "yaml": "^1.10.2"
-    "eslint": "^7.12.1",
-    "eslint-config-prettier": "^6.14.0",
-    "esm": "^3.2.25",
-    "jsbi": "^3.1.4",
-    "mithril": "^2.0.4",
-    "postcss": "^8.1.4",
-    "postcss-import": "^13.0.0",
-    "postcss-preset-env": "^6.7.0",
-    "prettier": "^2.1.2",
-    "rollup": "^2.32.1",
-    "rollup-plugin-terser": "^7.0.2",
-    "source-map-loader": "^1.1.1",
-    "svgo": "^1.3.2",
-    "ts-node": "^9.0.0",
-    "typescript": "^4.0.5",
-    "webpack": "^5.2.0"
   },
   "engines": {
     "node": ">=12.13.0"
   },
   "scripts": {
-    "test": "ava --verbose --color",
+    "test": "esbuild --log-level=warning --bundle --platform=node --target=node18 --packages=external --sourcemap=inline --outdir=test test/*.ts && node --test --enable-source-maps test/*.js && rm test/*.js",
-    "lint-eslint": "eslint --fix \"bin/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/*.ts\"",
+    "lint": "esbuild build/lint.ts --bundle --platform=node --target=node16 --packages=external | node",
-    "lint-prettier": "prettier --parser typescript --write \"bin/**/*.ts\" \"lib/**/*.ts\" \"ui/**/*.ts\" \"test/**/*.ts\" \"build/**/*.ts\" \"tools/**/*.ts\"",
+    "build": "esbuild build/build.ts --bundle --platform=node --target=node12 --packages=external | node"
-    "lint-tsc": "tsc --noEmit",
-    "lint": "npm run lint-eslint && npm run lint-prettier && npm run lint-tsc",
-    "build": "node -r esm -r ts-node/register/transpile-only build/build"
   }
 }

test/auth.ts

@@ -1,8 +1,9 @@
-import ava from "ava";
+import test from "node:test";
-import { randomBytes } from "crypto";
+import assert from "node:assert";
-import * as auth from "../lib/auth";
+import { randomBytes } from "node:crypto";
+import * as auth from "../lib/auth.ts";
 
-ava("digest", (t) => {
+void test("digest", () => {
   const username = "test";
   const password = "test";
   const uri = "/";
@@ -17,8 +18,6 @@ ava("digest", (t) => {
     `Digest realm="${realm}",nonce="${nonce}",qop="auth-int"`,
   ];
 
-  t.plan(challenges.length);
-
   for (const challenge of challenges) {
     const wwwAuthHeader = auth.parseWwwAuthenticateHeader(challenge);
     const solution = auth.solveDigest(
@@ -27,10 +26,10 @@ ava("digest", (t) => {
       uri,
       method,
       body,
-      wwwAuthHeader
+      wwwAuthHeader,
     );
     const authHeader = auth.parseAuthorizationHeader(solution);
-    t.is(
+    assert.strictEqual(
       authHeader["response"],
       auth.digest(
         username,
@@ -42,8 +41,8 @@ ava("digest", (t) => {
         authHeader["qop"],
         body,
         authHeader["cnonce"],
-        authHeader["nc"]
+        authHeader["nc"],
-      )
+      ),
     );
   }
 });

test/db.ts

@@ -0,0 +1,168 @@
+import test from "node:test";
+import assert from "node:assert";
+import { EJSON } from "bson";
+import { Filter } from "mongodb";
+import { stringify, parse } from "../lib/common/expression/parser.ts";
+import { convertOldPrecondition } from "../lib/db/util.ts";
+import { toMongoQuery } from "../lib/db/synth.ts";
+
+void test("convertOldPrecondition", () => {
+  const tests = [
+    [{}, "true"],
+    [{ test: "test" }, 'test = "test"'],
+    [{ test: { $eq: "test" } }, 'test = "test"'],
+    [{ test: { $ne: "test" } }, 'test <> "test" OR test IS NULL'],
+    [{ test: { $gte: "test" } }, 'test >= "test"'],
+    [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
+    [{ "Tags.test": false }, "Tags.test IS NULL"],
+    [{ "Tags.test": { $exists: true } }, "Tags.test IS NOT NULL"],
+    [{ "Tags.test": { $exists: false } }, "Tags.test IS NULL"],
+    [{ "Tags.test": { $ne: true } }, "Tags.test IS NULL"],
+    [{ "Tags.test": { $ne: false } }, "Tags.test IS NOT NULL"],
+    [{ "Tags.test": { $eq: true } }, "Tags.test IS NOT NULL"],
+    [{ "Tags.test": { $eq: false } }, "Tags.test IS NULL"],
+    [{ _tags: "test" }, "Tags.test IS NOT NULL"],
+    [{ _tags: { $ne: "test" } }, "Tags.test IS NULL"],
+    [{ _tags: { $eq: "test" } }, "Tags.test IS NOT NULL"],
+    [
+      { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
+      'test = "test" AND (test <> "test" OR test IS NULL)',
+    ],
+    [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
+    [
+      { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
+      'test = "test" OR (test <> "test" OR test IS NULL)',
+    ],
+    [
+      { test: { $gte: "test1", $ne: "test2" } },
+      'test >= "test1" AND (test <> "test2" OR test IS NULL)',
+    ],
+    [
+      { test: "test", test2: { $ne: "test2" } },
+      'test = "test" AND (test2 <> "test2" OR test2 IS NULL)',
+    ],
+  ];
+
+  const shouldFailTests = [
+    [{ test: { $gee: "test" } }, "Operator $gee not supported"],
+    [{ test: [] }, "Invalid type"],
+    [{ "Tags.test": { $gt: true } }, "Invalid tag query"],
+    [{ _tags: [] }, "Invalid type"],
+    [{ _tags: { $gt: "test" } }, "Invalid tag query"],
+    [{ $nor: [] }, "Operator $nor not supported"],
+  ] as [Filter<unknown>, string][];
+
+  for (const t of tests) {
+    assert.strictEqual(
+      stringify(convertOldPrecondition(t[0] as Record<string, unknown>)),
+      t[1],
+    );
+  }
+
+  for (const t of shouldFailTests) {
+    const func = (): void => {
+      convertOldPrecondition(t[0]);
+    };
+    assert.throws(func, new Error(t[1]));
+  }
+});
+
+void test("toMongoQuery", async () => {
+  const queries: [string, Filter<unknown> | false][] = [
+    ["true", {}],
+    ["Tags.tag1 = true", { _tags: { $eq: "tag1" } }],
+    ["Tags.tag1 <> false", { _tags: { $eq: "tag1" } }],
+    ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }],
+    ["Tags.tag1 = 123", false],
+    ["Param1 = 'value1'", { "Param1._value": { $eq: "value1" } }],
+    [
+      "Param1 <> 'value1'",
+      {
+        "Param1._value": { $ne: "value1" },
+        $and: [{ "Param1._value": { $ne: null } }],
+      },
+    ],
+    [
+      "Param1 <> 1657844103524",
+      {
+        "Param1._value": { $ne: 1657844103524 },
+
+        $and: [
+          {
+            "Param1._value": { $ne: { $date: "2022-07-15T00:15:03.524Z" } },
+          },
+          { "Param1._value": { $ne: null } },
+        ],
+      },
+    ],
+    [
+      "Param1 = 1657844103524",
+      {
+        $or: [
+          { "Param1._value": { $eq: { $date: "2022-07-15T00:15:03.524Z" } } },
+          { "Param1._value": { $eq: 1657844103524 } },
+        ],
+      },
+    ],
+    ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
+    ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
+    [
+      "Param1 LIKE 'value'",
+      {
+        "Param1._value": {
+          $regularExpression: { options: "s", pattern: "^value$" },
+        },
+      },
+    ],
+
+    [
+      "LOWER(Param1) LIKE 'value'",
+      {
+        "Param1._value": {
+          $regularExpression: { options: "is", pattern: "^value$" },
+        },
+      },
+    ],
+    [
+      "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')",
+      {
+        $or: [
+          {
+            "Param2._value": { $ne: null },
+            $and: [{ "Param2._value": { $ne: "value1" } }],
+            "Param1._value": { $eq: "value2" },
+          },
+          {
+            "Param1._value": { $ne: "value2" },
+            $and: [{ "Param1._value": { $ne: null } }],
+          },
+        ],
+      },
+    ],
+    [
+      "Param1 <> 'value2' OR Param1 IS NULL",
+      { "Param1._value": { $ne: "value2" } },
+    ],
+  ];
+
+  for (const [expStr, expect] of queries) {
+    const exp = parse(expStr);
+    let query = toMongoQuery(exp, "devices");
+    if (query) query = EJSON.serialize(query);
+    assert.deepStrictEqual(query, expect);
+  }
+
+  const failQueries: [any, string][] = [
+    ["Param1 = Param2", "Right-hand operand must be a literal value"],
+    ["Param1 LIKE Param2", "Right-hand operand of 'LIKE' must be a string"],
+    ["NOW() = 1", "Left-hand operand must be a parameter"],
+    ["param{param2} = 1", "Left-hand operand must be a parameter"],
+  ];
+
+  for (const [expStr, err] of failQueries) {
+    const exp = parse(expStr);
+    assert.throws(() => toMongoQuery(exp, "devices"), {
+      message: err,
+    });
+  }
+});

test/device.ts

@@ -1,28 +1,28 @@
-import ava from "ava";
+import test from "node:test";
-import Path from "../lib/common/path";
+import assert from "node:assert";
-import * as device from "../lib/device";
+import Path from "../lib/common/path.ts";
-import PathSet from "../lib/common/path-set";
+import * as device from "../lib/device.ts";
-import VersionedMap from "../lib/versioned-map";
+import PathSet from "../lib/common/path-set.ts";
-import { Attributes } from "../lib/types";
+import VersionedMap from "../lib/versioned-map.ts";
+import { Attributes } from "../lib/types.ts";
 
-ava("getAliasDeclarations", (t) => {
+void test("getAliasDeclarations", () => {
   const path = Path.parse("a.[aa:10,bb.[aaa:100].cc:1].b");
   const decs = device.getAliasDeclarations(path, 99);
 
   const expected = ["a.*.b", "a.*.aa", "a.*.bb.*.cc", "a.*.bb.*.aaa"];
 
-  t.plan(24);
   for (const [i, d] of decs.entries()) {
-    t.is(d.path.toString(), expected[i]);
+    assert.strictEqual(d.path.toString(), expected[i]);
-    t.is(d.pathGet, 99);
+    assert.strictEqual(d.pathGet, 99);
-    t.is(d.pathSet, null);
+    assert.strictEqual(d.pathSet, null);
-    t.deepEqual(d.attrGet, i ? { value: 99 } : null);
+    assert.deepStrictEqual(d.attrGet, i ? { value: 99 } : null);
-    t.is(d.attrSet, null);
+    assert.strictEqual(d.attrSet, null);
-    t.is(d.defer, true);
+    assert.strictEqual(d.defer, true);
   }
 });
 
-ava("unpack", (t) => {
+void test("unpack", () => {
   const now = Date.now();
   const deviceData = {
     paths: new PathSet(),
@@ -85,13 +85,13 @@ ava("unpack", (t) => {
   let unpacked: Path[];
   unpacked = device.unpack(
     deviceData,
-    Path.parse("a.[b:b,c:c].a.[b:b1,c:c1].a")
+    Path.parse("a.[b:b,c:c].a.[b:b1,c:c1].a"),
   );
-  t.is(unpacked.length, 2);
+  assert.strictEqual(unpacked.length, 2);
-  t.is(unpacked[0].toString(), "a.1.a.1.a");
+  assert.strictEqual(unpacked[0].toString(), "a.1.a.1.a");
-  t.is(unpacked[1].toString(), "a.2.a.1.a");
+  assert.strictEqual(unpacked[1].toString(), "a.2.a.1.a");
 
   unpacked = device.unpack(deviceData, Path.parse("a.*.a.[b:c1,c:b1].a"));
-  t.is(unpacked.length, 1);
+  assert.strictEqual(unpacked.length, 1);
-  t.is(unpacked[0].toString(), "a.2.a.2.a");
+  assert.strictEqual(unpacked[0].toString(), "a.2.a.2.a");
 });

test/mongodb-functions.ts

@@ -1,132 +0,0 @@
-import ava from "ava";
-import * as mongodbFunctions from "../lib/mongodb-functions";
-import { stringify, parse } from "../lib/common/expression-parser";
-
-ava("convertOldPrecondition", (t) => {
-  const tests = [
-    [{}, "true"],
-    [{ test: "test" }, 'test = "test"'],
-    [{ test: { $eq: "test" } }, 'test = "test"'],
-    [{ test: { $ne: "test" } }, 'test <> "test" OR test IS NULL'],
-    [{ test: { $gte: "test" } }, 'test >= "test"'],
-    [{ "Tags.test": true }, "Tags.test IS NOT NULL"],
-    [{ "Tags.test": false }, "Tags.test IS NULL"],
-    [{ "Tags.test": { $exists: true } }, "Tags.test IS NOT NULL"],
-    [{ "Tags.test": { $exists: false } }, "Tags.test IS NULL"],
-    [{ "Tags.test": { $ne: true } }, "Tags.test IS NULL"],
-    [{ "Tags.test": { $ne: false } }, "Tags.test IS NOT NULL"],
-    [{ "Tags.test": { $eq: true } }, "Tags.test IS NOT NULL"],
-    [{ "Tags.test": { $eq: false } }, "Tags.test IS NULL"],
-    [{ _tags: "test" }, "Tags.test IS NOT NULL"],
-    [{ _tags: { $ne: "test" } }, "Tags.test IS NULL"],
-    [{ _tags: { $eq: "test" } }, "Tags.test IS NOT NULL"],
-    [
-      { $and: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" AND (test <> "test" OR test IS NULL)',
-    ],
-    [{ test: "test", test2: "test2" }, 'test = "test" AND test2 = "test2"'],
-    [
-      { $or: [{ test: "test" }, { test: { $ne: "test" } }] },
-      'test = "test" OR (test <> "test" OR test IS NULL)',
-    ],
-    [
-      { test: { $gte: "test1", $ne: "test2" } },
-      'test >= "test1" AND (test <> "test2" OR test IS NULL)',
-    ],
-    [
-      { test: "test", test2: { $ne: "test2" } },
-      'test = "test" AND (test2 <> "test2" OR test2 IS NULL)',
-    ],
-  ];
-
-  const shouldFailTests = [
-    [{ test: { $gee: "test" } }, "Operator $gee not supported"],
-    [{ test: [] }, "Invalid type"],
-    [{ "Tags.test": { $gt: true } }, "Invalid tag query"],
-    [{ _tags: [] }, "Invalid type"],
-    [{ _tags: { $gt: "test" } }, "Invalid tag query"],
-    [{ $nor: [] }, "Operator $nor not supported"],
-  ];
-
-  t.plan(tests.length + 2 * shouldFailTests.length);
-  for (const test of tests)
-    t.is(stringify(mongodbFunctions.convertOldPrecondition(test[0])), test[1]);
-
-  for (const test of shouldFailTests) {
-    const func = (): void => {
-      mongodbFunctions.convertOldPrecondition(
-        test[0] as Record<string, unknown>
-      );
-    };
-    const error = t.throws(func, { instanceOf: Error });
-    t.is(error.message, test[1]);
-  }
-});
-
-ava("filterToMongoQuery", async (t) => {
-  const queries: [string, Record<string, unknown>][] = [
-    ["true", {}],
-    ["Tags.tag1 = true", { _tags: "tag1" }],
-    ["Tags.tag1 <> false", { _tags: "tag1" }],
-    ["Tags.tag1 IS NULL", { _tags: { $ne: "tag1" } }],
-    ["Tags.tag1 = 123", { "Tags.tag1": 123 }],
-    ["Param1 = 'value1'", { "Param1._value": "value1" }],
-    ["Param1 <> 'value1'", { "Param1._value": { $nin: ["value1", null] } }],
-    ["Param1 > 'value'", { "Param1._value": { $gt: "value" } }],
-    ["Param1 IS NOT NULL", { "Param1._value": { $ne: null } }],
-    ["Param1 LIKE 'value'", { "Param1._value": /^value$/ }],
-    ["LOWER(Param1) LIKE 'value'", { "Param1._value": /^value$/i }],
-    [
-      "Param1 <> 'value2' OR NOT (Param2 = 'value1' OR Param1 < 'value2')",
-      {
-        $or: [
-          { "Param1._value": { $nin: ["value2", null] } },
-          {
-            $and: [
-              { "Param2._value": { $nin: ["value1", null] } },
-              { "Param1._value": { $gte: "value2" } },
-            ],
-          },
-        ],
-      },
-    ],
-    [
-      "Param1 <> 'value2' OR Param1 IS NULL",
-      {
-        $or: [
-          { "Param1._value": { $nin: ["value2", null] } },
-          { "Param1._value": null },
-        ],
-      },
-    ],
-  ];
-
-  for (const [expStr, expect] of queries) {
-    const exp = parse(expStr);
-    const query = mongodbFunctions.filterToMongoQuery(
-      mongodbFunctions.processDeviceFilter(exp)
-    );
-    t.deepEqual(query, expect);
-  }
-
-  const failQueries: [any, string][] = [
-    ["Param1 = Param2", "Invalid RHS operand of = clause"],
-    ["Param1 LIKE Param2", "Invalid RHS operand of LIKE clause"],
-    ["1 LIKE '1'", "Invalid LHS operand of LIKE clause"],
-    ["1 = 2", "Invalid LHS operand of = clause"],
-    ["1 IS NULL", "Invalid LHS operand of IS NULL clause"],
-    ["false", "Primitives are not valid queries"],
-    ["UPPER(Param1) LIKE 'value'", "Invalid RHS operand of LIKE clause"],
-  ];
-
-  for (const [expStr, err] of failQueries) {
-    const exp = parse(expStr);
-    t.throws(
-      () =>
-        mongodbFunctions.filterToMongoQuery(
-          mongodbFunctions.processDeviceFilter(exp)
-        ),
-      { message: err }
-    );
-  }
-});

test/pagination.ts

@@ -0,0 +1,120 @@
+import test from "node:test";
+import assert from "node:assert";
+import initSqlJs from "sql.js/dist/sql-asm.js";
+import { parse, stringify } from "../lib/common/expression/parser.ts";
+import {
+  bookmarkToExpression,
+  paginate,
+  toBookmark,
+} from "../lib/common/expression/pagination.ts";
+import { Expression } from "../lib/types.ts";
+import { covers, minimize } from "../lib/common/expression/synth.ts";
+
+const VALUES = [null, -1, false, "a"];
+const PARAMS = ["param1", "param2"];
+
+let db;
+
+async function query(filter: string): Promise<{ id: string }[]> {
+  if (!db) {
+    const sql = await initSqlJs();
+    db = new sql.Database();
+
+    db.run(`CREATE TABLE test (id INTEGER PRIMARY KEY, ${PARAMS.join(", ")})`);
+
+    const stmt = db.prepare(
+      `INSERT INTO test (${PARAMS.join(", ")}) VALUES (${PARAMS.map(
+        () => "?",
+      ).join(", ")})`,
+    );
+    const count = VALUES.length ** PARAMS.length;
+    for (let i = 0; i < count; ++i) {
+      const values: (boolean | number | string)[] = [];
+      for (let j = 0; j < PARAMS.length; ++j)
+        values.push(VALUES[Math.trunc(i / VALUES.length ** j) % VALUES.length]);
+      stmt.run(values);
+    }
+    stmt.free();
+  }
+
+  const res = db.exec(`SELECT * FROM test WHERE ${filter}`);
+  if (!res.length) return [];
+  return res[0].values.map((row) =>
+    Object.fromEntries(row.map((v, i) => [res[0].columns[i], v])),
+  );
+}
+
+function getAllSortOrders(columns: string[]): Array<Record<string, number>> {
+  const sortOrders: Array<Record<string, number>> = [];
+
+  function generateOrders(
+    remaining: string[],
+    current: Record<string, number>,
+  ): void {
+    if (remaining.length === 0) {
+      sortOrders.push({ ...current });
+      return;
+    }
+
+    for (const column of remaining) {
+      const newRemaining = remaining.filter((c) => c !== column);
+      current[column] = -1;
+      generateOrders(newRemaining, current);
+      current[column] = 1;
+      generateOrders(newRemaining, current);
+      delete current[column];
+    }
+  }
+
+  generateOrders(columns, {});
+  return sortOrders;
+}
+
+async function testPaginate(
+  q1: Expression,
+  q2: Expression,
+  sort: Record<string, number>,
+): Promise<void> {
+  const orderBy = Object.entries(sort)
+    .map(([k, v]) => `${k} ${v > 0 ? "ASC" : "DESC"}`)
+    .join(", ");
+
+  const allMatches = await query(`${stringify(q2)} ORDER BY ${orderBy}`);
+  const [fulfilled, diff] = paginate(q1, q2, sort);
+  assert.ok(covers(q1, fulfilled));
+
+  const fulfilledMatches = await query(
+    `${stringify(fulfilled)} ORDER BY ${orderBy}`,
+  );
+
+  const diffMatches = await query(`${stringify(diff)} ORDER BY ${orderBy}`);
+  assert.deepStrictEqual(allMatches, [...fulfilledMatches, ...diffMatches]);
+
+  if (allMatches.length === fulfilledMatches.length) return;
+
+  const nextMatches = allMatches.slice(
+    fulfilledMatches.length,
+    fulfilledMatches.length + 1,
+  );
+  const bookmark = toBookmark(sort, nextMatches[nextMatches.length - 1]);
+  const bookmarkFilter = bookmarkToExpression(bookmark, sort);
+  const capped = ["AND", q2, bookmarkFilter];
+
+  assert.ok(!covers(q1, capped));
+  const cappedMatches = await query(
+    `(${stringify(capped)}) ORDER BY ${orderBy}`,
+  );
+  assert.deepStrictEqual(cappedMatches, [...fulfilledMatches, ...nextMatches]);
+  const min = minimize(["OR", q1, capped]);
+  await testPaginate(min, q2, sort);
+}
+
+void test("paginate", async () => {
+  const cases: [string, string][] = [["param2 < 'a'", "param1 >= 'a'"]];
+  const params = ["id", ...PARAMS];
+  const sortOrders = getAllSortOrders(params);
+
+  for (const [q1, q2] of cases)
+    for (const sort of sortOrders)
+      await testPaginate(parse(q1), parse(q2), sort);
+});

test/path-set.ts

@@ -1,34 +1,35 @@
-import ava from "ava";
+import test from "node:test";
-import Path from "../lib/common/path";
+import assert from "node:assert";
-import PathSet from "../lib/common/path-set";
+import Path from "../lib/common/path.ts";
+import PathSet from "../lib/common/path-set.ts";
 
-ava("depth", (t) => {
+void test("depth", () => {
   const pathSet = new PathSet();
-  t.is(pathSet.depth, 0);
+  assert.strictEqual(pathSet.depth, 0);
   pathSet.add(Path.parse(""));
-  t.is(pathSet.depth, 1);
+  assert.strictEqual(pathSet.depth, 1);
   pathSet.add(Path.parse("a"));
-  t.is(pathSet.depth, 2);
+  assert.strictEqual(pathSet.depth, 2);
 });
 
-ava("add", (t) => {
+void test("add", () => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a"));
   pathSet.add(Path.parse("a"));
-  t.is(pathSet.find(Path.parse("a"), true, true, 99).length, 1);
+  assert.strictEqual(pathSet.find(Path.parse("a"), true, true, 99).length, 1);
 });
 
-ava("get", (t) => {
+void test("get", () => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a.*"));
   pathSet.add(Path.parse("a.a"));
   pathSet.add(Path.parse("*.*"));
 
-  t.is(pathSet.get(Path.parse("a.*")).toString(), "a.*");
+  assert.strictEqual(pathSet.get(Path.parse("a.*")).toString(), "a.*");
-  t.is(pathSet.get(Path.parse("*.a")), null);
+  assert.strictEqual(pathSet.get(Path.parse("*.a")), null);
 });
 
-ava("find", (t) => {
+void test("find", () => {
   const pathSet = new PathSet();
   pathSet.add(Path.parse("a"));
   pathSet.add(Path.parse("a.*"));
@@ -36,33 +37,33 @@ ava("find", (t) => {
   pathSet.add(Path.parse("*.a"));
   pathSet.add(Path.parse("*.*"));
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse(""), true, true, 1).map((p) => p.toString()),
-    ["a"]
+    ["a"],
   );
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse(""), false, false, 2).map((p) => p.toString()),
-    ["a", "a.*", "a.a", "*.a", "*.*"]
+    ["a", "a.*", "a.a", "*.a", "*.*"],
   );
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse("a.*"), false, false).map((p) => p.toString()),
-    ["a.*"]
+    ["a.*"],
   );
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse("a.*"), false, true).map((p) => p.toString()),
-    ["a.*", "a.a"]
+    ["a.*", "a.a"],
   );
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse("a.*"), true, false).map((p) => p.toString()),
-    ["a.*", "*.*"]
+    ["a.*", "*.*"],
   );
 
-  t.deepEqual(
+  assert.deepStrictEqual(
     pathSet.find(Path.parse("a.*"), true, true).map((p) => p.toString()),
-    ["a.*", "a.a", "*.a", "*.*"]
+    ["a.*", "a.a", "*.a", "*.*"],
   );
 });

test/path.ts

@@ -1,25 +1,45 @@
-import ava from "ava";
+import test from "node:test";
-import Path from "../lib/common/path";
+import assert from "node:assert";
+import Path from "../lib/common/path.ts";
 
-ava("toString", (t) => {
+void test("parse", () => {
+  assert.throws(() => Path.parse("."));
+  assert.throws(() => Path.parse("a "));
+  assert.throws(() => Path.parse(".a"));
+  assert.throws(() => Path.parse("a."));
+  assert.throws(() => Path.parse("a..b"));
+  assert.throws(() => Path.parse("b*"));
+  assert.throws(() => Path.parse("*b"));
+  assert.throws(() => Path.parse("a.b c.d"));
+  assert.throws(() => Path.parse("a["));
+  assert.throws(() => Path.parse("a[b"));
+  assert.throws(() => Path.parse("a[b:"));
+  assert.throws(() => Path.parse('a[b:"waef]'));
+  assert.doesNotThrow(() => Path.parse("[]"));
+  assert.doesNotThrow(() => Path.parse("[a:]"));
+  assert.doesNotThrow(() => Path.parse("*"));
+  assert.doesNotThrow(() => Path.parse(""));
+});
+
+void test("toString", () => {
   const path1 = Path.parse('abc.[ abc : 123 , 123 : abc , 123: " abc "].123');
   const path2 = Path.parse('abc.[123:  " abc ",abc:123,123:abc].123');
-  t.is(path1.toString(), path2.toString());
+  assert.strictEqual(path1.toString(), path2.toString());
 });
 
-ava("slice", (t) => {
+void test("slice", () => {
   const path = Path.parse("a.*.b.[x:y].c");
   const sliced = path.slice(1, -1);
-  t.is(sliced.toString(), '*.b.[x:"y"]');
+  assert.strictEqual(sliced.toString(), '*.b.[x:"y"]');
-  t.is(sliced.alias, 0b100);
+  assert.strictEqual(sliced.alias, 0b100);
-  t.is(sliced.wildcard, 0b1);
+  assert.strictEqual(sliced.wildcard, 0b1);
 });
 
-ava("concat", (t) => {
+void test("concat", () => {
   const path1 = Path.parse("a");
   const path2 = Path.parse("*.[a:b]");
   const concat = path1.concat(path2);
-  t.is(concat.toString(), 'a.*.[a:"b"]');
+  assert.strictEqual(concat.toString(), 'a.*.[a:"b"]');
-  t.is(concat.alias, 0b100);
+  assert.strictEqual(concat.alias, 0b100);
-  t.is(concat.wildcard, 0b10);
+  assert.strictEqual(concat.wildcard, 0b10);
 });

test/ping.ts

@@ -0,0 +1,37 @@
+import test from "node:test";
+import assert from "node:assert";
+import { parsePing } from "../lib/ping.ts";
+
+void test("linux Case-1", () => {
+  const platform = "linux";
+  const stdout =
+    "PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data\n64 bytes from 8.8.8.8: icmp_seq=1 ttl=118 time=6.66 ms\n64 bytes from 8.8.8.8: icmp_seq=2 ttl=118 time=6.50 ms\n64 bytes from 8.8.8.8: icmp_seq=3 ttl=118 time=6.38 ms\n\n--- 8.8.8.8 ping statistics ---\n3 packets transmitted, 3 received, 0% packet loss, time 402ms\nrtt min/avg/max/mdev = 6.381/6.511/6.656/0.112 ms";
+  const parsedResult = {
+    packetsTransmitted: 3,
+    packetsReceived: 3,
+    packetLoss: 0,
+    min: 6.381,
+    avg: 6.511,
+    max: 6.656,
+    mdev: 0.112,
+  };
+  const parsed = parsePing(platform, stdout);
+  assert.deepStrictEqual(parsedResult, parsed);
+});
+
+void test("linux Case-2", () => {
+  const platform = "linux";
+  const stdout =
+    "PING 10.251.9.108 (10.251.9.108): 56 data bytes\n64 bytes from 10.251.9.108: icmp_seq=0 ttl=57 time=36.758 ms\n64 bytes from 10.251.9.108: icmp_seq=1 ttl=57 time=69.094 ms\n64 bytes from 10.251.9.108: icmp_seq=2 ttl=57 time=28.868 ms\n--- 10.251.9.108 ping statistics ---3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max/stddev = 28.868/44.907/69.094/17.404 ms";
+  const parsedResult = {
+    packetsTransmitted: 3,
+    packetsReceived: 3,
+    packetLoss: 0,
+    min: 28.868,
+    avg: 44.907,
+    max: 69.094,
+    mdev: 17.404,
+  };
+  const parsed = parsePing(platform, stdout);
+  assert.deepStrictEqual(parsedResult, parsed);
+});

test/synth.ts

@@ -0,0 +1,117 @@
+import test from "node:test";
+import assert from "node:assert";
+import initSqlJs from "sql.js/dist/sql-asm.js";
+import { minimize, unionDiff } from "../lib/common/expression/synth.ts";
+import { parse, stringify } from "../lib/common/expression/parser.ts";
+
+const STRING_VALUES = [null, "", "a", "ab", "ab10", "ab-10"];
+const DECIMAL_VALUES = [null, 0, -10, 10];
+
+let db;
+
+async function query(filter: string): Promise<Set<number>> {
+  if (!db) {
+    const sql = await initSqlJs();
+    db = new sql.Database();
+
+    db.run(
+      "CREATE TABLE test (id INTEGER PRIMARY KEY, string STRING, decimal DECIMAL(4,2))",
+    );
+
+    const stmt = db.prepare("INSERT INTO test (string, decimal) VALUES (?, ?)");
+    const count = STRING_VALUES.length * DECIMAL_VALUES.length;
+    for (let i = 0; i < count; ++i) {
+      const str = i % STRING_VALUES.length;
+      const dec = Math.trunc(i / STRING_VALUES.length) % DECIMAL_VALUES.length;
+      stmt.run([STRING_VALUES[str], DECIMAL_VALUES[dec]]);
+    }
+    stmt.free();
+  }
+
+  const res = db.exec(`SELECT id FROM test WHERE ${filter}`);
+  if (!res.length) return new Set();
+  return new Set(res[0].values.flat());
+}
+
+function setsEqual(set1: Set<number>, set2: Set<number>): boolean {
+  if (set1.size !== set2.size) return false;
+  for (const s of set1) if (!set2.has(s)) return false;
+  return true;
+}
+
+function getPermutations(...arrs: any[][]): any[][] {
+  const count = arrs.reduce((total, arr) => total * arr.length, 1);
+  const res = [];
+  for (let i = 0; i < count; ++i) {
+    let j = i;
+    const row = [];
+    for (const arr of arrs) {
+      const v = arr[j % arr.length];
+      j = Math.trunc(j / arr.length);
+      row.push(v);
+    }
+    res.push(row);
+  }
+  return res;
+}
+
+void test("minimize", async () => {
+  const cases: string[] = [];
+
+  cases.push("null");
+  cases.push("false");
+  cases.push("true");
+  cases.push("string");
+  cases.push("(string + decimal) IS NULL");
+  cases.push("(string + decimal) = NULL");
+  cases.push("COALESCE(string, decimal) = 0");
+
+  for (const [s1, s2, s3, op1, op2] of getPermutations(
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s),
+    [">", "=", "<"],
+    ["<>", ">="],
+  )) {
+    cases.push(
+      `string ${op1} "${s1}" OR string ='${s2}' OR NOT string ${op2} '${s3}'`,
+    );
+  }
+
+  for (const [s1, s2] of getPermutations(
+    STRING_VALUES.filter((s) => s),
+    STRING_VALUES.filter((s) => s),
+  ))
+    cases.push(`string > "${s1}" AND string < '${s2}'`);
+
+  for (const c of cases) {
+    const res1 = await query(c);
+    const min = stringify(minimize(parse(c), true));
+    const res2 = await query(min);
+    assert.strictEqual(setsEqual(res1, res2), true);
+  }
+});
+
+void test("unionDiff", async () => {
+  const cases = [
+    "true",
+    "decimal > 0",
+    "decimal > 10",
+    "UPPER(string || decimal) LIKE 'AB10'",
+    "COALESCE(string, decimal) = 0",
+  ];
+
+  for (const [c1, c2] of getPermutations(cases, cases)) {
+    const res1 = await query(c1);
+    const res2 = await query(c2);
+    const [union, diff] = unionDiff(parse(c1), parse(c2));
+    const res3 = await query(stringify(union));
+    const res4 = await query(stringify(diff));
+
+    const unionSet = new Set([...res1, ...res2]);
+    const diffSet = new Set(Array.from(res2).filter((r) => !res1.has(r)));
+
+    assert.strictEqual(setsEqual(res3, unionSet), true);
+    assert.strictEqual(setsEqual(res4, diffSet), true);
+  }
+});

test/util.ts

@@ -1,7 +1,8 @@
-import ava from "ava";
+import test from "node:test";
-import * as common from "../lib/common";
+import assert from "node:assert";
+import * as common from "../lib/util.ts";
 
-ava("generateDeviceId", (t) => {
+void test("generateDeviceId", () => {
   const space = [" ", "%20"];
   const special = [";", "%3B"];
   const cases = [
@@ -28,14 +29,17 @@ ava("generateDeviceId", (t) => {
       `TestOUIWith${space[1]}_${special[1]}2912-TestSerialNumberWith${space[1]}_${special[1]}2912`,
     ],
   ];
-  t.plan(cases.length);
-  for (const c of cases)
-    t.is(common.generateDeviceId(c[0] as Record<string, string>), c[1]);
-});
 
-ava("escapeRegExp", (t) => {
+  for (const c of cases)
-  t.is(
+    assert.strictEqual(
+      common.generateDeviceId(c[0] as Record<string, string>),
+      c[1],
+    );
+});
+
+void test("escapeRegExp", () => {
+  assert.strictEqual(
     common.escapeRegExp("\\ ^ $ * + ? . ( ) | { } [ ]"),
-    "\\\\ \\^ \\$ \\* \\+ \\? \\. \\( \\) \\| \\{ \\} \\[ \\]"
+    "\\\\ \\^ \\$ \\* \\+ \\? \\. \\( \\) \\| \\{ \\} \\[ \\]",
   );
 });

test/xml-parser.ts

@@ -1,16 +1,17 @@
-import ava from "ava";
+import test from "node:test";
+import assert from "node:assert";
 import {
   parseXmlDeclaration,
   decodeEntities,
   parseXml,
-} from "../lib/xml-parser";
+} from "../lib/xml-parser.ts";
 
-ava("parseXmlDeclaration", (t) => {
+void test("parseXmlDeclaration", () => {
   const buf = Buffer.from(
-    '<?xml version="1.0" encoding="UTF-8"?>\n<soap-env:Envelope />'
+    '<?xml version="1.0" encoding="UTF-8"?>\n<soap-env:Envelope />',
   );
   const attrs = parseXmlDeclaration(buf);
-  t.deepEqual(attrs, [
+  assert.deepStrictEqual(attrs, [
     {
       name: "version",
       namespace: "",
@@ -26,18 +27,18 @@ ava("parseXmlDeclaration", (t) => {
   ]);
 });
 
-ava("decodeEntities", (t) => {
+void test("decodeEntities", () => {
-  t.is(
+  assert.strictEqual(
     decodeEntities("&&amp;&lt;&gt;&quot;&apos;&gt;&#167;&#xd842;&#xDFB7;;"),
-    "&&<>\"'>§𠮷;"
+    "&&<>\"'>§𠮷;",
   );
 });
 
-ava("parse", (t) => {
+void test("parse", () => {
   const xml =
     '<?xml version="1.0"?>\n<a-b:c><d f="1<g>"/><!-- comment --><h >i</h></a-b:c>';
   const parsed = parseXml(xml);
-  t.deepEqual(parsed, {
+  assert.deepStrictEqual(parsed, {
     name: "root",
     namespace: "",
     localName: "root",

test/yaml-tests.json

@@ -0,0 +1,861 @@
+[
+  [
+    {
+      "name": "Mark McGwire",
+      "hr": 65,
+      "avg": 0.278
+    },
+    {
+      "name": "Sammy Sosa",
+      "hr": 63,
+      "avg": 0.288
+    }
+  ],
+  {
+    "top1": {
+      "key1": "scalar1"
+    },
+    "top2": {
+      "key2": "scalar2"
+    },
+    "top3": {
+      "scalar1": "scalar3"
+    },
+    "top4": {
+      "scalar2": "scalar4"
+    },
+    "top5": "scalar5",
+    "top6": {
+      "key6": "scalar6"
+    }
+  },
+  "text",
+  ["a", "b", 42, "d"],
+  {
+    "a!\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~": "safe",
+    "?foo": "safe question mark",
+    ":foo": "safe colon",
+    "-foo": "safe dash",
+    "this is#not": "a comment"
+  },
+  {
+    "": "a"
+  },
+  "foo",
+  {
+    "key": "value",
+    "foo": "key"
+  },
+  [1, -2, 33],
+  {
+    "plain": "a b\nc"
+  },
+  [["s1_i1", "s1_i2"], "s2"],
+  {
+    "First occurrence": "Foo",
+    "Second occurrence": "Foo",
+    "Override anchor": "Bar",
+    "Reuse anchor": "Bar"
+  },
+  "k:#foo &a !t s",
+  ["a"],
+  {
+    "escaped slash": "a/b"
+  },
+  {
+    "plain": "This unquoted scalar spans many lines.",
+    "quoted": "So does this quoted scalar.\n"
+  },
+  "here's to \"quotes\"",
+  {
+    "foo": "bar"
+  },
+  "ab cd\nef\n\ngh\n",
+  ["detected\n", "\n\n# detected\n", " explicit\n", "detected\n"],
+  "foo: bar\": baz",
+  "plain\\value\\with\\backslashes",
+  {
+    "plain": "text lines",
+    "quoted": "text lines",
+    "block": "text\n \tlines\n"
+  },
+  "a",
+  {
+    "foo": "you",
+    "bar": "far"
+  },
+  {
+    "sequence": ["entry", ["nested"]],
+    "mapping": {
+      "foo": "bar"
+    }
+  },
+  {
+    "literal": "some\ntext\n",
+    "folded": "some text\n"
+  },
+  [
+    {
+      "one": "two",
+      "three": "four"
+    },
+    {
+      "five": "six",
+      "seven": "eight"
+    }
+  ],
+  {
+    "Folding": "Empty line\nas a line feed",
+    "Chomping": "Clipped empty lines\n"
+  },
+  [
+    ["one", "two"],
+    ["three", "four"]
+  ],
+  {
+    "foo": "bar"
+  },
+  {
+    "key": "value"
+  },
+  {
+    "explicit key": null,
+    "block key\n": ["one", "two"]
+  },
+  ["foo"],
+  [
+    {
+      "foo": "bar"
+    },
+    ["baz", "baz"]
+  ],
+  "ab\n\n \n",
+  {
+    "foo: bar\\": "baz'"
+  },
+  {
+    "Not indented": {
+      "By one space": "By four\n  spaces\n",
+      "Flow style": ["By two", "Also by two", "Still by two"]
+    }
+  },
+  "\\//||\\/||\n// ||  ||__\n",
+  {
+    "foo": [
+      "a",
+      {
+        "key": "value"
+      }
+    ]
+  },
+  {
+    "a": null,
+    "b": null
+  },
+  "foo",
+  {
+    "a": "b",
+    "": "a"
+  },
+  {
+    "foo\nbar:baz\tx \\$%^&*()x": 23,
+    "x\\ny:z\\tx $%^&*()x": 24
+  },
+  "Sammy Sosa completed another fine season with great stats.\n\n  63 Home Runs\n  0.288 Batting Average\n\nWhat a year!\n",
+  " foo\nbar\nbaz ",
+  { "matches %": 20 },
+  [
+    "flow in block",
+    "Block scalar\n",
+    {
+      "foo": "bar"
+    }
+  ],
+  {
+    "a": "b",
+    "c": 42,
+    "e": "f",
+    "g": "h",
+    "23": false
+  },
+  "ab",
+  " 1st non-empty\n2nd non-empty 3rd non-empty ",
+  {
+    "top1": {
+      "key1": "one"
+    },
+    "top2": {
+      "key2": "two"
+    },
+    "top3": {
+      "key3": "three"
+    },
+    "top4": {
+      "key4": "four"
+    },
+    "top5": {
+      "key5": "five"
+    },
+    "top6": "six",
+    "top7": "seven"
+  },
+  {
+    "hr": ["Mark McGwire", "Sammy Sosa"],
+    "rbi": ["Sammy Sosa", "Ken Griffey"]
+  },
+  "\nfolded line\nnext line\n  * bullet\n\n  * list\n  * lines\n\nlast line\n",
+  ["word1", "word2"],
+  {
+    "a": null,
+    "b": null,
+    "c": null
+  },
+  {
+    "nested sequences": [[[[]]], [[{}]]],
+    "key1": [],
+    "key2": {}
+  },
+  "---word1 word2",
+  {
+    "implicit block key": [
+      {
+        "implicit flow key": "value"
+      }
+    ]
+  },
+  {
+    "key ends with two colons::": "value"
+  },
+  [
+    {
+      "single line": null,
+      "a": "b"
+    },
+    {
+      "multi line": null,
+      "a": "b"
+    }
+  ],
+  "a",
+  {
+    "key": ["item1", "item2"]
+  },
+  [
+    "double quoted",
+    "single quoted",
+    "plain text",
+    ["nested"],
+    {
+      "single": "pair"
+    }
+  ],
+  ["unicode anchor"],
+  [
+    {
+      "key": "value",
+      "key2": "value2"
+    },
+    {
+      "key3": "value3"
+    }
+  ],
+  "trimmed\n\n\nas space",
+  "Mark McGwire's year was crippled by a knee injury.\n",
+  [
+    {
+      "single line": null,
+      "a": "b"
+    },
+    {
+      "multi line": null,
+      "a": "b"
+    }
+  ],
+  {
+    "a": {
+      "b": {
+        "c": "d"
+      },
+      "e": {
+        "f": "g"
+      }
+    },
+    "h": "i"
+  },
+  {
+    "foo": {
+      "bar": "baz"
+    }
+  },
+  [
+    {
+      "single line": "value"
+    },
+    {
+      "multi line": "value"
+    }
+  ],
+  {
+    "single": "text",
+    "double": "text"
+  },
+  " 1st non-empty\n2nd non-empty 3rd non-empty ",
+  [
+    {
+      "item": "Super Hoop",
+      "quantity": 1
+    },
+    {
+      "item": "Basketball",
+      "quantity": 4
+    },
+    {
+      "item": "Big Shoes",
+      "quantity": 1
+    }
+  ],
+  "a b c d\ne",
+  {
+    "a": ["b", ["c", "d"]]
+  },
+  {
+    "strip": "text",
+    "clip": "text\n",
+    "keep": "text\n"
+  },
+  {
+    "a": "b c",
+    "d": "e f"
+  },
+  ["single multiline - sequence entry"],
+  {
+    "1": [2, 3],
+    "4": 5
+  },
+  [
+    {
+      "bla\"keks": "foo"
+    },
+    {
+      "bla]keks": "foo"
+    }
+  ],
+  "folded text\n",
+  "foo",
+  {
+    "key": {
+      "a": "b"
+    }
+  },
+  {
+    "adjacent": "value",
+    "readable": "value",
+    "empty": null
+  },
+  [
+    {
+      "a": "b"
+    },
+    {
+      "c": "d"
+    },
+    {
+      "e": "f"
+    },
+    {
+      "g": "h"
+    }
+  ],
+  {
+    "tab": "\tstring"
+  },
+  [
+    {
+      "foo bar": "baz"
+    }
+  ],
+  {
+    "anchored": "value",
+    "alias": "value"
+  },
+  ["explicit indent and chomp", "chomp and explicit indent"],
+  {
+    "a": ["b", "c"]
+  },
+  {
+    "foo": "bar"
+  },
+  [
+    "::vector",
+    ": - ()",
+    "Up, up, and away!",
+    -123,
+    "http://example.com/foo#bar",
+    [
+      "::vector",
+      ": - ()",
+      "Up, up and away!",
+      -123,
+      "http://example.com/foo#bar"
+    ]
+  ],
+  {
+    "a": "b",
+    "seq": ["a"],
+    "c": "d"
+  },
+  ["foo", "bar", 42],
+  "line1 # no comment line3\n",
+  "\n\nliteral\n \n\ntext\n",
+  {
+    "a": "b"
+  },
+  {
+    "k": ["a", "b"]
+  },
+  "a b c d\ne",
+  "---word1 word2",
+  ["a", 2, 4, "d"],
+  {
+    "a": [
+      "b",
+      "c",
+      {
+        "d": ["e", "f"]
+      }
+    ]
+  },
+  {
+    "a": " more indented\nregular\n",
+    "b": "\n\n more indented\nregular\n"
+  },
+  {
+    "strip": "# text",
+    "clip": "# text\n",
+    "keep": "# text\n\n"
+  },
+  {
+    "safe": "a!\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~ !\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~",
+    "safe question mark": "?foo",
+    "safe colon": ":foo",
+    "safe dash": "-foo"
+  },
+  "line1 line2 line3\n",
+  ["Mark McGwire", "Sammy Sosa", "Ken Griffey"],
+  ["a"],
+  ["a", ["b", "c"]],
+  {
+    "unicode": "Sosa did fine.☺",
+    "control": "\b1998\t1999\t2000\n",
+    "hex esc": "\r\n is \r\n",
+    "single": "\"Howdy!\" he cried.",
+    "quoted": " # Not a 'comment'.",
+    "tie-fighter": "|\\-*-/|"
+  },
+  [["-", "-"]],
+  "folded text\n",
+  {
+    "a": 13,
+    "1.5": "d"
+  },
+  {
+    "foo": 1,
+    "bar": 2,
+    "text": "a\n  \nb\n\nc\n\nd\n"
+  },
+  {
+    "wanted": "love ♥ and peace ☮"
+  },
+  {
+    "name": "Mark McGwire",
+    "accomplishment": "Mark set a major league home run record in 1998.\n",
+    "stats": "65 Home Runs\n0.278 Batting Average\n"
+  },
+  {
+    "foo": "bar",
+    "baz": "foo"
+  },
+  "1st non-empty\n2nd non-empty 3rd non-empty",
+  {
+    "quoted": "Quoted \t",
+    "block": "void main() {\n\tprintf(\"Hello, world!\\n\");\n}\n"
+  },
+  {
+    "foo": "blue",
+    "bar": "arrr",
+    "baz": "jazz"
+  },
+  [
+    {
+      "Mark McGwire": 65
+    },
+    {
+      "Sammy Sosa": 63
+    },
+    {
+      "Ken Griffy": 58
+    }
+  ],
+  {
+    "1": 2,
+    "3": 4
+  },
+  {
+    "hr": ["Mark McGwire", "Sammy Sosa"],
+    "rbi": ["Sammy Sosa", "Ken Griffey"]
+  },
+  "k:#foo &a !t s",
+  {
+    "block sequence": [
+      "one",
+      {
+        "two": "three"
+      }
+    ]
+  },
+  {
+    "First occurrence": "Value",
+    "Second occurrence": "Value"
+  },
+  {
+    "a true": "null d",
+    "e 42": null
+  },
+  {
+    "foo": "bar"
+  },
+  ["foo", "bar", 42],
+  "trimmed\n\n\nas space",
+  "scalar",
+  {
+    "strip": "",
+    "clip": "",
+    "keep": "\n"
+  },
+  {
+    "foo": {
+      "bar": 1
+    },
+    "baz": 2
+  },
+  {
+    "a": 47,
+    "c": "d"
+  },
+  {
+    "implicit block key": [
+      {
+        "implicit flow key": "value"
+      }
+    ]
+  },
+  ["a", "b", "c", "c", ""],
+  [
+    ["a", "b", "c"],
+    {
+      "a": "b",
+      "c": "d",
+      "e": "f"
+    },
+    []
+  ],
+  {
+    "implicit block key": [
+      {
+        "implicit flow key": "value"
+      }
+    ]
+  },
+  {
+    "a": "ab\n\ncd\nef\n"
+  },
+  {
+    "literal": "value\n",
+    "folded": "value\n"
+  },
+  {
+    "a": [
+      "b",
+      "c",
+      {
+        "d": ["e", "f"]
+      }
+    ]
+  },
+  "literal\n\ttext\n",
+  "foo \n\n\t bar\n\nbaz\n",
+  [
+    {
+      "a": "b"
+    }
+  ],
+  "ab",
+  ["plain", "double quoted", "single quoted", "block\n", "plain again"],
+  {
+    "a": " ",
+    "b": " ",
+    "c": " ",
+    "d": " ",
+    "e": "\n",
+    "f": "\n",
+    "g": "\n\n",
+    "h": "\n\n"
+  },
+  {
+    "key": "value with\ntabs"
+  },
+  {
+    "": null
+  },
+  [
+    {
+      "single line": "value"
+    },
+    {
+      "multi line": "value"
+    }
+  ],
+  "folded to a space,\nto a line feed, or \t \tnon-content",
+  ["literal\n", " folded\n", "keep\n\n", " strip"],
+  {
+    "key": "value"
+  },
+  {
+    "american": ["Boston Red Sox", "Detroit Tigers", "New York Yankees"],
+    "national": ["New York Mets", "Chicago Cubs", "Atlanta Braves"]
+  },
+  " 1st non-empty\n2nd non-empty 3rd non-empty ",
+  {},
+  [
+    ["a", "b"],
+    {
+      "a": "b"
+    },
+    "a",
+    "b",
+    "c"
+  ],
+  "folded to a space,\nto a line feed, or \t \tnon-content",
+  [
+    {
+      "foo": "bar"
+    }
+  ],
+  ["detected\n", "\n\n# detected\n", " explicit\n", "\t\ndetected\n"],
+  {
+    "top1": [
+      "item1",
+      {
+        "key2": "value2"
+      },
+      "item3"
+    ],
+    "top2": "value2"
+  },
+  {
+    "foo": [42],
+    "bar": [44]
+  },
+  {
+    "23": "d",
+    "a": 4.2
+  },
+  "Document",
+  {
+    "Date": "2001-11-23 15:03:17 -5",
+    "User": "ed",
+    "Fatal": "Unknown variable \"bar\"",
+    "Stack": [
+      {
+        "file": "TopClass.py",
+        "line": 23,
+        "code": "x = MoreObject(\"345\\n\")\n"
+      },
+      { "file": "MoreClass.py", "line": 58, "code": "foo = bar" }
+    ]
+  },
+  {
+    "plain key": "in-line value",
+    "": null,
+    "quoted key": ["entry"]
+  },
+  ["12", 12, "12"],
+  {
+    "aaa": "bbb"
+  },
+  [":,"],
+  {
+    "sequence": ["one", "two"],
+    "mapping": {
+      "sky": "blue",
+      "sea": "green"
+    }
+  },
+  {
+    "seq": ["a", "b"]
+  },
+  "here's to \"quotes\"",
+  {
+    "hr": 65,
+    "avg": 0.278,
+    "rbi": 147
+  },
+  "\n\nliteral\n \n\ntext\n",
+  " 1st non-empty\n2nd non-empty 3rd non-empty ",
+  "literal\n\ttext\n",
+  {
+    "block mapping": {
+      "key": "value"
+    }
+  },
+  " foo\nbar\nbaz ",
+  "ab cd\nef\n\ngh\n",
+  "foo",
+  {
+    "top1": {
+      "key1": "one"
+    },
+    "top2": {
+      "key2": "two"
+    },
+    "top3": {
+      "key3": "three"
+    },
+    "top4": {
+      "key4": "four"
+    },
+    "top5": {
+      "key5": "five"
+    },
+    "top6": "six",
+    "top7": "seven"
+  },
+  [
+    {
+      "url": "http://example.org"
+    }
+  ],
+  {
+    "sequence": ["one", "two"],
+    "mapping": {
+      "sky": "blue",
+      "sea": "green"
+    }
+  },
+  {
+    "invoice": 34843,
+    "date": "2001-01-23",
+    "bill-to": {
+      "given": "Chris",
+      "family": "Dumars",
+      "address": {
+        "lines": "458 Walkman Dr.\nSuite #292\n",
+        "city": "Royal Oak",
+        "state": "MI",
+        "postal": 48046
+      }
+    },
+    "ship-to": {
+      "given": "Chris",
+      "family": "Dumars",
+      "address": {
+        "lines": "458 Walkman Dr.\nSuite #292\n",
+        "city": "Royal Oak",
+        "state": "MI",
+        "postal": 48046
+      }
+    },
+    "product": [
+      {
+        "sku": "BL394D",
+        "quantity": 4,
+        "description": "Basketball",
+        "price": 450
+      },
+      {
+        "sku": "BL4438H",
+        "quantity": 1,
+        "description": "Super Hoop",
+        "price": 2392
+      }
+    ],
+    "tax": 251.42,
+    "total": 4443.52,
+    "comments": "Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338."
+  },
+  ["a", "b", "a", "b"],
+  [
+    null,
+    "block node\n",
+    ["one", "two"],
+    {
+      "one": "two"
+    }
+  ],
+  {
+    "a": "scalar a",
+    "b": "scalar a"
+  },
+  {
+    "foo": "",
+    "": "bar"
+  },
+  {
+    "key": "value"
+  },
+  "scalar %YAML 1.2",
+  {
+    "Folding": "Empty line\nas a line feed",
+    "Chomping": "Clipped empty lines\n"
+  },
+  {
+    "key": "value"
+  },
+  [
+    ["name", "hr", "avg"],
+    ["Mark McGwire", 65, 0.278],
+    ["Sammy Sosa", 63, 0.288]
+  ],
+  {
+    "literal": "value\n",
+    "folded": "value\n"
+  },
+  {
+    "Mark McGwire": {
+      "hr": 65,
+      "avg": 0.278
+    },
+    "Sammy Sosa": {
+      "hr": 63,
+      "avg": 0.288
+    }
+  },
+  {
+    "a": "b",
+    "c": "d"
+  },
+  {
+    "key": [[["value"]]]
+  },
+  {
+    "a": 1,
+    "b": null,
+    "c": 3
+  },
+  {
+    "event": "outgoing HTTP response",
+    "timestamp": "2021-01-21T18:44:57.116Z",
+    "remoteAddress": "127.0.0.1",
+    "deviceId": "202BC1-BM632w-000000",
+    "connection": "2021-01-21T18:44:57.105Z",
+    "statusCode": 200,
+    "headers": {
+      "content-length": 528,
+      "server": "GenieACS/1.2.3+20210121000910",
+      "soapserver": "GenieACS/1.2.3+20210121000910",
+      "content-type": "text/xml; charset=\"utf-8\"",
+      "set-cookie": "session=ea31d97ccf6832fb"
+    },
+    "body": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<soap-env:Envelope xmlns:soap-enc=\"http://schemas.xmlsoap.org/soap/encoding/\" xmlns:soap-env=\"http://schemas.xmlsoap.org/soap/envelope/\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:cwmp=\"urn:dslforum-org:cwmp-1-0\"><soap-env:Header><cwmp:ID soap-env:mustUnderstand=\"1\">wdsx50vq</cwmp:ID></soap-env:Header><soap-env:Body><cwmp:InformResponse><MaxEnvelopes>1</MaxEnvelopes></cwmp:InformResponse></soap-env:Body></soap-env:Envelope>"
+  }
+]