1.2.16

This reverts commit f9ee39dfd0.

.eslintrc.json

@@ -0,0 +1,36 @@
+{
+  "env": {
+    "es2022": true,
+    "node": true
+  },
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "project": "./tsconfig.json"
+  },
+  "plugins": ["@typescript-eslint"],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
+  "rules": {
+    "no-shadow": ["error", { "allow": ["err"] }],
+    "handle-callback-err": "error",
+    "prefer-arrow-callback": "error",
+    "no-buffer-constructor": "error",
+    "prefer-const": ["error", { "destructuring": "all" }],
+    "eqeqeq": ["error", "always", { "null": "ignore" }],
+    "@typescript-eslint/no-explicit-any": 0,
+    "@typescript-eslint/no-use-before-define": [
+      "error",
+      { "functions": false }
+    ],
+    "@typescript-eslint/explicit-function-return-type": [
+      "error",
+      { "allowExpressions": true }
+    ],
+    "@typescript-eslint/no-floating-promises": "error",
+    "@typescript-eslint/no-misused-promises": "error",
+    "no-prototype-builtins": 0
+  }
+}

.gitignore

@@ -1,27 +1,4 @@
 *~
-lib/*.map
-lib/api-functions.js
-lib/auth.js
-lib/common.js
-lib/config.js
-lib/cwmp.js
-lib/db.js
-lib/fs.js
-lib/nbi.js
-lib/query.js
-lib/server.js
-lib/session.js
 node_modules
-debug
-config/auth.js
-config/config.json
-config/cwmp.key
-config/cwmp.crt
-config/cwmp.ca-bundle
-config/nbi.key
-config/nbi.crt
-config/nbi.ca-bundle
-config/fs.key
-config/fs.crt
-config/fs.ca-bundle
-config/ext
+dist
+docs/_build

.npmignore

@@ -1,21 +0,0 @@
-*~
-.npmignore
-
-config/auth.js
-config/config.json
-config/cwmp.key
-config/cwmp.crt
-config/cwmp.ca-bundle
-config/nbi.key
-config/nbi.crt
-config/nbi.ca-bundle
-config/fs.key
-config/fs.crt
-config/fs.ca-bundle
-config/ext
-
-debug/*
-!debug/.gitkeep
-
-lib/*.coffee
-lib/*.map

.prettierignore

@@ -0,0 +1,2 @@
+dist
+npm-shrinkwrap.json

CHANGELOG.md

@@ -1,70 +1,371 @@
 # Change Log
 
+## 1.2.16 (2026-03-29)
+
+- Fix regression in v1.2.15 that broke the device search component in the UI by
+  pinning Mithril dependency to v2.2.2 due to multiple unresolved regressions in
+  later versions.
+
+## 1.2.15 (2026-03-26)
+
+- Fix critical RCE vulnerability in /api/ping endpoint (follow-up to
+  CVE-2021-46704). Credit: JuHwiSang.
+
+- Fix device IDs containing special characters (e.g., %) not being handled
+  correctly in the UI due to a regression in Mithril 2.3.8.
+
+## 1.2.14 (2026-03-12)
+
+- Prevent UI crash when a malformed URL is sent to the server.
+
+- Fix potential edge-case bugs in expression evaluation and session
+  serialization.
+
+## 1.2.13 (2024-06-06)
+
+- Increase connection timeout for UI and NBI from 30 to 120 seconds to avoid
+  timeouts when running unindexed queries in large deployments.
+
+- Fix race condition causing 503 error when deleting multiple faults at once.
+
+- Fix some UI config options not being evaluated as dynamic expressions.
+
+- Fix an issue where certain edge-case query expressions were not being
+  correctly converted to MongoDB queries, resulting in inaccurate search
+  results.
+
+## 1.2.12 (2024-03-28)
+
+- Fix broken XMPP support in the previous release.
+
+- Fix regression causing CSV downloads to be buffered in memory before being
+  streamed to the client.
+
+## 1.2.11 (2024-03-21)
+
+- Resolved an issue from the previous release that caused incompatibility with
+  Node.js versions 12 through 15.
+
+## 1.2.10 (2024-03-18)
+
+- Add support for XMPP connection requests. Use the environment variables
+  `XMPP_JID` and `XMPP_PASSWORD` to configure the XMPP connection for the ACS.
+
+- The environment variables `CWMP_SSL_CERT` and `CWMP_SSL_KEY`, as well as their
+  counterparts for UI, NBI, and FS, now accept PEM-encoded certificates and keys
+  in addition to file paths.
+
+- The UI no longer requires users to refresh the page after modifying presets,
+  provisions, or virtual parameters. Refreshing is now only necessary for
+  changes to users, permissions, or UI configurations.
+
+- Improved conversion of GenieACS expressions into MongoDB queries for more
+  optimized queries and better index utilization.
+
+- Refactor UI pagination and sorting to fix issues from the previous approach,
+  especially with sorting by rapidly changing parameters such as 'last inform'
+  time.
+
+- The file server now supports HEAD requests, the Range header, and conditional
+  requests.
+
+- Addressed an issue causing file server disconnections for slow clients over
+  HTTPS.
+
+- Fix bug preventing users from changing their own passwords.
+
+- Introduced a new 'locks' collection for database locking, replacing the
+  previous use of the 'cache' collection for this purpose.
+
+- Various other minor fixes and enhancements.
+
+## 1.2.9 (2022-08-22)
+
+- New config option `cwmp.skipRootGpn` to enable a workaround for some
+  problematic CPEs that reject GPN requests on data model root.
+- Stream query results and CSV downloads as data becomes available instead of
+  buffering the entire response.
+- Log HTTP/HTTPS client errors in debug log.
+- Fix occasional lock expired errors after updating presets, etc.
+- Fix bug where queries containing `<>` operator may return incorrect results.
+- Fix a performance hit caused by DB calls containing the entire CPE data model
+  rather then just the updated parameters.
+- Fix bug where tags containing special characters are saved in their encoded
+  form when set via a Provision script.
+- Fix issue causing invalid `xsd:dateTime` values to be saved in DB as `NaN`
+  rather than maintain the original string value.
+- Fix bug using `$regex` operator with a numeric or a datetime string.
+- Fix ping not working on certain platforms.
+- Ping requests are now authenticated.
+- Fix error when the `FORWARDED_HEADER` environment variable contains IPv4 CIDR
+  while the listening interface is IPv6 and vice versa.
+- Fix false warning "unexpected parameter in response" showing in
+  `genieacs-cwmp` logs.
+- Other minor fixes and stability improvements.
+
+## 1.2.8 (2021-10-27)
+
+- Fix a remote code execution security vulnerability in genieacs-ui.
+- All UI components can now be configured using fully dynamic expressions.
+  Previously some component types only accept fixed string values as properties.
+- The `container` UI component can now be configured with an `element` property
+  that's either a string or a nested pair of `tag` and `attributes` properties.
+  The various attributes under the `attributes` property can now make use of a
+  new function `ENCODEURICOMPONENT()` to facilitate creating custom hyperlinks
+  in the UI.
+- Improve sorting buttons' behavior in the device listing page and other pages.
+  Sorting by multiple columns should now feel more intuitive.
+- Support the modulo (%) operator in expressions.
+- Fix a regression in the previous release where the config option 'ui.pageSize'
+  no longer works.
+- Fix process crash when a CPE sends an unsupported ACS method.
+
+## 1.2.7 (2021-09-18)
+
+- Fix regression causing frequent invalid session errors.
+- Fix regression breaking digest authentication in connection requests.
+
+## 1.2.6 (2021-09-16)
+
+- New config option `cwmp.downloadSuccessOnTimeout` to enable a workaround for
+  CPEs that neglect to send a TransferComplete message after a successful
+  download.
+- Display a progress bar when uploading new files.
+- Default to dual stack interface binding (i.e. `::` instead of `0.0.0.0`).
+  Unless the binding interface is explicitly set, this will cause IPv4 addresses
+  in the logs to be displayed as IPv4-mapped IPv6 addresses (e.g.
+  `::ffff:127.0.0.1` instead of `127.0.0.1`).
+- Detect and correct for client side (browser) clock skew that would otherwise
+  alter the numbers in the pie charts.
+- Various improvements relating to dealing with buggy TR-069 client
+  implementations.
+- Fix a bug causing "lock expired" exceptions when a CWMP session remains open
+  for a very long time due to slow clients.
+- Fix metadata of uploaded files going missing due to nginx stripping away what
+  it considers to be invalid headers. The nginx directive
+  `ignore_invalid_headers` is no longer required.
+- Fix crash when a CPE is assigned a tag containing a dot.
+- Fix bug preventing the user from closing the preset pop-up after being
+  presented with unsupported preset message.
+- Fix exceptions raised from ext scripts manifesting as timeout faults.
+- Fix download getting triggered repeatedly when the value passed to `Download`
+  parameter is greater than the current timestamp.
+- Fix crash when passing invalid attributes to `declare`.
+- Fix crash in NBI when pushing tasks to non-existing devices.
+- Fix crash in NBI when passing invalid JSON to various API endpoints.
+- Fix crash when the output from `ping` command cannot be parsed in some rare
+  edge cases.
+- A number of other fixes and stability improvements.
+
+## 1.2.5 (2021-03-12)
+
+- Support specifying custom types when uploading files.
+- Fix JS compatibility issue with Safari browser.
+
+## 1.2.4 (2021-02-24)
+
+- The data model state of a CPE is no longer forgotten after unsuccessful
+  session termination (e.g. timeout). This addresses a number of undesired side
+  effects that arise when a CPE does not terminate the session properly.
+- Executing tasks that take a long time to complete (e.g. refreshing the entire
+  data model) no longer shows a timeout error while the task is still being
+  processed.
+- New function `ROUND()` available to expressions. It works similar to the
+  function by the same name in SQLite and PostgreSQL.
+- Log access events for `genieacs-nbi` service.
+- Pipe stdout/stderr from extension scripts to the `genieacs-cwmp` process log.
+- Parameter values of type `xsd:dateTime` are now displayed in the UI and CSV
+  downloads as a date string rather than a numeric value.
+- Add file download link in the files listing page.
+- Display spinner loading animation throughout the UI.
+- Display GenieACS version and build number underneath the logo.
+- New option to specify how many parameters are displayed at a time in the
+  all-parameters component. Simply set `limit` property in the component config.
+- Reduce overly strong Brotli compression level which was causing significant
+  page load slowdown when Brotli is used.
+- Retire dump-data-model tool. `genieacs-sim` can now use a CSV file as its data
+  model.
+- Reduce the number of concurrent database connections from each process.
+- Remove dependency on 'hostInfo' MongoDB command which is a privileged action.
+  It is now possible to use shared MongoDB instances with limited privileged.
+- Fix bug in NBI where querying files returns 404 error.
+- Fix ping not working for devices with an IPv6 address.
+- Fix an elusive memory leak in `genieacs-fs` that slowly eats up memory and can
+  go unnoticed for long periods of time.
+- Fix a rare edge case where a `declare()` call to set a parameter value may not
+  work as intended if the parameter was originally received as part of the
+  Inform message.
+- A number of other fixes and stability improvements.
+
+## 1.2.3 (2020-10-26)
+
+- New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
+  report writable parameters as non-writable. When set to true, the scripts will
+  no longer respect the 'writable' attribute of the CPE parameters and will send
+  a SetParamteerValues, AddObject, or DeleteObject request anyway.
+- Tags no longer restrict what characters are allowed. Any character other than
+  alphanumeric characters, hyphen, or underscore is now encoded in the data
+  model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
+- Ask for a confirmation before closing a pop-up dialog with unsaved changes.
+- Better XML validation to avoid crashes caused by invalid CPE requests.
+- Fix confusing 404 error message when the user attempts to modify a resource
+  when they don't have the necessary permissions.
+- Fix a rare issue where genieacs-cwmp stops accepting new connections after
+  running for a few weeks.
+- Fix exception when IS NULL operator is used in certain situations.
+
+## 1.2.2 (2020-10-03)
+
+- Added button to push files to selected devices from device listing page.
+- A few minor UI improvements.
+- Fix exception that can happen and persist after a Download request.
+- Fix validation bug preventing running refreshObject task on data model root.
+- Fix invalid arguments fault in refresh preset configuration when upgrading
+  from v1.1.
+
+## 1.2.1 (2020-09-08)
+
+- Fix bug causing faults to not be displayed in the UI.
+- Fix bug where deleting objects does not get reflected immediately in the UI.
+- Improve conversion between filters written in the expression format and
+  MongoDB queries. There should now be fewer edge cases where the two are not
+  equisatisfiable.
+
+## 1.2.0 (2020-09-01)
+
+- Support GetParameterAttributes and SetParameterAttributes TR-069 methods.
+- Support CASE statement and COALESCE function in expressions.
+- Provision arguments can now be a list of expressions that are dynamically
+  evaluated.
+- Support Forwarded HTTP header to display in the logs the correct IP of CPEs
+  behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
+- Config expressions can now access all available device parameters, not only
+  serial number, product class, and OUI.
+- Use relative URLs throughout the UI to allow serving from a subdirectory using
+  a reverse proxy.
+- Make Date.parse() and Date.UTC() available to provision scripts.
+- libxmljs has been entirely removed in favor of our bespoke XML parser.
+- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now used
+  to determine the TCP connection timeout.
+- The all-parameters component now limits the number of parameters displayed for
+  better performance.
+- The process genieacs-cwmp is now much less likely to throw exceptions as a
+  result of invalid requests from CPE.
+- A large number of bug fixes and stability improvements.
+
+## 1.2.0-beta.0 (2019-07-30)
+
+- A brand new UI superseding genieacs-gui.
+- New initialization wizard on first run.
+- New expression/query language used in search filters and preset preconditions.
+- CPE -> ACS authentication is now supported.
+- New config option (CWMP_KEEP_ALIVE_TIMEOUT) to specify how long to wait for a
+  reply from the CPE before closing the TCP connection.
+- Debug logging has been reimplemented utilizing YAML format for logs.
+- Handle 9005 faults (Invalid Parameter Name) gracefully by attempting to
+  rediscover the path of the missing parameter recursively.
+- declare() statements not followed by an explicit commit() are now deferred
+  until all currently active scripts have been executed.
+- FS_HOSTNAME now defaults to the server's hostname or IP.
+- The API now validates the structure of task objects before saving.
+- New XML parser implementation for better performance. You can revert to the
+  old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
+  v10.
+- Performance optimizations. While performance has improved for the majority of
+  use cases, there may be situations where performance has degraded. It's
+  recommended to revisit your hardware requirements.
+- Connection request authentication no longer uses 'auth.js' file. Instead, the
+  connection request authentication behavior can now be customized using an
+  'expression'.
+- The config file (config.json) has been deprecated. System configuration (e.g.
+  listen ports, worker count) are now recommended to be passed as environment.
+  variables. Other general configuration options are stored in the database so
+  as to not require service restart for changes to take effect.
+- Optional redis dependency has been removed completely.
+- Tags now allow only alphanumeric characters and underscore.
+- Supported versions of Node.js and MongoDB are 10.x and up and 2.6 and up
+  respectively.
+
+## 1.1.3 (2018-10-23)
+
+- New config option (MAX_COMMIT_ITERATIONS) to avoid max commit iterations
+  faults for more complex scripts.
+- Support base64 and hexBinary parameter types.
+- Strict parsing of number values in queries (e.g. "123abc" no longer accepted
+  as 123).
+- Mixing $ne and $not operators is not allowed. Now it throws an error instead
+  of returning incorrect results.
+- When a task expires, any associated fault is also deleted.
+- API now accepts 'timeout' argument when posting a task.
+- A number of stability fixes.
+
 ## 1.1.2 (2018-02-24)
 
 - A large number of bug fixes as well as stability and performance improvements.
 - Three security vulnerabilities disclosed by Maximilian Hils have been patched.
 - New config option UDP_CONNECTION_REQUEST_PORT to specify binding port for UDP
-connection requests.
+  connection requests.
 - New config option DATETIME_MILLISECONDS to strip milliseconds from dateTime
-values.
+  values.
 - New config option BOOLEAN_LITERAL to use 1/0 or true/false for boolean values.
 - Parameter values that cannot be parsed according to the reported type now show
-a warning message.
+  a warning message.
 - Virtual parameter scripts now use the variable 'args' instead of the special
-'TIMESTAMPS' and 'VALUES' variables. The content of the args array is: {declare
-timestamps}, {declare values}, {current timestamps}, {current values}.
+  'TIMESTAMPS' and 'VALUES' variables. The content of the args array is:
+  {declare timestamps}, {declare values}, {current timestamps}, {current
+  values}.
 - Virtual parameter value types are now inferred from the JavaScript type if the
-returned value attribute is not a value-type pair.
+  returned value attribute is not a value-type pair.
 - Show a fault when a virtual parameter script doesn't return the required
-attributes.
+  attributes.
 - Redis is now optional (and disabled by default), reducing the complexity of
-scalable deployments.
+  scalable deployments.
 - Better detection of cyclical presets resulting in fewer faults for complex
-provisioning scripts.
+  provisioning scripts.
 - Math.random() is now deterministic on per-device basis. A function has been
-added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
+  added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
 - Overload spikes are now handled gracefully by refusing to accept new sessions
-temporarily when under abnormal load.
+  temporarily when under abnormal load.
 - Added log messages for session timeouts, connection drops, and XML parsing
-errors.
+  errors.
 - Date.now() now takes an optional argument to specify "time steps" (in
-milliseconds). This can be used to ensure a group of parameters are all
-refreshed at the same time intervals.
+  milliseconds). This can be used to ensure a group of parameters are all
+  refreshed at the same time intervals.
 - Only the non-default configuration options are now logged at process start.
 - Faults caused by errors from extensions now show a cleaner stack trace.
 - Exit main process if there are too many worker crashes (e.g. when DB is down).
 - Updated dependencies and included a lockfile to ensure installations get the
-exact dependencies it was tested against.
+  exact dependencies it was tested against.
 
 ## 1.1.1 (2017-03-23)
 
 - Avoid crashing when connection request credentials are missing.
 - Show a warning instead of crashing when failing to parse parameter values
-according to the expected value type.
+  according to the expected value type.
 - Add missing "Registered" event.
 - Fix bug where in certain cases many more instances than declared are created.
 - Fix parameter discovery bug when declared path timestamp is 1 or is not set.
 - Fix preset precondition failing when testing against datetime parameters and
-certain other parameters like \_deviceId.\_ProductClass.
+  certain other parameters like \_deviceId.\_ProductClass.
 
 ## 1.1.0 (2017-03-10)
 
 - Provisions enable implementing dynamic device configuration or complex device
-provisioning work flow using arbitrary scripts.
+  provisioning work flow using arbitrary scripts.
 - Virtual parameters are user-defined parameters whose values are evaluated from
-a custom script.
+  a custom script.
 - Extensions are sandboxed Node.js scripts that are accessible from provision
-and virtual parameter scripts to facilitate integration with external entities.
-- Support for UDP/STUN based connection requests for reaching devices behind
-NAT (TR-069 Annex G).
+  and virtual parameter scripts to facilitate integration with external
+  entities.
+- Support for UDP/STUN based connection requests for reaching devices behind NAT
+  (TR-069 Annex G).
 - Presets can now be scheduled using a cron-like expression.
 - Presets can now be tied to specific device events (e.g. boot).
-- Presets precondition queries no longer support "$or" or other MongoDB logical
-operators.
+- Presets precondition queries no longer support "\$or" or other MongoDB logical
+  operators.
 - Faults are no longer a part of tasks but are now first class objects.
 - Presets are now assigned to channels. A fault in one channel only blocks
-presets in that channel.
+  presets in that channel.
 - New API CRUD functions for provisions, virtual parameters, and faults.
 - New config options for XML output.
 - API responses now include "GenieACS-Version" header.
@@ -73,19 +374,19 @@ presets in that channel.
 - Supported Node.js versions are 6.x and 7.x.
 - Supported MongoDB versions are 2.6 through 3.4.
 - Expect performance differences due to major under the hood changes. Some
-operations are faster and some are slower. Overall performance is improved.
+  operations are faster and some are slower. Overall performance is improved.
 - GenieACS will no longer fetch the entire device data model upon first contact
-but will instead only fetch the parameters it needs to fulfill the presets.
+  but will instead only fetch the parameters it needs to fulfill the presets.
 - Logs have been overhauled and split into two streams: process log (stderr) and
-access log (stdout). Also added config options to dump logs to files rather than
-standard streams.
+  access log (stdout). Also added config options to dump logs to files rather
+  than standard streams.
 - Connection request authentication credentials are picked up from the device
-data model if available. config/auth.js is still supported as a fallback and now
-supports an optional callback argument.
+  data model if available. config/auth.js is still supported as a fallback and
+  now supports an optional callback argument.
 - Custom commands have been removed. Use virtual parameters and/or extensions.
 - Aliases and value normalizers (config/parameters.json) have been removed. Use
-virtual parameters.
+  virtual parameters.
 - The API /devices/<device_id>/preset has been removed.
 - Rarely used RequestDownload method no longer supported.
 - The TR-069 client simulator has moved to its own repo at
-https://github.com/zaidka/genieacs-sim
+  https://github.com/zaidka/genieacs-sim

CONTRIBUTING.md

@@ -1,22 +1,21 @@
 # Have a question?
 
-Please ask in the [Users mailing list][mailing-lists] instead of opening a
-Github Issue. The Github issue tracker is strictly for bug reports.
+Please ask in the [forum] instead of opening a Github Issue. The Github issue
+tracker is strictly for bug reports.
 
 # Issues
 
-We use GitHub issues to track bugs. Please ensure your bug description is
-clear and has sufficient instructions to be able to reproduce the issue.
+We use GitHub issues to track bugs. Please ensure your bug description is clear
+and has sufficient instructions to be able to reproduce the issue.
 
-If you face interoperability issues with your device, more often than not it's
-a device related issue. Please consult the [Users mailing list][mailing-lists]
-before submitting an Issue.
+If you face interoperability issues with your CPE, more often than not it's a
+device related issue. Please consult the [forum] before submitting an Issue.
 
 # Pull Requests
 
 Pull requests are welcome. If you have a fix for a bug, go ahead and create a
-pull request. For other kinds of enhancements, please discuss in the [Users
-mailing list][mailing-lists] first.
+pull request. For other kinds of enhancements, please discuss in the [forum]
+first.
 
 Please follow the guidelines [here][commit-guidelines] when committing your
 changes.
@@ -27,13 +26,14 @@ By submitting a pull request to this repository, you acknowledge that, while
 maintaining copyright, you grant Zaid Abdulla a perpetual, worldwide, non-
 exclusive, no-charge, royalty-free, irrevocable license to reproduce, prepare
 derivative works of, publicly display, publicly perform, sublicense, and
-distribute your contributions and such derivative works under the AGPLv3
-license or any other license terms, including, but not limited to, proprietary
-or commercial license terms.
+distribute your contributions and such derivative works under the AGPLv3 license
+or any other license terms, including, but not limited to, proprietary or
+commercial license terms.
 
-You confirm that you own or have rights to distribute and sublicense the
-source code contained therein, and that your content does not infringe upon
-the intellectual property rights of a third party.
+You confirm that you own or have rights to distribute and sublicense the source
+code contained therein, and that your content does not infringe upon the
+intellectual property rights of a third party.
 
-[commit-guidelines]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
-[mailing-lists]: http://lists.genieacs.com
+[commit-guidelines]:
+  http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
+[forum]: https://forum.genieacs.com

README.md

@@ -1,18 +1,10 @@
 # GenieACS
 
-GenieACS is a fast and lightweight TR-069 auto configuration server (ACS). This
-is the core back end component. A GUI front end is available at
-https://github.com/genieacs/genieacs-gui.
-
-## Features
-
-- **Battle-tested at scale**: Can scale to manage hundreds of thousands and
-potentially millions of devices
-- **Powerful provisioning system**: Scriptable configuration to handle complex
-device provisioning scenarios and other automated operations such as firmware
-upgrade
-- **Extensive API**: A rich HTTP-based API allows easy integration with other
-systems
+GenieACS is a high performance Auto Configuration Server (ACS) for remote
+management of TR-069 enabled devices. It utilizes a declarative and fault
+tolerant configuration engine for automating complex provisioning scenarios at
+scale. It's battle-tested to handle hundreds of thousands and potentially
+millions of concurrent devices.
 
 ## Quick Start
 
@@ -20,61 +12,65 @@ Install [Node.js](http://nodejs.org/) and [MongoDB](http://www.mongodb.org/).
 Refer to their corresponding documentation for installation instructions. The
 supported versions are:
 
-- Node.js: 6.x and 8.x (8.x recommended)
-- MongoDB: 2.6 through 3.4
+- Node.js: 12.3+
+- MongoDB: 3.6+
 
-Install build tools and libxml2 development files from your system's package
-manager.
+Install GenieACS from NPM:
 
-Then install GenieACS using NPM:
+    sudo npm install -g genieacs
 
-    npm install -g genieacs
+To build from source instead, clone this repo or download the source archive
+then _cd_ into the source directory then run:
 
-Alternatively, you can install from source by cloning the git repository:
-
-    git clone https://github.com/genieacs/genieacs.git
-    cd genieacs
-    git checkout $(git tag -l v1.1.* --sort=-v:refname | head -n 1)
     npm install
-    npm run compile
+    npm run build
 
-Before proceeding, find and review the file "config.json" in "config" directory
-where GenieACS is downloaded.
+Finally, run the following services (found under `./dist/bin/` if building from
+source):
 
-Finally, run the following (from bin directory if installing from source):
+### genieacs-cwmp
 
-    genieacs-cwmp
+This is the service that the CPEs will communicate with. It listens on port 7547
+by default. Configure the ACS URL in your devices accordingly.
 
-This is the service that the CPEs will communicate with. It listens to port 7547
-by default (see config/config.json). Configure the ACS URL of your devices
-accordingly.
+You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
+as a dummy TR-069 simulator if you don't have a CPE at hand.
 
-    genieacs-nbi
+### genieacs-nbi
 
 This is the northbound interface module. It exposes a REST API on port 7557 by
-default. This must be running for the GUI front end to work.
+default. This one is only required if you have an external system integrating
+with GenieACS using this API.
 
-    genieacs-fs
+### genieacs-fs
 
 This is the file server from which the CPEs will download firmware images and
-such.
+such. It listens on port 7567 by default.
 
-Note: For production deployment make sure to run those as background services.
+### genieacs-ui
 
-For further details about installation and configuration, refer to the
-[wiki section](https://github.com/genieacs/genieacs/wiki).
+This serves the web based user interface. It listens on port 3000 by default.
+You must pass _--ui-jwt-secret_ argument to supply the secret key used for
+signing browser cookies:
+
+    genieacs-ui --ui-jwt-secret secret
+
+The UI has plenty of configuration options. When you open GenieACS's UI in a
+browser you'll be greeted with a database initialization wizard to help you
+populate some initial configuration.
+
+Visit [docs.genieacs.com](https://docs.genieacs.com) for more documentation and
+a complete installation guide for production deployments.
 
 ## Support
 
-The [Users mailing list](http://lists.genieacs.com) is a good place to get
-guidance and help from the community. Head on over and join the conversation!
-In addition, the [wiki](https://github.com/genieacs/genieacs/wiki) provides useful
-documentation and tips from GenieACS users.
+The [forum](https://forum.genieacs.com) is a good place to get guidance and help
+from the community. Head on over and join the conversation!
 
-For commercial support options and professional services, please visit
-[genieacs.com](https://genieacs.com).
+For commercial support options, please visit
+[genieacs.com](https://genieacs.com/support/).
 
 ## License
 
-Copyright 2013-2018 GenieACS Inc. GenieACS is released under the
+Copyright 2013-2021 GenieACS Inc. GenieACS is released under the
 [AGPLv3 license terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

bin/genieacs-cwmp

@@ -1,9 +0,0 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
-
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
-
-var cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('cwmp');

bin/genieacs-cwmp.ts

@@ -0,0 +1,118 @@
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import * as cwmp from "../lib/cwmp.ts";
+import * as db from "../lib/db/db.ts";
+import * as extensions from "../lib/extensions.ts";
+import { version as VERSION } from "../package.json";
+
+logger.init("cwmp", VERSION);
+
+const SERVICE_ADDRESS = config.get("CWMP_INTERFACE") as string;
+const SERVICE_PORT = config.get("CWMP_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  void extensions.killAll().finally(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-cwmp starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("CWMP_SSL_KEY") as string;
+  const cert = config.get("CWMP_SSL_CERT") as string;
+
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    onConnection: cwmp.onConnection,
+    onClientError: cwmp.onClientError,
+    timeout: 30000,
+    keepAliveTimeout: 0,
+    requestTimeout: cwmp.REQUEST_TIMEOUT,
+  };
+
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    server.stop(false).then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, cwmp.listener);
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    void initPromise.finally(() => {
+      server
+        .stop(false)
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    void initPromise.finally(() => {
+      server
+        .stop(false)
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-ext.ts

@@ -0,0 +1,82 @@
+import { Fault } from "../lib/types.ts";
+
+const jobs = new Set();
+const fileName = process.argv[2];
+let script;
+
+function errorToFault(err: Error): Fault {
+  if (!err) return null;
+
+  if (!err.name) return { code: "ext", message: `${err}` };
+
+  const fault: Fault = {
+    code: `ext.${err.name}`,
+    message: err.message,
+    detail: {
+      name: err.name,
+      message: err.message,
+    },
+  };
+
+  if (err.stack) {
+    fault.detail["stack"] = err.stack;
+    // Trim the stack trace
+    const stackTrimIndex = fault.detail["stack"].match(
+      /\s+at\s[^\s]+\s\(.*genieacs-ext:.+\)/,
+    );
+    if (stackTrimIndex) {
+      fault.detail["stack"] = fault.detail["stack"].slice(
+        0,
+        stackTrimIndex.index,
+      );
+    }
+  }
+
+  return fault;
+}
+
+// Need this for Node < 15
+process.on("unhandledRejection", (err) => {
+  throw err;
+});
+
+process.on("uncaughtException", (err) => {
+  const fault = errorToFault(err);
+  jobs.forEach((jobId) => {
+    process.send([jobId, fault, null]);
+  });
+  jobs.clear();
+  process.disconnect();
+});
+
+process.on("message", (message) => {
+  jobs.add(message[0]);
+
+  if (!script) {
+    const cwd = process.env["GENIEACS_EXT_DIR"];
+    process.chdir(cwd);
+    script = require(`${cwd}/${fileName}`);
+  }
+
+  const funcName = message[1][0];
+
+  if (!script[funcName]) {
+    const fault = {
+      code: "ext",
+      message: `No such function '${funcName}' in extension '${fileName}'`,
+    };
+    process.send([message[0], fault, null]);
+    return;
+  }
+
+  script[funcName](message[1].slice(1), (err, res) => {
+    if (!jobs.delete(message[0])) return;
+
+    process.send([message[0], errorToFault(err), res]);
+  });
+});
+
+// Ignore SIGINT
+process.on("SIGINT", () => {
+  // Ignore
+});

bin/genieacs-fs

@@ -1,9 +0,0 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
-
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
-
-var cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('fs');

bin/genieacs-fs.ts

@@ -0,0 +1,100 @@
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import { listener } from "../lib/fs.ts";
+import * as db from "../lib/db/db.ts";
+import { version as VERSION } from "../package.json";
+
+logger.init("fs", VERSION);
+
+const SERVICE_ADDRESS = config.get("FS_INTERFACE") as string;
+const SERVICE_PORT = config.get("FS_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([db.disconnect(), cluster.worker.disconnect()])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  process.exit(1);
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-fs starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("FS_SSL_KEY") as string;
+  const cert = config.get("FS_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
+  };
+
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, listener);
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-nbi

@@ -1,9 +0,0 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
-
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
-
-var cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('nbi');

bin/genieacs-nbi.ts

@@ -0,0 +1,107 @@
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import { listener } from "../lib/nbi.ts";
+import * as db from "../lib/db/db.ts";
+import * as extensions from "../lib/extensions.ts";
+import { version as VERSION } from "../package.json";
+
+logger.init("nbi", VERSION);
+
+const SERVICE_ADDRESS = config.get("NBI_INTERFACE") as string;
+const SERVICE_PORT = config.get("NBI_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  void extensions.killAll().finally(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-nbi starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("NBI_SSL_KEY") as string;
+  const cert = config.get("NBI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 120000,
+  };
+
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, listener);
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-ui.ts

@@ -0,0 +1,107 @@
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import { listener } from "../lib/ui.ts";
+import * as extensions from "../lib/extensions.ts";
+import * as db from "../lib/db/db.ts";
+import { version as VERSION } from "../package.json";
+
+logger.init("ui", VERSION);
+
+const SERVICE_ADDRESS = config.get("UI_INTERFACE") as string;
+const SERVICE_PORT = config.get("UI_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  void extensions.killAll().finally(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-ui starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("UI_SSL_KEY") as string;
+  const cert = config.get("UI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 120000,
+  };
+
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, async (req, res) => listener(req, res));
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+}

build/assets.ts

@@ -0,0 +1,5 @@
+export const APP_JS = "app.js";
+export const APP_CSS = "app.css";
+export const ICONS_SVG = "icons.svg";
+export const LOGO_SVG = "logo.svg";
+export const FAVICON_PNG = "favicon.png";

build/build.ts

@@ -0,0 +1,359 @@
+import path from "node:path";
+import fs from "node:fs";
+import { createHash } from "node:crypto";
+import { promisify } from "node:util";
+import { exec } from "node:child_process";
+import * as esbuild from "esbuild";
+import { optimize } from "svgo";
+import * as xmlParser from "../lib/xml-parser.ts";
+
+const fsAsync = {
+  readdir: promisify(fs.readdir),
+  readFile: promisify(fs.readFile),
+  writeFile: promisify(fs.writeFile),
+  copyFile: promisify(fs.copyFile),
+  rename: promisify(fs.rename),
+  chmod: promisify(fs.chmod),
+  lstat: promisify(fs.lstat),
+  exists: promisify(fs.exists),
+  rmdir: promisify(fs.rmdir),
+  unlink: promisify(fs.unlink),
+  mkdir: promisify(fs.mkdir),
+};
+
+const execAsync = promisify(exec);
+
+const MODE = process.env["NODE_ENV"] || "production";
+
+const INPUT_DIR = process.cwd();
+const OUTPUT_DIR = path.join(INPUT_DIR, "dist");
+
+async function rmDir(dirPath: string): Promise<void> {
+  if (!(await fsAsync.exists(dirPath))) return;
+  const files = await fsAsync.readdir(dirPath);
+
+  for (const file of files) {
+    const filePath = path.join(dirPath, file);
+    if ((await fsAsync.lstat(filePath)).isDirectory()) await rmDir(filePath);
+    else await fsAsync.unlink(filePath);
+  }
+  await fsAsync.rmdir(dirPath);
+}
+
+// For lockfileVersion = 1
+function stripDevDeps(deps): void {
+  if (!deps["dependencies"]) return;
+  for (const [k, v] of Object.entries(deps["dependencies"])) {
+    if (v["dev"]) delete deps["dependencies"][k];
+    else stripDevDeps(v);
+  }
+  if (!Object.keys(deps["dependencies"]).length) delete deps["dependencies"];
+}
+
+// For lockfileVersion = 2
+function stripDevDeps2(deps): void {
+  if (!deps["packages"]) return;
+  for (const [k, v] of Object.entries(deps["packages"])) {
+    delete v["devDependencies"];
+    if (v["dev"]) delete deps["packages"][k];
+  }
+}
+
+function xmlTostring(xml): string {
+  const children = [];
+  for (const c of xml.children || []) children.push(xmlTostring(c));
+
+  return xml.name === "root" && xml.bodyIndex === 0
+    ? children.join("")
+    : `<${xml.name} ${xml.attrs}>${children.join("")}</${xml.name}>`;
+}
+
+function assetHash(buffer: Buffer | string): string {
+  return createHash("md5").update(buffer).digest("hex").slice(0, 8);
+}
+
+const ASSETS = {} as {
+  APP_JS?: string;
+  APP_CSS?: string;
+  ICONS_SVG?: string;
+  LOGO_SVG?: string;
+  FAVICON_PNG?: string;
+};
+
+const assetsPlugin = {
+  name: "assets",
+  setup(build) {
+    build.onLoad({ filter: /\/build\/assets.ts$/ }, () => {
+      const lines = Object.entries(ASSETS).map(
+        ([k, v]) => `export const ${k} = ${JSON.stringify(v)};`,
+      );
+      return { contents: lines.join("\n") };
+    });
+  },
+} as esbuild.Plugin;
+
+const packageDotJsonPlugin = {
+  name: "packageDotJson",
+  setup(build) {
+    const sourcePath = path.join(INPUT_DIR, "package.json");
+    build.onResolve({ filter: /\/package.json$/ }, (args) => {
+      const p = path.join(args.resolveDir, args.path);
+      if (p !== sourcePath) return undefined;
+      return { path: path.join(OUTPUT_DIR, "package.json") };
+    });
+  },
+} as esbuild.Plugin;
+
+const inlineDepsPlugin = {
+  name: "inlineDeps",
+  setup(build) {
+    const deps = [
+      "parsimmon",
+      "espresso-iisojs",
+      "codemirror",
+      "mithril",
+      "yaml",
+    ];
+    build.onResolve({ filter: /^[^.]/ }, async (args) => {
+      if (deps.some((d) => args.path.startsWith(d))) return undefined;
+      return { sideEffects: false, external: true };
+    });
+  },
+} as esbuild.Plugin;
+
+function generateSymbol(id: string, svgStr: string): string {
+  const xml = xmlParser.parseXml(svgStr);
+  const svg = xml.children[0];
+  const svgAttrs = xmlParser.parseAttrs(svg.attrs);
+  let viewBox = "";
+  for (const a of svgAttrs) {
+    if (a.name === "viewBox") {
+      viewBox = `viewBox="${a.value}"`;
+      break;
+    }
+  }
+  const symbolBody = xml.children[0].children
+    .map((c) => xmlTostring(c))
+    .join("");
+  return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
+}
+
+async function getBuildMetadata(): Promise<string> {
+  const date = new Date().toISOString().slice(2, 10).replaceAll("-", "");
+
+  const [commit, diff, newFiles] = await Promise.all([
+    execAsync("git rev-parse HEAD"),
+    execAsync("git diff HEAD"),
+    execAsync("git ls-files --others --exclude-standard"),
+  ]).then((res) => res.map((r) => r.stdout.trim()));
+
+  if (!diff && !newFiles) return date + commit.slice(0, 4);
+
+  const hash = createHash("md5");
+  hash.update(commit).update(diff).update(newFiles);
+  for (const file of newFiles.split("\n").filter((f) => f))
+    hash.update(await fsAsync.readFile(file));
+  return date + hash.digest("hex").slice(0, 4);
+}
+
+async function init(): Promise<void> {
+  const [buildMetadata, packageJsonFile, npmShrinkwrapFile] = await Promise.all(
+    [
+      getBuildMetadata(),
+      fsAsync.readFile(path.join(INPUT_DIR, "package.json")),
+      fsAsync.readFile(path.join(INPUT_DIR, "npm-shrinkwrap.json")),
+    ],
+  );
+
+  const packageJson = JSON.parse(packageJsonFile.toString());
+  delete packageJson["devDependencies"];
+  delete packageJson["private"];
+  delete packageJson["scripts"];
+  packageJson["version"] = `${packageJson["version"]}+${buildMetadata}`;
+
+  const npmShrinkwrap = JSON.parse(npmShrinkwrapFile.toString());
+  npmShrinkwrap["version"] = packageJson["version"];
+  stripDevDeps(npmShrinkwrap);
+  stripDevDeps2(npmShrinkwrap);
+
+  await rmDir(OUTPUT_DIR);
+
+  await fsAsync.mkdir(OUTPUT_DIR);
+
+  await Promise.all([
+    fsAsync.mkdir(path.join(OUTPUT_DIR, "bin")),
+    fsAsync.mkdir(path.join(OUTPUT_DIR, "public")),
+    fsAsync.writeFile(
+      path.join(OUTPUT_DIR, "package.json"),
+      JSON.stringify(packageJson, null, 2),
+    ),
+    fsAsync.writeFile(
+      path.join(OUTPUT_DIR, "npm-shrinkwrap.json"),
+      JSON.stringify(npmShrinkwrap, null, 2),
+    ),
+  ]);
+}
+
+async function copyStatic(): Promise<void> {
+  const files = [
+    "LICENSE",
+    "README.md",
+    "CHANGELOG.md",
+    "public/logo.svg",
+    "public/favicon.png",
+  ];
+
+  const [logo, favicon] = await Promise.all([
+    fsAsync.readFile(path.join(INPUT_DIR, "public/logo.svg")),
+    fsAsync.readFile(path.join(INPUT_DIR, "public/favicon.png")),
+  ]);
+
+  ASSETS.LOGO_SVG = `logo-${assetHash(logo)}.svg`;
+  ASSETS.FAVICON_PNG = `favicon-${assetHash(favicon)}.png`;
+
+  const filenames = {} as Record<string, string>;
+  filenames["public/logo.svg"] = path.join("public", ASSETS.LOGO_SVG);
+  filenames["public/favicon.png"] = path.join("public", ASSETS.FAVICON_PNG);
+
+  await Promise.all(
+    files.map((f) =>
+      fsAsync.copyFile(
+        path.join(INPUT_DIR, f),
+        path.join(OUTPUT_DIR, filenames[f] || f),
+      ),
+    ),
+  );
+}
+
+async function generateCss(): Promise<void> {
+  const res = await esbuild.build({
+    bundle: true,
+    absWorkingDir: INPUT_DIR,
+    minify: MODE === "production",
+    sourcemap: "linked",
+    sourcesContent: false,
+    entryPoints: ["ui/css/app.css"],
+    entryNames: "[dir]/[name]-[hash]",
+    outfile: path.join(OUTPUT_DIR, "public/app.css"),
+    target: ["chrome109", "safari15.6", "firefox115", "opera102", "edge118"],
+    metafile: true,
+  });
+
+  for (const [k, v] of Object.entries(res.metafile.outputs)) {
+    if (v.entryPoint === "ui/css/app.css") {
+      ASSETS.APP_CSS = path.relative(
+        path.join(OUTPUT_DIR, "public"),
+        path.join(INPUT_DIR, k),
+      );
+      break;
+    }
+  }
+}
+
+async function generateBackendJs(): Promise<void> {
+  const services = [
+    "genieacs-cwmp",
+    "genieacs-ext",
+    "genieacs-nbi",
+    "genieacs-fs",
+    "genieacs-ui",
+  ];
+
+  await esbuild.build({
+    bundle: true,
+    absWorkingDir: INPUT_DIR,
+    minify: MODE === "production",
+    sourcemap: "inline",
+    sourcesContent: false,
+    platform: "node",
+    target: "node12.13.0",
+    packages: "external",
+    banner: { js: "#!/usr/bin/env node" },
+    entryPoints: services.map((s) => `bin/${s}.ts`),
+    outdir: path.join(OUTPUT_DIR, "bin"),
+    plugins: [packageDotJsonPlugin, assetsPlugin],
+  });
+
+  for (const bin of services) {
+    const p = path.join(OUTPUT_DIR, "bin", bin);
+    await fsAsync.rename(`${p}.js`, p);
+    // Mark as executable
+    const mode = (await fsAsync.lstat(p)).mode;
+    await fsAsync.chmod(p, mode | 73);
+  }
+}
+
+async function generateFrontendJs(): Promise<void> {
+  const res = await esbuild.build({
+    bundle: true,
+    absWorkingDir: INPUT_DIR,
+    splitting: true,
+    minify: MODE === "production",
+    sourcemap: "linked",
+    sourcesContent: false,
+    platform: "browser",
+    format: "esm",
+    target: ["chrome109", "safari15.6", "firefox115", "opera102", "edge118"],
+    entryPoints: ["ui/app.ts"],
+    entryNames: "[dir]/[name]-[hash]",
+    outdir: path.join(OUTPUT_DIR, "public"),
+    plugins: [packageDotJsonPlugin, inlineDepsPlugin, assetsPlugin],
+    metafile: true,
+  });
+
+  for (const [k, v] of Object.entries(res.metafile.outputs)) {
+    for (const imp of v.imports)
+      if (imp.external) throw new Error(`External import found: ${imp.path}`);
+
+    if (v.entryPoint === "ui/app.ts") {
+      ASSETS.APP_JS = path.relative(
+        path.join(OUTPUT_DIR, "public"),
+        path.join(INPUT_DIR, k),
+      );
+    }
+  }
+}
+
+async function generateIconsSprite(): Promise<void> {
+  const symbols = [] as string[];
+  const iconsDir = path.join(INPUT_DIR, "ui/icons");
+  for (const file of await fsAsync.readdir(iconsDir)) {
+    const id = path.parse(file).name;
+    const filePath = path.join(iconsDir, file);
+    const src = (await fsAsync.readFile(filePath)).toString();
+    const { data } = await optimize(src, {
+      plugins: [
+        {
+          name: "preset-default",
+          params: {
+            overrides: {
+              removeViewBox: false,
+            },
+          },
+        },
+      ],
+    });
+    symbols.push(generateSymbol(id, data));
+  }
+  const data = `<svg xmlns="http://www.w3.org/2000/svg">${symbols.join(
+    "",
+  )}</svg>`;
+  ASSETS.ICONS_SVG = `icons-${assetHash(data)}.svg`;
+  await fsAsync.writeFile(
+    path.join(OUTPUT_DIR, "public", ASSETS.ICONS_SVG),
+    data,
+  );
+}
+
+init()
+  .then(() =>
+    Promise.all([
+      Promise.all([generateIconsSprite(), copyStatic()]).then(
+        generateFrontendJs,
+      ),
+      generateCss(),
+    ]).then(generateBackendJs),
+  )
+  .catch((err) => {
+    process.stderr.write(err.stack + "\n");
+  });

build/lint.ts

@@ -0,0 +1,55 @@
+import { exec } from "node:child_process";
+import { promisify } from "node:util";
+
+const execPromise = promisify(exec);
+
+async function runEslint(): Promise<string> {
+  const CMD =
+    "eslint 'bin/*.ts' 'lib/**/*.ts' 'ui/**/*.ts' 'test/**/*.ts' 'build/**/*.ts'";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  try {
+    const { stdout, stderr } = await execPromise(CMD, { env });
+    if (stderr) throw new Error(stderr);
+    return stdout;
+  } catch (err) {
+    if (err.killed || err.signal || err.stderr || err.code !== 1) throw err;
+    return err.stdout;
+  }
+}
+
+async function runTsc(): Promise<string> {
+  const CMD = "tsc --noEmit";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  const { stdout, stderr } = await execPromise(CMD, { env });
+  if (stderr) throw new Error(stderr);
+  return stdout;
+}
+
+async function runPrettier(): Promise<string> {
+  const CMD = "prettier --prose-wrap always --write .";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  const { stdout, stderr } = await execPromise(CMD, { env });
+  if (stderr) throw new Error(stderr);
+  return stdout;
+}
+
+async function runAll(): Promise<void> {
+  const prom1 = runPrettier();
+  const prom2 = runEslint();
+  const prom3 = runTsc();
+
+  console.log(await prom1);
+  console.log(await prom2);
+  console.log(await prom3);
+}
+
+runAll().catch(console.error);

build/spellcheck-dict.pws

@@ -0,0 +1,180 @@
+personal_ws-1.1 en 0
+genieacs
+GenieACS
+javascript
+sudo
+config
+cwmp
+CPE
+systemctl
+nbi
+CWMP
+NBI
+ACS
+SSL
+xsd
+args
+DeviceID
+GENIEACS
+env
+param
+arg
+AUTH
+igd
+TCP
+toctree
+auth
+EnvironmentFile
+JSON
+json
+parameterValues
+setParameterValues
+stderr
+stdout
+attr
+cfg
+CPEs
+ExecStart
+faq
+FileType
+GenieACS's
+JWT
+maxdepth
+npm
+objectName
+refreshObject
+SerialNumber
+systemd
+usr
+WantedBy
+yaml
+chown
+Config
+getParameterValues
+https
+InternetGatewayDevice
+mkdir
+OUI
+sql
+TODO
+UDP
+addObject
+AGPLv
+const
+cpe
+cron
+deleteObject
+FactoryReset
+factoryReset
+FileName
+fileType
+Github
+hostname
+HTTPS
+jwt
+latlong
+NPM
+oui
+parameterNames
+ProductClass
+productClass
+repo
+SSID
+sublicense
+TLS
+WANIPConnection
+WPA
+YourProvisionName
+abc
+Abdulla
+accessors
+AddObject
+AddressingType
+api
+APIs
+chmod
+cwmp's
+dateext
+DATETIME
+dateTime
+datetime
+declaratively
+delaycompress
+DeleteObject
+deviceId
+DHCP
+dir
+distro
+equisatisfiable
+ExternalIPAddress
+failover
+favor
+func
+GetParameterAttributes
+getPassword
+gte
+gui
+hexBinary
+Hils
+HOSTNAME
+init
+journalctl
+journald
+LastFileName
+LastFileType
+lastInform
+LIBXMLJS
+libxmljs
+literalinclude
+lockfile
+logrotate
+lte
+mipsbe
+normalizers
+pre
+quickstart
+README
+redis
+reimplemented
+RequestDownload
+RESTful
+rollout
+sandboxed
+scalable
+SetParameterAttributes
+SetParamteerValues
+SIGINT
+SIGTERM
+skipWritableCheck
+stringify
+subdirectory
+TargetFileName
+useradd
+VirtualParameters
+vparam
+xml
+YAML
+Zaid
+cacheExpire
+iss
+http
+statusCode
+rawData
+pos
+Brotli
+CSV
+hostInfo
+IPv
+downloadSuccessOnTimeout
+TransferComplete
+nginx
+ENCODEURICOMPONENT
+pageSize
+skipRootGpn
+GPN
+NaN
+CIDR
+XMPP
+PEM
+JID
+unindexed

build/spellcheck.sh

@@ -0,0 +1,12 @@
+#! /bin/sh
+
+cd "$(dirname "$0")"
+
+FILES=`ls ../docs/*.rst ../docs/*.js ../*.md`
+
+for FILE in $FILES
+do
+    echo $FILE
+    cat $FILE | aspell list --lang=en --add-extra-dicts=./spellcheck-dict.pws --ignore 2
+    echo
+done

config/auth-sample.js

@@ -1,7 +0,0 @@
-"use strict";
-
-function connectionRequest(deviceId, url, username, password, callback) {
-  return callback(username || deviceId, password || "");
-}
-
-exports.connectionRequest = connectionRequest;

config/config-sample.json

@@ -1,12 +0,0 @@
-{
-  "MONGODB_CONNECTION_URL" : "mongodb://127.0.0.1/genieacs",
-  "CWMP_INTERFACE" : "0.0.0.0",
-  "CWMP_PORT" : 7547,
-  "CWMP_SSL" : false,
-  "NBI_INTERFACE" : "0.0.0.0",
-  "NBI_PORT" : 7557,
-  "FS_INTERFACE" : "0.0.0.0",
-  "FS_PORT" : 7567,
-  "FS_HOSTNAME" : "acs.example.com",
-  "DEBUG" : false
-}

config/ext-sample.js

@@ -1,34 +0,0 @@
-// This is an exmaple GenieACS extension to get the current latitude/longitude
-// of the International Space Station. Why, you ask? Because why not.
-// To install, copy this file to config/ext/iss.js.
-
-"use strict";
-
-const http = require("http");
-
-let cache = null;
-let cacheExpire = 0;
-
-function latlong(args, callback) {
-  if (Date.now() < cacheExpire)
-    return callback(null, cache);
-
-  http.get("http://api.open-notify.org/iss-now.json", (res) => {
-    if (res.statusCode !== 200)
-      return callback(new Error(`Request failed (status code: ${res.statusCode})`));
-
-    let rawData = "";
-    res.on("data", (chunk) => rawData += chunk);
-
-    res.on("end", () => {
-      let pos = JSON.parse(rawData)["iss_position"];
-      cache = [+pos["latitude"], +pos["longitude"]];
-      cacheExpire = Date.now() + 10000;
-      callback(null, cache);
-    });
-  }).on("error", (err) => {
-    callback(err);
-  });
-}
-
-exports.latlong = latlong;

docs/.readthedocs.yaml

@@ -0,0 +1,18 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+
+sphinx:
+  configuration: docs/conf.py
+
+formats: all
+
+python:
+  install:
+    - requirements: docs/requirements.txt

docs/administration-faq.rst

@@ -0,0 +1,64 @@
+.. _administration-faq:
+
+Administration FAQ
+==================
+
+.. _administration-faq-duplicate-log-entries:
+
+Duplicate log entries when using :func:`log()` function
+-------------------------------------------------------
+
+Because GenieACS uses a full fledged scripting language for device
+configuration, the only way to guarantee that it has satisfied the 'desired
+state' is by repeatedly executing the script until there's no more
+discrepancies with the current device state. Though it may seem like this will
+cause duplicate requests going to the device, this isn't actually the case
+because device configuration are stated declaratively and that the scripts
+themselves are pure functions in the context of a session (e.g. Date.now()
+always returns the same value within the session).
+
+To illustrate with an example, consider the following script:
+
+.. code:: javascript
+
+  log("Executing script");
+  declare("Device.param", null, {value: 1});
+  commit();
+  declare("Device.param", null, {value: 2});
+
+This will set the value of the 'Device.param' to 1, then to 2. Then as the
+script is run again the value is set back to 1 and so on. A stable state will
+never be reached so GenieACS will execute the script a few times until it gives
+up and throws a fault. This is an edge case that should be avoided. A more
+typical case is where the script is run once or twice. Essentially if an
+execution doesn't result in any request to the CPE or a change in the data
+model then a stable state is deemed to have been reached.
+
+Configurations not pushed to device after factory reset
+---------------------------------------------------------
+
+After a device is reset to its factory default state, the cached data model in
+GenieACS's database needs to be invalidated to force rediscovery. Ensure the
+following lines are called on ``0 BOOTSTRAP`` event:
+
+.. code:: javascript
+
+  const now = Date.now();
+
+  // Clear cached data model to force a refresh
+  clear("Device", now);
+  clear("InternetGatewayDevice", now);
+
+
+Most device parameters are missing
+----------------------------------
+
+For performance reasons (server, client, and network), GenieACS by default only
+fetches parts of the data model that are necessary to satisfy the declarations
+in your provision scripts. Create declarations for any parameters you need
+fetched by default.
+
+If you're unsure and want to explore the available parameters exposed by the
+device, refresh the root parameter (e.g. ``InternetGatewayDevice``) from
+GenieACS's UI. You typically only need to do that one time for a given CPE
+model.

docs/api-reference.rst

@@ -0,0 +1,538 @@
+API Reference
+=============
+
+GenieACS exposes a rich RESTful API through its NBI component. This document
+serves as a reference for the available APIs.
+
+This API makes use of MongoDB's query language in some of its endpoints. Refer
+to MongoDB's documentation for details.
+
+.. note::
+
+  The examples below use ``curl`` command for simplicity and ease of testing.
+  Query parameters are URL-encoded, but the original pre-encoding values are
+  shown for reference. These examples assume genieacs-nbi is running locally
+  and listening on the default NBI port (7557).
+
+.. warning::
+
+  A common pitfall is not properly percent-encoding special characters in the
+  device ID or query in the URL.
+
+Endpoints
+---------
+
+GET /\<collection\>/?query=\<query\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Search for records in the database (e.g. devices, tasks, presets, files).
+Returns a JSON representation of all items in the given collection that match
+the search criteria.
+
+*collection*: The data collection to search. Could be one of: tasks, devices,
+presets, objects.
+
+*query*: Search query. Refer to MongoDB queries for reference.
+
+Examples
+^^^^^^^^
+
+- Find a device by its ID:
+
+.. code:: javascript
+
+  query = {"_id": "202BC1-BM632w-000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D'
+
+- Find a device by its MAC address:
+
+.. code:: javascript
+
+  query = {
+    "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress": "20:2B:C1:E0:06:65"
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress%22%3A%2220:2B:C1:E0:06:65%22%7D'
+
+- Search for devices that have not initiated an inform in the last 7 days.
+
+.. code:: javascript
+
+  query = {
+    "_lastInform": {
+      "$lt" : "2017-12-11 13:16:23 +0000"
+    }
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_lastInform%22%3A%7B%22%24lt%22%3A%222017-12-11%2013%3A16%3A23%20%2B0000%22%7D%7D'
+
+- Show pending tasks for a given device:
+
+.. code:: javascript
+
+  query = {"device": "202BC1-BM632w-000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-000000%22%7D'
+
+- Return specific parameters for a given device:
+
+.. code:: javascript
+
+  query = {"_id": "202BC1-BM632w-000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
+
+The ``projection`` URL param is a comma-separated list of the parameters to receive.
+
+POST /devices/\<device_id\>/tasks?[connection_request]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Enqueue task(s) and optionally trigger a connection request to the device.
+Refer to :ref:`tasks` section for information about the task object format.
+Returns status code 200 if the tasks have been successfully executed, and 202
+if the tasks have been queued to be executed at the next inform.
+
+*device_id*: The ID of the device.
+
+*connection_request*: Indicates that a connection request will be triggered to
+execute the tasks immediately. Otherwise, the tasks will be queued and be
+processed at the next inform.
+
+The response body is the task object as it is inserted in the database. The
+object will include ``_id`` property which you can use to look up the task
+later.
+
+Examples
+^^^^^^^^
+
+- Refresh all device parameters now:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
+  -X POST \
+  --data '{"name": "refreshObject", "objectName": ""}'
+
+- Change WiFi SSID and password:
+
+.. code:: javascript
+
+  {
+    "name": "setParameterValues",
+    "parameterValues": [
+      ["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],
+      ["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
+  -X POST \
+  --data '{"name":"setParameterValues", "parameterValues": [["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]]}'
+
+POST /tasks/\<task_id\>/retry
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Retry a faulty task at the next inform.
+
+*task_id*: The ID of the task as returned by 'GET /tasks' request.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/5403908ef28ea3a25c138adc/retry' -X POST
+
+DELETE /tasks/\<task_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given task.
+
+*task_id*: The ID of the task as returned by 'GET /tasks' request.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/5403908ef28ea3a25c138adc' -X DELETE
+
+DELETE /faults/\<fault_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given fault.
+
+*fault_id*: The ID of the fault as returned by 'GET /faults' request. The ID
+format is "\<device_id\>:\<channel\>".
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/faults/202BC1-BM632w-000000:default' -X DELETE
+
+DELETE /devices/\<device_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given device from the database.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -X DELETE -i 'http://localhost:7557/devices/202BC1-BM632w-000001'
+
+.. note::
+
+  Note that the device will be registered again when/if it contacts the ACS
+  again (e.g. on the next periodic inform).
+
+POST /devices/\<device_id\>/tags/\<tag\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Assign a tag to a device. Has no effect if such tag already exists.
+
+*device_id*: The ID of the device.
+
+*tag*: The tag to be assigned.
+
+Example
+^^^^^^^
+
+Assign the tag "testing" to a device:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X POST
+
+DELETE /devices/\<device_id\>/tags/\<tag\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Remove a tag from a device.
+
+*device_id*: The ID of the device.
+
+*tag*: The tag to be removed.
+
+Example
+^^^^^^^
+
+Remove the tag "testing" from a device:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X DELETE
+
+PUT /presets/\<preset_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Create or update a preset. Returns status code 200 if the preset has been
+added/updated successfully. The body of the request is a JSON representation of
+the preset. Refer to :ref:`presets` section below for details about its format.
+
+*preset_name*: The name of the preset.
+
+Example
+^^^^^^^
+
+Create a preset to set 5 minutes inform interval for all devices tagged with
+"test":
+
+.. code:: javascript
+
+  query = {
+    "weight": 0,
+    "precondition": "{\"_tags\": \"test\"}"
+    "configurations": [
+      {
+        "type": "value",
+        "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable",
+        "value": "true"
+      },
+      {
+        "type": "value",
+        "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval",
+        "value": "300"
+      }
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/presets/inform' \
+  -X PUT \
+  --data '{"weight": 0, "precondition": "{\"_tags\": \"test\"}", "configurations": [{"type": "value", "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable", "value": "true"}, {"type": "value", "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval", "value": "300"}]}'
+
+DELETE /presets/\<preset_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+	curl -i 'http://localhost:7557/presets/inform' -X DELETE
+
+PUT /files/\<file_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Upload a new file or overwrite an existing one. Returns status code 200 if the
+file has been added/updated successfully. The file content should be sent as
+the request body.
+
+*file_name*: The name of the uploaded file.
+
+The following file metadata may be sent as request headers:
+
+- ``fileType``: For firmware images it should be "1 Firmware Upgrade Image".
+  Other common types are "2 Web Content" and "3 Vendor Configuration File".
+
+- ``oui``: The OUI of the device model that this file belongs to.
+
+- ``productClass``: The product class of the device.
+
+- ``version``: In case of firmware images, this refer to the firmware version.
+
+Example
+^^^^^^^
+
+Upload a firmware image file:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/files/new_firmware_v1.0.bin' \
+  -X PUT \
+  --data-binary @"./new_firmware_v1.0.bin" \
+  --header "fileType: 1 Firmware Upgrade Image" \
+  --header "oui: 123456" \
+  --header "productClass: ABC" \
+  --header "version: 1.0"
+
+DELETE /files/\<file_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete a previously uploaded file:
+
+.. code:: bash
+
+	curl -i 'http://localhost:7557/files/new_firmware_v1.0.bin' -X DELETE
+
+GET /files/
+~~~~~~~~~~~
+
+Gets all previously uploaded files.
+
+GET /files/?query={"filename":"\<filename\>"}
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Find files using a query.
+
+.. _tasks:
+
+Tasks
+-----
+
+Find the different available tasks and their object structure.
+
+``getParameterValues``
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  query = {
+    "name": "getParameterValues",
+    "parameterNames": [
+      "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnectionNumberOfEntries",
+      "InternetGatewayDevice.Time.NTPServer1", "InternetGatewayDevice.Time.Status"
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-96318REF-SR360NA0A4%252D0003196/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "getParameterValues", "parameterNames": ["InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnectionNumberOfEntries", "InternetGatewayDevice.Time.NTPServer1", "InternetGatewayDevice.Time.Status"] }'
+
+You may request a single or multiple parameters at once.
+
+After the task has been executed successfully you can then fetch the CPE object
+and read the parameters from the JSON object.
+
+.. code:: javascript
+
+  query = {"_id": "00236a-96318REF-SR360NA0A4%2D0003196"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%2200236a-96318REF-SR360NA0A4%252D0003196%22%7D'
+
+``refreshObject``
+~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "refreshObject", "objectName": "InternetGatewayDevice.WANDevice.1.WANConnectionDevice"}'
+
+``setParameterValues``
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "setParameterValues", "parameterValues": [["InternetGatewayDevice.ManagementServer.UpgradesManaged",false]]}'
+
+Multiple values can be set at once by adding multiple arrays to the
+parameterValues key. For example:
+
+.. code:: javascript
+
+  {
+    name: "setParameterValues",
+    parameterValues: [["InternetGatewayDevice.ManagementServer.UpgradesManaged", false], ["InternetGatewayDevice.Time.Enable", true], ["InternetGatewayDevice.Time.NTPServer1", "pool.ntp.org"]]
+  }
+
+``addObject``
+~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name":"addObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection"}'
+
+``deleteObject``
+~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name":"deleteObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection.1"}'
+
+``reboot``
+~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "reboot"}'
+
+``factoryReset``
+~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "factoryReset"}'
+
+``download``
+~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "download", "file": "mipsbe-6-42-lite.xml"}'
+
+.. _presets:
+
+Presets
+-------
+
+Presets assign a set of configuration or a Provision script to devices based on
+a precondition (search filter), schedule (cron expression), and events.
+
+Precondition
+~~~~~~~~~~~~
+
+The ``precondition`` property is a JSON string representation of the search
+filter to test if the preset applies to a given device. Examples preconditions
+are:
+
+- ``{"param": "value"}``
+- ``{"param": value", "param2": {"$ne": "value2"}}``
+
+Other operators that can be used are ``$gt``, ``$lt``, ``$gte`` and ``$lte``.
+
+Configuration
+~~~~~~~~~~~~~
+
+The configuration property is an array containing the different configurations
+to be applied to a device, as shown below:
+
+.. code:: javascript
+
+  [
+    {
+      "type": "value",
+      "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable",
+      "value": "true"
+    },
+    {
+      "type": "value",
+      "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval",
+      "value": "300"
+    },
+    {
+      "type": "delete_object",
+      "name": "object_parent",
+      "object": "object_name"
+    },
+    {
+      "type": "add_object",
+      "name": "object_parent",
+      "object": "object_name"
+    },
+    {
+      "type": "provision",
+      "name": "YourProvisionName"
+    },
+  ] 
+
+The configuration type ``provision`` triggers a Provision script. In the
+example above, the provision named "YourProvisionName" will be executed.
+
+Provisions
+----------
+
+Create a provision
+~~~~~~~~~~~~~~~~~~
+
+The Provision's JavaScript code is the body of the HTTP PUT request.
+
+.. code:: bash
+
+  curl -X PUT -i 'http://localhost:7557/provisions/mynewprovision' --data 'log("Provision started at " + now);'
+
+Delete a provision
+~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -X DELETE -i 'http://localhost:7557/provisions/mynewprovision'
+
+Get provisions
+~~~~~~~~~~~~~~
+
+Get all provisions:
+
+.. code:: bash
+
+  curl -X GET -i 'http://localhost:7557/provisions/'

docs/conf.py

@@ -0,0 +1,65 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+import json
+
+# -- Project information -----------------------------------------------------
+
+project = 'GenieACS Documentation'
+copyright = '2024, GenieACS Inc.'
+author = 'GenieACS Inc.'
+
+# The full version, including alpha/beta/rc tags
+release = json.load(open("../package.json"))["version"]
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+# html_theme = 'alabaster'
+html_theme = 'sphinx_rtd_theme'
+html_logo = "logo.svg"
+html_theme_options = {
+  "logo_only": True
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']
+
+master_doc = 'index'
+
+highlight_language = 'javascript'

docs/cpe-authentication.rst

@@ -0,0 +1,72 @@
+.. _cpe-authentication:
+
+CPE Authentication
+==================
+
+CPE to ACS
+----------
+
+.. note::
+
+  By default GenieACS will accept any incoming connection via HTTP/HTTPS and
+  respond to it.
+
+The following parameters are used to set and get (password is redacted but
+can be set) the username/password used to authenticate against the ACS:
+
+Username: ``Device.ManagementServer.Username`` or ``InternetGatewayDevice.ManagementServer.Username``
+
+Password: ``Device.ManagementServer.Password`` or ``InternetGatewayDevice.ManagementServer.Password``
+
+Enable CPE to ACS Authentication
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+CPE to ACS authentication can be configured in the web interface by using the
+`Config` option in the `Admin` tab.
+
+Go to the `Admin` -> `Config` page and click on `New config` button at the
+bottom of the page. This will open pop-up which requires you to fill in a key
+and value. The key should be ``cwmp.auth``. The value accepts a boolean.
+Setting the value to ``true`` makes it so that GenieACS accepts any incoming
+connection, setting it to ``false`` makes GenieACS deny all incoming
+connections. This can be further configured using the ``AUTH()`` and ``EXT()``
+functions.
+
+The ``AUTH()`` function
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``AUTH()`` function accepts two parameters, username and password. It
+checks the given username and password with the incoming request to determine
+whether to return true or false.
+
+Basic usage of the ``AUTH()`` function could be as follows:
+
+.. code:: sql
+
+   AUTH("fixed-username", "fixed-password")
+
+This will only accept incoming request who authenticate with
+"fixed-username" and "fixed-password".
+
+The various device parameters can be referenced from within the ``cwmp.auth``
+expression. For example:
+
+.. code:: sql
+
+   AUTH(Device.ManagementServer.Username, Device.ManagementServer.Password)
+
+The ``EXT()`` function
+~~~~~~~~~~~~~~~~~~~~~~
+
+The ``EXT()`` function makes it possible to call an :ref:`extension
+<extensions>` script from the auth expression. This can be used to fetch
+the credentials from an external source:
+
+.. code:: sql
+
+   AUTH(DeviceID.SerialNumber, EXT("authenticate", "getPassword", DeviceID.SerialNumber))
+
+ACS to CPE
+----------
+
+TODO

docs/environment-variables.rst

@@ -0,0 +1,217 @@
+.. _environment-variables:
+
+Environment Variables
+=====================
+
+Configuring GenieACS services can be done through the following environment
+variables:
+
+.. attention::
+
+  All GenieACS environment variables must be prefixed with ``GENIEACS_``.
+
+MONGODB_CONNECTION_URL
+  MongoDB connection string.
+
+  Default: ``mongodb://127.0.0.1/genieacs``
+
+EXT_DIR
+  The directory from which to look up extension scripts.
+
+  Default: ``<installation dir>/config/ext``
+
+EXT_TIMEOUT
+  Timeout (in milliseconds) to allow for calls to extensions to return a
+  response.
+
+  Default: ``3000``
+
+DEBUG_FILE
+  File to dump CPE debug log.
+
+  Default: unset
+
+DEBUG_FORMAT
+  Debug log format. Valid values are 'yaml' and 'json'.
+
+  Default: ``yaml``
+
+LOG_FORMAT
+  The format used for the log entries in ``CWMP_LOG_FILE``, ``NBI_LOG_FILE``,
+  ``FS_LOG_FILE``, and ``UI_LOG_FILE``. Possible values are ``simple`` and
+  ``json``.
+
+  Default: ``simple``
+
+ACCESS_LOG_FORMAT
+  The format used for the log entries in ``CWMP_ACCESS_LOG_FILE``,
+  ``NBI_ACCESS_LOG_FILE``, ``FS_ACCESS_LOG_FILE``, and ``UI_ACCESS_LOG_FILE``.
+  Possible values are ``simple`` and ``json``.
+
+  Default: ``simple``
+
+CWMP_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-cwmp. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+CWMP_PORT
+  The TCP port that genieacs-cwmp listens on.
+
+  Default: ``7547``
+
+CWMP_INTERFACE
+  The network interface that genieacs-cwmp binds to.
+
+  Default: ``::``
+
+CWMP_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_LOG_FILE
+  File to log process related events for genieacs-cwmp. If omitted, logs will
+  go to stderr.
+
+  Default: unset
+
+CWMP_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-cwmp. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+NBI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-nbi. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+NBI_PORT
+  The TCP port that genieacs-nbi listens on.
+
+  Default: ``7557``
+
+NBI_INTERFACE
+  The network interface that genieacs-nbi binds to.
+
+  Default: ``::``
+
+NBI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+NBI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+NBI_LOG_FILE
+  File to log process related events for genieacs-nbi. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+NBI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-nbi. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-fs. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+FS_PORT
+  The TCP port that genieacs-fs listens on.
+
+  Default: ``7567``
+
+FS_INTERFACE
+  The network interface that genieacs-fs binds to.
+
+  Default: ``::``
+
+FS_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_LOG_FILE
+  File to log process related events for genieacs-fs. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+FS_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-fs. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_URL_PREFIX
+  The URL prefix (e.g. 'https://example.com:7567/') to use when generating the
+  file URL for TR-069 Download requests. Set this if genieacs-fs and
+  genieacs-cwmp are behind a proxy or running on different servers.
+
+  Default: auto generated based on the hostname from the ACS URL, FS_PORT
+  config, and whether or not SSL is enabled for genieacs-fs.
+
+UI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-ui. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+UI_PORT
+  The TCP port that genieacs-ui listens on.
+
+  Default: ``3000``
+
+UI_INTERFACE
+  The network interface that genieacs-ui binds to.
+
+  Default: ``::``
+
+UI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_LOG_FILE
+  File to log process related events for genieacs-ui. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+UI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-ui. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+UI_JWT_SECRET
+  The key used for signing JWT tokens that are stored in browser cookies. The
+  string can be up to 64 characters in length.
+
+  Default: unset

docs/ext-sample.js

@@ -0,0 +1,37 @@
+// This is an example GenieACS extension to get the current latitude/longitude
+// of the International Space Station. Why, you ask? Because why not.
+// To install, copy this file to config/ext/iss.js.
+
+"use strict";
+
+const http = require("http");
+
+let cache = null;
+let cacheExpire = 0;
+
+function latlong(args, callback) {
+  if (Date.now() < cacheExpire) return callback(null, cache);
+
+  http
+    .get("http://api.open-notify.org/iss-now.json", (res) => {
+      if (res.statusCode !== 200)
+        return callback(
+          new Error(`Request failed (status code: ${res.statusCode})`),
+        );
+
+      let rawData = "";
+      res.on("data", (chunk) => (rawData += chunk));
+
+      res.on("end", () => {
+        let pos = JSON.parse(rawData)["iss_position"];
+        cache = [+pos["latitude"], +pos["longitude"]];
+        cacheExpire = Date.now() + 10000;
+        callback(null, cache);
+      });
+    })
+    .on("error", (err) => {
+      callback(err);
+    });
+}
+
+exports.latlong = latlong;

docs/extensions.rst

@@ -0,0 +1,33 @@
+.. _extensions:
+
+Extensions
+==========
+
+Given that :ref:`provisions` and :ref:`virtual-parameters` are executed in a
+sandbox environment, it is not possible to interact with external sources or
+execute any action that requires OS, file system, or network access. Extensions
+exist to bridge that gap.
+
+Extensions are fully-privileged Node.js modules and as such have access to
+standard Node libraries and 3rd party packages. Functions exposed by the
+extension can be called from Provision scripts using the ``ext()`` function. A
+typical use case for extensions is fetching credentials from a database to have
+that pushed to the device during provisioning.
+
+By default, the extension JS code must be placed under ``config/ext``
+directory. You may need to create that directory if it doesn't already exist.
+
+The example extension below fetches data from an external REST API and returns
+that to the caller:
+
+.. literalinclude:: ext-sample.js
+  :language: javascript
+
+To call this extension from a Provision or a Virtual Parameter script:
+
+.. code:: javascript
+
+  // The arguments "arg1" and "arg2" are passed to the latlong. Though they are
+  // unused in this particular example.
+  const res = ext("ext-sample", "latlong", "arg1", "arg2");
+  log(JSON.stringify(res));

docs/https.rst

@@ -0,0 +1,6 @@
+.. _https:
+
+HTTPS
+=====
+
+TODO

docs/index.rst

@@ -0,0 +1,56 @@
+.. GenieACS documentation master file, created by
+   sphinx-quickstart on Wed Jun  5 13:47:06 2019.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to GenieACS's documentation!
+====================================
+
+.. toctree::
+  :caption: Table of Contents
+
+.. raw:: latex
+
+   \part{Installation}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Installation
+
+  installation-guide
+  environment-variables
+
+.. raw:: latex
+
+   \part{Administration}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Administration
+
+  provisions
+  virtual-parameters
+  administration-faq
+
+.. raw:: latex
+
+   \part{Integration}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Integration
+
+  extensions
+  api-reference
+
+.. raw:: latex
+
+   \part{Security}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Security
+
+  https
+  cpe-authentication
+  roles-and-permissions

docs/installation-guide.rst

@@ -0,0 +1,238 @@
+Installation Guide
+==================
+
+This guide is for installing GenieACS on a single server on any Linux distro
+that uses *systemd* as its init system.
+
+The various GenieACS services are independent of each other and may be
+installed on different servers. You may also run multiple instances of each in
+a load-balancing/failover setup.
+
+.. attention::
+
+  For production deployments make sure to configure TLS and change
+  ``UI_JWT_SECRET`` to a unique and secure string. Refer to :ref:`https`
+  section for how to enable TLS to encrypt traffic.
+
+Prerequisites
+-------------
+
+.. topic:: Node.js
+
+  GenieACS requires Node.js 12.13 and up. Refer to https://nodejs.org/ for
+  instructions.
+
+.. topic:: MongoDB
+
+  GenieACS requires MongoDB 3.6 and up. Refer to https://www.mongodb.com/ for
+  instructions.
+
+Install GenieACS
+-------------------
+
+.. topic:: Installing from NPM:
+
+  .. parsed-literal::
+
+    sudo npm install -g genieacs@\ |release|
+
+.. topic:: Installing from source
+
+  If you prefer installing from source, such as when running a GenieACS copy
+  with custom patches, refer to README.md file in the source package. Adjust
+  the next steps below accordingly.
+
+Configure systemd
+-----------------
+
+.. topic:: Create a system user to run GenieACS daemons
+
+  .. code:: bash
+
+    sudo useradd --system --no-create-home --user-group genieacs
+
+.. topic:: Create directory to save extensions and environment file
+
+  We'll use :file:`/opt/genieacs/ext/` directory to store extension scripts (if any).
+
+  .. code:: bash
+    
+    mkdir /opt/genieacs
+    mkdir /opt/genieacs/ext
+    chown genieacs:genieacs /opt/genieacs/ext
+
+  Create the file :file:`/opt/genieacs/genieacs.env` to hold our configuration
+  options which we pass to GenieACS as environment variables. See
+  :ref:`environment-variables` section for a list of all available
+  configuration options.
+
+  .. code:: bash
+
+    GENIEACS_CWMP_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-cwmp-access.log
+    GENIEACS_NBI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-nbi-access.log
+    GENIEACS_FS_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-fs-access.log
+    GENIEACS_UI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-ui-access.log
+    GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
+    NODE_OPTIONS=--enable-source-maps
+    GENIEACS_EXT_DIR=/opt/genieacs/ext
+
+  Generate a secure JWT secret and append to :file:`/opt/genieacs/genieacs.env`:
+
+  .. code:: bash
+
+    node -e "console.log(\"GENIEACS_UI_JWT_SECRET=\" + require('crypto').randomBytes(128).toString('hex'))" >> /opt/genieacs/genieacs.env
+  
+  Set file ownership and permissions:
+
+  .. code:: bash
+
+    sudo chown genieacs:genieacs /opt/genieacs/genieacs.env
+    sudo chmod 600 /opt/genieacs/genieacs.env
+
+.. topic:: Create logs directory
+
+  .. code:: bash
+    
+    mkdir /var/log/genieacs
+    chown genieacs:genieacs /var/log/genieacs
+
+.. topic:: Create systemd unit files
+
+  Create a systemd unit file for each of the four GenieACS services. Note that
+  we're using EnvironmentFile directive to read the environment variables from
+  the file we created earlier.
+
+  Each service has two streams of logs: access log and process log. Access logs
+  are configured here to be dumped in a log file under
+  :file:`/var/log/genieacs/` while process logs go to *journald*. Use
+  ``journalctl`` command to view process logs.
+
+  .. attention::
+
+    If the command :command:`systemctl edit --force --full` fails, you can
+    create the unit file manually.
+
+  1. Run the following command to create ``genieacs-cwmp`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-cwmp
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS CWMP
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-cwmp
+
+      [Install]
+      WantedBy=default.target
+
+  2. Run the following command to create ``genieacs-nbi`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-nbi
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS NBI
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-nbi
+
+      [Install]
+      WantedBy=default.target
+
+  3. Run the following command to create ``genieacs-fs`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-fs
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS FS
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-fs
+
+      [Install]
+      WantedBy=default.target
+
+  4. Run the following command to create ``genieacs-ui`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-ui
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS UI
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-ui
+
+      [Install]
+      WantedBy=default.target
+
+.. topic:: Configure log file rotation using logrotate
+
+  Save the following as :file:`/etc/logrotate.d/genieacs`
+
+  .. code::
+  
+    /var/log/genieacs/*.log /var/log/genieacs/*.yaml {
+        daily
+        rotate 30
+        compress
+        delaycompress
+        dateext
+    }
+
+.. topic:: Enable and start services
+
+  .. code:: bash
+
+    sudo systemctl enable genieacs-cwmp
+    sudo systemctl start genieacs-cwmp
+    sudo systemctl status genieacs-cwmp
+
+    sudo systemctl enable genieacs-nbi
+    sudo systemctl start genieacs-nbi
+    sudo systemctl status genieacs-nbi
+
+    sudo systemctl enable genieacs-fs
+    sudo systemctl start genieacs-fs
+    sudo systemctl status genieacs-fs
+
+    sudo systemctl enable genieacs-ui
+    sudo systemctl start genieacs-ui
+    sudo systemctl status genieacs-ui
+
+  Review the status message for each to verify that the services are running
+  successfully.

docs/logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="310" height="64" viewBox="0 0 82.021 16.933"><path d="M7.341 9.892l.687 1.307c-.305.53-.718.99-1.238 1.378-.851.503-1.543.421-2.076-.244 1.03-.466 1.907-1.28 2.627-2.44M4.676 5.97a2.144 2.144 0 0 1 .237 1.228 8.358 8.358 0 0 1-.518 2.21 7.386 7.386 0 0 0-.424 1.708 2.38 2.38 0 0 0 .168 1.205c.163.393.457.73.883 1.01-.458.138-.859.175-1.202.112-.6-.114-.992-.476-1.175-1.084a3.571 3.571 0 0 1-.093-1.787 11.742 11.742 0 0 1 .435-1.644 11.606 11.606 0 0 0 .432-1.708c.037-.22.041-.45.01-.687a1.66 1.66 0 0 0-.273-.706 1.582 1.582 0 0 0-.6-.506c.245-.046.487-.083.728-.113a1.45 1.45 0 0 1 .728.123c.295.13.517.343.665.639m7.901-1.802l.222.218-1.847 1.678.555.555 1.678-1.846.218.22-1.843 1.679.521.521L13.76 5.35l.222.218-1.847 1.678.556.556 1.764-1.765.375.368-1.584 1.584h.004l-2.065 2.061-1.253.515-1.795-1.348-.007-.008-.008.008.004.007 1.194 1.592-.398.161-1.75-1.75.444-1.08 1.636 1.228.004.003.011-.007-.003-.004-1.39-1.85.225-.553 1.694-1.689h-.004l1.956-1.952.372.368-1.769 1.768.552.552 1.678-1.843m1.456 9.91c1.55-1.549 2.324-3.42 2.324-5.612 0-2.187-.774-4.058-2.323-5.612C12.48 1.304 10.61.53 8.423.53 6.23.53 4.36 1.305 2.81 2.854c-.95.953-1.609 2.025-1.974 3.213l.158-.086c.788-.415 1.486-.203 2.094.638-.55.113-.992.232-1.325.357a1.923 1.923 0 0 0-.503.289c-.337.268-.55.608-.638 1.02a3.98 3.98 0 0 0 .011 1.761c.285 1.527 1.011 2.87 2.178 4.032 1.549 1.552 3.42 2.328 5.612 2.328 2.19 0 4.06-.776 5.612-2.328" fill="white"/><path d="M47.104 1.767c-.456 0-.816.388-.816.849 0 .46.355.816.816.816a.82.82 0 0 0 .833-.816.85.85 0 0 0-.833-.849zM25.602 3.171c-.205 0-.382.046-.5.13-.118.085-.187.201-.187.4v.981l-.151-.031a9.89 9.89 0 0 0-1.872-.173c-1.288 0-2.148.066-2.657.374a1.264 1.264 0 0 0-.548.676c-.119.313-.177.73-.177 1.28v6.123c0 .982.25 1.508.787 1.824.539.315 1.407.41 2.628.41 1.308 0 2.178-.13 2.701-.484.524-.354.745-.932.745-1.942 0-.821-.147-1.37-.543-1.722-.395-.352-1.075-.528-2.186-.528h-2.787V9.255l.148.03c.478.092 1.094.11 1.89.11 1.292 0 2.148-.112 2.658-.453.51-.34.724-.915.724-1.958V3.7c0-.198-.07-.314-.186-.399a.843.843 0 0 0-.487-.13zm6.112 1.29c-1.226 0-2.038.137-2.526.495-.488.357-.697.945-.697 1.947v2.84c0 .935.23 1.54.743 1.932.512.39 1.337.571 2.544.571 1.392 0 2.17-.058 2.567-.196.199-.069.298-.15.355-.246.058-.097.078-.227.078-.405 0-.208-.04-.378-.117-.485-.076-.108-.186-.171-.396-.171-.794 0-1.057.19-2.408.19-.593 0-1.06-.078-1.395-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.717.717 0 0 0 .539-.213c.13-.138.213-.342.213-.62V6.903c0-1.037-.179-1.623-.654-1.97-.476-.348-1.3-.472-2.617-.472zm8.83 0c-1.303 0-2.117.086-2.59.49-.472.403-.665 1.168-.665 2.623v4.002c0 .215.074.338.197.426.123.088.31.134.524.134.223 0 .412-.046.537-.134.125-.088.198-.211.198-.426v-3.81c0-.424.006-.754.04-1.019.036-.265.102-.47.236-.623.135-.152.328-.236.574-.28.246-.044.552-.053.949-.053.4 0 .712.01.963.053.25.044.447.126.587.278.139.151.211.358.25.623.038.265.046.597.046 1.02v3.81c0 .217.07.34.19.427.122.088.308.134.53.134.215 0 .401-.046.525-.134.123-.088.196-.211.196-.426V7.574c0-1.455-.201-2.219-.682-2.623-.48-.404-1.303-.49-2.605-.49zm12.996 0c-1.227 0-2.038.137-2.527.495-.488.357-.696.945-.696 1.947v2.84c0 .935.23 1.54.742 1.932.513.39 1.338.571 2.545.571 1.392 0 2.17-.058 2.567-.196.2-.069.298-.15.355-.246.058-.097.079-.227.079-.405 0-.208-.04-.378-.117-.485-.077-.108-.188-.171-.398-.171-.793 0-1.056.19-2.406.19-.594 0-1.062-.078-1.396-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.716.716 0 0 0 .539-.213c.13-.138.213-.342.213-.62V6.903c0-1.037-.18-1.623-.655-1.97-.475-.348-1.3-.472-2.616-.472zm-6.436.13c-.212 0-.398.047-.522.14-.124.09-.198.22-.198.435v6.41c0 .215.073.338.196.426.123.088.31.134.524.134.223 0 .414-.046.54-.134.124-.088.198-.211.198-.426v-6.41c0-.215-.075-.342-.201-.434-.126-.092-.316-.142-.537-.142zM22.892 5.726c.662 0 1.4.065 1.918.146l.105.016v.92c0 .246-.01.453-.058.632a.757.757 0 0 1-.292.441c-.305.216-.798.263-1.673.263h-.123v-.013c-.761 0-1.26-.002-1.55-.16a.637.637 0 0 1-.307-.395c-.049-.17-.057-.37-.057-.625v-.062c0-.227.03-.418.108-.58a.811.811 0 0 1 .373-.373c.337-.171.826-.21 1.556-.21zm8.839.049c.677 0 1.116.054 1.412.237a.775.775 0 0 1 .316.373c.063.154.085.33.085.535v.811h-3.611V6.92c0-.437.135-.755.439-.928.304-.174.739-.217 1.359-.217zm21.825 0c.677 0 1.117.054 1.413.237a.779.779 0 0 1 .318.373c.062.154.084.33.084.535v.811h-3.613V6.92c0-.437.136-.755.44-.928.304-.174.738-.217 1.358-.217zm-32.701 5.867h2.388c.613 0 1.05.045 1.355.226.306.18.443.505.443.95 0 .429-.179.743-.533.907-.354.164-.861.206-1.568.206-.652 0-1.151-.045-1.51-.186a.952.952 0 0 1-.425-.3.789.789 0 0 1-.15-.483v-1.195z" fill="white"/><path d="M62.339 4.462c-1.399 0-2.18.078-2.58.225-.2.073-.3.157-.357.254-.058.097-.079.22-.079.386 0 .199.043.362.122.469.079.107.19.172.392.172.38 0 .616-.048.947-.096a9.457 9.457 0 0 1 1.475-.098c.58 0 .966.043 1.233.192a.685.685 0 0 1 .292.312c.06.13.082.281.082.452v.857h-2.484c-1.134 0-1.763.19-2.11.559-.346.367-.443.95-.443 1.802 0Line truncated

docs/provisions.rst

@@ -0,0 +1,279 @@
+.. _provisions:
+
+Provisions
+==========
+
+A Provision is a piece of JavaScript code that is executed on the server on a
+per-device basis. It enables implementing complex provisioning scenarios and
+other operations such as automated firmware upgrade rollout. Apart from a few
+special functions, the script is essentially a standard ES6 code executed in
+strict mode.
+
+Provisions are mapped to devices using presets. Note that the added performance
+overhead when using Provisions as opposed to simple preset configuration
+entries is relatively small. Anything that can be done via preset
+configurations can be done using a Provision script. In fact, the now
+deprecated configuration format is still supported primarily for backward
+compatibility and it is recommended to use Provision scripts for all
+configuration.
+
+When assigning a Provision script to a preset, you may pass arguments to the
+script. The arguments can be accessed from the script through the global
+``args`` variable.
+
+.. note::
+
+  Provision scripts may get executed multiple times in a given session.
+  Although all data model-mutating operations are idempotent, a script as a
+  whole may not be. It is, therefore, necessary to repeatedly run the script
+  until there are no more side effects and a stable state is reached.
+
+Built-in functions
+------------------
+
+``declare(path, timestamps, values)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This function is for declaring parameter values to be set, as well as specify
+constraints on how recent you'd like the parameter value (or other attributes)
+to have been refreshed from the device. If the given timestamp is lower than
+the timestamp of the last refresh from the device, then this function will
+return the last known value. Otherwise, the value will be fetched from the
+device before being returned to the caller.
+
+The timestamp argument is an object where the key is the attribute name (e.g.
+``value``, ``object``, ``writable``, ``path``) and the value is an integer
+representing a Unix timestamp.
+
+The values argument is an object similar to the timestamp argument but its
+property values being the parameter values to be set.
+
+The possible attributes in 'timestamps' and 'values' arguments are:
+
+- ``value``: a [<value>, <type>] pair
+
+This attribute is not available for objects or object instances. If the value
+is not a [<value>, <type>] array then it'll assumed to be a value without a
+type and therefore the type will be inferred from the parameter's type.
+
+- ``writable``: boolean
+
+The meaning of this attribute can vary depending on the type of the parameter.
+In the case of regular parameters, it indicates if its value is writable. In
+the case of objects, it's whether or not it's possible to add new object
+instances. In the case of object instances, it indicates whether or not this
+instance can be deleted.
+
+- ``object``: boolean
+
+True if this is an object or object instance, false otherwise.
+
+- ``path``: string
+
+This attribute is special in that it's not a parameter attribute per se, but it
+refers to the presence of parameters matching the given path. For example,
+given the following wildcard path:
+
+``InternetGatewayDevice.LANDevice.1.Hosts.Host.*.MACAddress``
+
+Using a recent timestamp for path in ``declare()`` will result in a sync with
+the device to rediscover all Host instances (``Host.*``). The path attribute
+can also be used to create or delete object instances as described in
+:ref:`path-format` section.
+
+The return value of ``declare()`` is an iterator to access parameters that
+match the given path. Each item in the iterator has the attribute 'path' in
+addition to any other attribute given in the ``declare()`` call. The iterator
+object itself has convenience attribute accessors which come in handy when
+you're expecting a single parameter (e.g. when path does not contain wildcards
+or aliases).
+
+.. code:: javascript
+
+  // Example: Setting the SSID as the last 6 characters of the serial number
+  let serial = declare("Device.DeviceInfo.SerialNumber", {value: 1});
+  declare("Device.LANDevice.1.WLANConfiguration.1.SSID", null, {value: serial.value[0]});
+
+``clear(path, timestamp)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This function invalidates the database copy of parameters (and their child
+parameters) that match the given path and have a last refresh timestamp that is
+less than the given timestamp. The most obvious use for this function is to
+invalidate the database copy of the entire data model after the device has been
+factory reset:
+
+.. code:: javascript
+
+  // Example: Clear cached device data model Note
+  // Make sure to apply only on "0 BOOTSTRAP" event
+  clear("Device", Date.now());
+  clear("InternetGatewayDevice", Date.now());
+
+``commit()``
+~~~~~~~~~~~~
+
+This function commits the pending declarations and performs any necessary sync
+with the device. It's usually not required to call this function as it called
+implicitly at the end of the script and when accessing any property of the
+promise-like object returned by the ``declare()`` function. Calling this
+explicitly is only necessary if you want to control the order in which
+parameters are configured.
+
+``ext(file, function, arg1, arg2, ...)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Execute an extension script and return the result. The first argument is the
+script filename while second argument is the function name within that script.
+Any remaining arguments will be passed to that function. See :ref:`extensions`
+for more details.
+
+``log(message)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Prints out a string in genieacs-cwmp's access log. It's meant to be used for
+debugging. Note that you may see multiple log entries as the script can be
+executed multiple times in a session. See :ref:`this FAQ
+<administration-faq-duplicate-log-entries>`.
+
+.. _path-format:
+
+Path format
+-----------
+
+A parameter path may contain a wildcard (``*``) or an alias filter
+(``[name:value]``). A wildcard segment in a parameter path will apply the
+declared configuration to zero or more parameters that match the given path
+where the wildcard segment can be anything.
+
+An alias filter is like a wildcard, but additionally performs filtering on the
+child parameters based on the key-value pairs provided. For example, the
+following path:
+
+``Device.WANDevice.1.WANConnectionDevice.1.WANIPConnection.[AddressingType:DHCP].ExternalIPAddress``
+
+will return a list of ExternalIPAddress parameters (0 or more) where the
+sibling parameter AddressingType is assigned the value "DHCP".
+
+This can be useful when the exact instance numbers may be different from one
+device to another. It is possible to use more than one key-value pair in the
+alias filter. It's also possible to use multiple filters or use a combination
+of filters and wildcards.
+
+Creating/deleting object instances
+----------------------------------
+
+Given the declarative nature of provisions, we cannot explicitly tell the
+device to create or delete an instance under a given object. Instead, we
+specify the number of instances we want there to be, and based on that GenieACS
+will determine whether or not it needs to create or delete instances. For
+example, the following declaration will ensure we have one and only one
+WANIPConnection object:
+
+.. code:: javascript
+
+  // Example: Ensure we have one and only one WANIPConnection object
+  declare("InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.*", null, {path: 1});
+
+Note the wildcard at the end of the parameter path.
+
+It is also possible to use alias filters as the last path segment which will
+ensure that the declared number of instances is satisfied given the alias
+filter:
+
+.. code:: javascript
+
+  // Ensure that *all* other instances are deleted
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[]", null, {path: 0});
+
+  // Add the two entries we care about
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[SourceIPAddress:192.168.1.0,SourceNetMask:255.255.255.0]",  {path: now}, {path: 1});
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[SourceIPAddress:172.16.12.0,SourceNetMask:255.255.0.0]", {path: now}, {path: 1});
+
+Special GenieACS parameters
+---------------------------
+
+In addition to the parameters exposed in the device's data model through
+TR-069, GenieACS has its own set of special parameters:
+
+``DeviceID``
+~~~~~~~~~~~~
+
+This parameter sub-tree includes the following read-only parameters:
+
+- ``DeviceID.ID``
+- ``DeviceID.SerialNumber``
+- ``DeviceID.ProductClass``
+- ``DeviceID.OUI``
+- ``DeviceID.Manufacturer``
+
+``Tags``
+~~~~~~~~
+
+The ``Tags`` root parameter is used to expose device tags in the data model.
+Tags appear as child parameters that are writable and have boolean value.
+Setting a tag to ``false`` will delete that tag, and setting the value of a
+non-existing tag parameter to ``true`` will create it.
+
+.. code:: javascript
+
+  // Example: Remove "tag1", add "tag2", and read "tag3"
+  declare("Tags.tag1", null, {value: false});
+  declare("Tags.tag2", null, {value: true});
+  let tag3 = declare("Tags.tag3", {value: 1});
+
+``Reboot``
+~~~~~~~~~~
+
+The ``Reboot`` root parameter hold the timestamp of the last reboot command.
+The parameter value is writable and declaring a timestamp value that is larger
+than the current value will trigger a reboot.
+
+.. code:: javascript
+
+  // Example: Reboot the device only if it hasn't been rebooted in the past 300 seconds
+  declare("Reboot", null, {value: Date.now() - (300 * 1000)});
+
+``FactoryReset``
+~~~~~~~~~~~~~~~~
+
+Works like ``Reboot`` parameter but for factory reset.
+
+.. code:: javascript
+
+  // Example: Default the device to factory settings
+  declare("FactoryReset", null, {value: Date.now()});
+
+``Downloads``
+~~~~~~~~~~~~~
+
+The ``Downloads`` sub-tree holds information about the last download
+command(s). A download command is represented as an instance (e.g.
+``Downloads.1``) containing parameters such as ``Download`` (timestamp),
+``LastFileType``, ``LastFileName``. The parameters ``FileType``, ``FileName``,
+``TargetFileName`` and ``Download`` are writable and can be used to trigger a
+new download.
+
+.. code:: javascript
+
+  declare("Downloads.[FileType:1 Firmware Upgrade Image]", {path: 1}, {path: 1});
+  declare("Downloads.[FileType:1 Firmware Upgrade Image].FileName", {value: 1}, {value: "firmware-2017.01.tar"});
+  declare("Downloads.[FileType:1 Firmware Upgrade Image].Download", {value: 1}, {value: Date.now()});
+
+Common file types are:
+
+- ``1 Firmware Upgrade Image``
+- ``2 Web Content``
+- ``3 Vendor Configuration File``
+- ``4 Tone File``
+- ``5 Ringer File``
+
+.. warning::
+
+  Pushing a file to the device is often a service-interrupting operation. It's
+  recommended to only trigger it on certain events such as ``1 BOOT`` or during
+  a predetermined maintenance window).
+
+After the CPE had finished downloading and applying the config file, it will
+send a ``7 TRANSFER COMPLETE`` event. You may use that to trigger a reboot
+after the firmware image or configuration file had been applied.

docs/requirements.txt

@@ -0,0 +1 @@
+sphinx_rtd_theme==2.0.0

docs/roles-and-permissions.rst

@@ -0,0 +1,6 @@
+.. _roles-and-permissions:
+
+Roles and Permissions
+=====================
+
+TODO

docs/virtual-parameters.rst

@@ -0,0 +1,105 @@
+.. _virtual-parameters:
+
+Virtual Parameters
+==================
+
+Virtual parameters are user-defined parameters whose values are generated using
+a custom Javascript code. Virtual parameters behave just like regular
+parameters and appear in the data model under ``VirtualParameters.`` path.
+Virtual parameter names cannot contain a period (``.``).
+
+The execution environment for virtual parameters is almost identical to that of
+provisions. See :ref:`provisions` for more details and examples. The only
+differences between the scripts of provisions and virtual parameters are:
+
+- You can't pass custom arguments to virtual parameter scripts. Instead, the
+  variable ``args`` will hold the current vparam timestamps and values as well
+  as the declared timestamps and values. Like this:
+
+.. code:: javascript
+
+  // [<declared attr timestamps, declared attr values>, <current attr timestamps>, <current attr values>]
+  [{path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}, {path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}]
+
+- Virtual parameter scripts must return an object containing the attributes of
+  this parameter.
+
+.. note::
+
+  Just like a regular parameter, creating a virtual parameter does not
+  automatically add it to the parameter list for a device. It needs to fetched
+  (manually or via a preset) before you can see it in the data model.
+
+Examples
+--------
+
+Unified MAC parameter across different device models
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Unified MAC parameter across different device models
+  let m = "00:00:00:00:00:00";
+  let d = declare("Device.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress", {value: Date.now()});
+  let igd = declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANPPPConnection.*.MACAddress", {value: Date.now()});
+
+  if (d.size) {
+    for (let p of d) {
+      if (p.value[0]) {
+        m = p.value[0];
+        break;
+      }
+    }  
+  }
+  else if (igd.size) {
+    for (let p of igd) {
+      if (p.value[0]) {
+        m = p.value[0];
+        break;
+      }
+    }  
+  }
+
+  return {writable: false, value: [m, "xsd:string"]};
+
+Expose an external value as a virtual parameter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Expose an external value as a virtual parameter
+  let serial = declare("DeviceID.SerialNumber", {value: 1});
+  if (args[1].value) {
+    ext("example-ext", "set", serial.value[0], args[1].value[0]);
+    return {writable: true, value: [args[1].value[0], "xsd:string"]};
+  }
+  else {
+    let v = ext("example-ext", "get", serial.value[0]);
+    return {writable: true, value: [v, "xsd:string"]};
+  }
+
+Create an editable virtual parameter for WPA passphrase
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Create an editable virtual parameter for WPA passphrase
+  let m = "";
+  if (args[1].value) {
+    m = args[1].value[0];
+    declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", null, {value: m});
+    declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", null, {value: m});
+  }
+  else {
+    let d = declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", {value: Date.now()});
+    let igd = declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", {value: Date.now()});
+
+    if (d.size) {
+      m = d.value[0];
+    }
+    else if (igd.size) {
+      m = igd.value[0];  
+    }
+  }
+
+  return {writable: true, value: [m, "xsd:string"]};

lib/api-functions.coffee

@@ -1,265 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-crypto = require 'crypto'
-dgram = require 'dgram'
-
-config = require './config'
-db = require './db'
-http = require 'http'
-common = require './common'
-util = require 'util'
-URL = require 'url'
-auth = require './auth'
-cache = require './cache'
-
-
-udpConReq = (address, un, key, callback) ->
-  return false if not address
-  [host, port] = address.split(':', 2)
-  if port is undefined
-    port = 80
-  else
-    port = parseInt(port)
-    return false if isNaN(port)
-
-  ts = Math.trunc(Date.now() / 1000)
-  id = Math.trunc(Math.random() * 4294967295)
-  cn = crypto.randomBytes(8).toString('hex')
-  sig = crypto.createHmac('sha1', key).update("#{ts}#{id}#{un}#{cn}").digest('hex')
-  uri = "http://#{address}?ts=#{ts}&id=#{id}&un=#{un}&cn=#{cn}&sig=#{sig}"
-
-  message = Buffer.from("GET #{uri} HTTP/1.1\r\nHost: #{address}\r\n\r\n")
-
-  client = dgram.createSocket({type: 'udp4', reuseAddr: true})
-  UDP_CONNECTION_REQUEST_PORT = config.get('UDP_CONNECTION_REQUEST_PORT')
-  if UDP_CONNECTION_REQUEST_PORT
-    # When a device is NAT'ed, the UDP Connection Request must originate from the same address and port used by the STUN server, in order to traverse the firewall.
-    # This does require that the Genieacs NBI and STUN server are allowed to bind to the same address and port.
-    # The STUN server needs to open its UDP port with the SO_REUSEADDR option, allowing the NBI to also bind to the same port.
-    client.bind({port: UDP_CONNECTION_REQUEST_PORT, exclusive: true})
-
-  count = 3
-  client.send(message, 0, message.length, port, host, f = (err) ->
-    if err or -- count <= 0
-      client.close()
-      return callback(err)
-    client.send(message, 0, message.length, port, host, f)
-  )
-  return true
-
-
-httpConReq = (url, username, password, allowBasicAuth, timeout, callback) ->
-  options = URL.parse(url)
-
-  if options.protocol != 'http:'
-    return callback(new Error('Invalid connection request URL or protocol'))
-
-  # Ensure socket is reused in case of digest authentication
-  options.agent = new http.Agent({maxSockets : 1})
-
-  statusToError = (statusCode) ->
-    switch statusCode
-      when 200, 204
-        null
-      when 401
-        new Error('Incorrect connection request credentials')
-      when 0
-        new Error('Device is offline')
-      else
-        new Error("Unexpected response code from device: #{statusCode}")
-
-  request = http.get(options, (res) ->
-    if res.statusCode == 401 and res.headers['www-authenticate']?
-      authHeader = auth.parseAuthHeader(res.headers['www-authenticate'])
-      if authHeader.method is 'Basic'
-        if not allowBasicAuth
-          request.abort()
-          callback(new Error('Basic HTTP authentication not allowed'))
-          return
-        options.headers = {'Authorization' : auth.basic(username or '', password or '')}
-      else if authHeader.method is 'Digest'
-        options.headers = {
-          'Authorization' : auth.digest(username or '', password or '', options.path, 'GET', null, authHeader)
-        }
-
-      request = http.get(options, (res) ->
-        if res.statusCode == 0
-          # Workaround for some devices unexpectedly closing the connection
-          request = http.get(options, (res) ->
-            callback(statusToError(res.statusCode))
-            res.resume()
-          ).on('error', (err) ->
-            request.abort()
-            callback(statusToError(0))
-          ).on('socket', (socket) ->
-            socket.setTimeout(timeout)
-            socket.on('timeout', () -> request.abort())
-          )
-        else
-          callback(statusToError(res.statusCode))
-        res.resume()
-      ).on('error', (err) ->
-        request.abort()
-        callback(statusToError(0))
-      ).on('socket', (socket) ->
-        socket.setTimeout(timeout)
-        socket.on('timeout', () -> request.abort())
-      )
-    else
-      callback(statusToError(res.statusCode))
-
-    # No listener for data so emit resume
-    res.resume()
-  ).on('error', (err) ->
-    request.abort()
-    callback(statusToError(0))
-  ).on('socket', (socket) ->
-    socket.setTimeout(timeout)
-    socket.on('timeout', () -> request.abort())
-  )
-
-
-connectionRequest = (deviceId, callback) ->
-  CONNECTION_REQUEST_TIMEOUT = config.get('CONNECTION_REQUEST_TIMEOUT', deviceId)
-  CONNECTION_REQUEST_ALLOW_BASIC_AUTH = config.get('CONNECTION_REQUEST_ALLOW_BASIC_AUTH', deviceId)
-  proj = {
-    'Device.ManagementServer.ConnectionRequestURL._value' : 1,
-    'Device.ManagementServer.UDPConnectionRequestAddress._value' : 1,
-    'Device.ManagementServer.ConnectionRequestUsername._value' : 1,
-    'Device.ManagementServer.ConnectionRequestPassword._value' : 1,
-    'InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value' : 1,
-    'InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress._value' : 1,
-    'InternetGatewayDevice.ManagementServer.ConnectionRequestUsername._value' : 1,
-    'InternetGatewayDevice.ManagementServer.ConnectionRequestPassword._value' : 1
-  }
-
-  db.devicesCollection.findOne({_id : deviceId}, proj, (err, device)->
-    return callback(err) if err
-
-    if device.Device? # TR-181 data model
-      connectionRequestUrl = device.Device.ManagementServer.ConnectionRequestURL._value
-      udpConnectionRequestAddress = device.Device.ManagementServer?.UDPConnectionRequestAddress?._value
-      username = device.Device.ManagementServer.ConnectionRequestUsername?._value
-      password = device.Device.ManagementServer.ConnectionRequestPassword?._value
-    else # TR-098 data model
-      connectionRequestUrl = device.InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value
-      udpConnectionRequestAddress = device.InternetGatewayDevice.ManagementServer?.UDPConnectionRequestAddress?._value
-      username = device.InternetGatewayDevice.ManagementServer.ConnectionRequestUsername?._value
-      password = device.InternetGatewayDevice.ManagementServer.ConnectionRequestPassword?._value
-
-    conReq = () ->
-      udpSent = udpConReq(udpConnectionRequestAddress, username, password, (err) -> throw err if err)
-
-      httpConReq(connectionRequestUrl, username, password, CONNECTION_REQUEST_ALLOW_BASIC_AUTH, CONNECTION_REQUEST_TIMEOUT, (err) ->
-        if udpSent
-          return callback()
-        callback(err)
-      )
-
-    if config.auth?.connectionRequest?
-      # Callback is optional for backward compatibility
-      if config.auth.connectionRequest.length > 4
-        return config.auth.connectionRequest(deviceId, connectionRequestUrl, username, password, (u, p) ->
-          username = u
-          password = p
-          conReq()
-        )
-      [username, password] = config.auth.connectionRequest(deviceId, connectionRequestUrl, username, password)
-    conReq()
-  )
-
-
-watchTask = (deviceId, taskId, timeout, callback) ->
-  setTimeout(() ->
-    db.tasksCollection.findOne({_id : taskId}, {'_id' : 1}, (err, task) ->
-      return callback(err) if err
-
-      if not task
-        return callback(null, 'completed')
-
-      db.faultsCollection.findOne({_id : "#{deviceId}:task_#{taskId}"}, {'_id' : 1}, (err, fault) ->
-        return callback(err) if err
-
-        if fault
-          return callback(null, 'fault')
-
-        if (timeout -= 500) <= 0
-          return callback(null, 'timeout')
-
-        watchTask(deviceId, taskId, timeout, callback)
-      )
-    )
-  , 500)
-
-
-sanitizeTask = (task, callback) ->
-  task.timestamp = new Date(task.timestamp ? Date.now())
-  if task.expiry?
-    if common.typeOf(task.expiry) is common.DATE_TYPE or isNaN(task.expiry)
-      task.expiry = new Date(task.expiry)
-    else
-      task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000)
-
-  callback(task)
-
-
-insertTasks = (tasks, callback) ->
-  if tasks? and common.typeOf(tasks) isnt common.ARRAY_TYPE
-    tasks = [tasks]
-  else if not tasks? or tasks.length == 0
-    return callback(null, tasks or [])
-
-  counter = tasks.length
-
-  for task in tasks
-    sanitizeTask(task, (t) ->
-      if t.uniqueKey?
-        db.tasksCollection.remove({device : t.device, uniqueKey : t.uniqueKey}, (err) ->
-        )
-
-      --counter
-      if counter == 0
-        db.tasksCollection.insert(tasks, (err, _tasks) ->
-          #util.log("#{_task.device}: Added task #{_task.name}(#{_task._id})") for _task in _tasks
-          callback(err, _tasks)
-        )
-    )
-
-
-deleteDevice = (deviceId, callback) ->
-  db.tasksCollection.remove({'device' : deviceId}, (err) ->
-    return callback(err) if err
-    db.devicesCollection.remove({'_id' : deviceId}, (err) ->
-      return callback(err) if err
-      db.faultsCollection.remove({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}, (err) ->
-        return callback(err) if err
-        db.operationsCollection.remove({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}, (err) ->
-          return callback(err) if err
-          cache.del("#{deviceId}_tasks_faults_operations", callback)
-        )
-      )
-    )
-  )
-
-
-exports.connectionRequest = connectionRequest
-exports.watchTask = watchTask
-exports.insertTasks = insertTasks
-exports.deleteDevice = deleteDevice

lib/api-functions.ts

@@ -0,0 +1,499 @@
+import { ObjectId } from "mongodb";
+import { collections } from "./db/db.ts";
+import {
+  deleteConfig,
+  deleteFault as dbDeleteFault,
+  deleteFile,
+  deletePermission,
+  deletePreset,
+  deleteProvision,
+  deleteTask,
+  deleteUser,
+  deleteVirtualParameter,
+  putConfig,
+  putPermission,
+  putPreset,
+  putProvision,
+  putUser,
+  putVirtualParameter,
+} from "./ui/db.ts";
+import * as common from "./util.ts";
+import * as cache from "./cache.ts";
+import { acquireLock, getToken, releaseLock } from "./lock.ts";
+import {
+  getRevision,
+  getConfig,
+  getConfigExpression,
+  getUsers,
+} from "./ui/local-cache.ts";
+import {
+  httpConnectionRequest,
+  udpConnectionRequest,
+  xmppConnectionRequest,
+} from "./connection-request.ts";
+import { Expression, Task } from "./types.ts";
+import { evaluate } from "./common/expression/util.ts";
+import { hashPassword } from "./auth.ts";
+import { flattenDevice } from "./ui/db.ts";
+import { ResourceLockedError } from "./common/errors.ts";
+import * as config from "../lib/config.ts";
+
+const XMPP_CONFIGURED = !!config.get("XMPP_JID");
+
+export async function connectionRequest(
+  deviceId: string,
+  device?: Record<string, { value?: [boolean | number | string, string] }>,
+): Promise<string> {
+  if (!device) {
+    const res = await collections.devices.findOne({ _id: deviceId });
+    if (!res) throw new Error("No such device");
+    device = flattenDevice(res);
+  }
+
+  let connectionRequestUrl,
+    udpConnectionRequestAddress,
+    stunEnable,
+    connReqJabberId,
+    username,
+    password;
+
+  if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
+    connectionRequestUrl = (device[
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL"
+    ].value || [""])[0];
+    udpConnectionRequestAddress = ((
+      device[
+        "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
+      ] || {}
+    ).value || [""])[0];
+    stunEnable = ((
+      device["InternetGatewayDevice.ManagementServer.STUNEnable"] || {}
+    ).value || [""])[0];
+    connReqJabberId = ((
+      device["InternetGatewayDevice.ManagementServer.ConnReqJabberID"] || {}
+    ).value || [""])[0];
+    username = ((
+      device[
+        "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
+      ] || {}
+    ).value || [""])[0];
+    password = ((
+      device[
+        "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword"
+      ] || {}
+    ).value || [""])[0];
+  } else {
+    connectionRequestUrl = (device[
+      "Device.ManagementServer.ConnectionRequestURL"
+    ].value || [""])[0];
+    udpConnectionRequestAddress = ((
+      device["Device.ManagementServer.UDPConnectionRequestAddress"] || {}
+    ).value || [""])[0];
+    stunEnable = ((device["Device.ManagementServer.STUNEnable"] || {})
+      .value || [""])[0];
+    connReqJabberId = ((device["Device.ManagementServer.ConnReqJabberID"] || {})
+      .value || [""])[0];
+    username = ((
+      device["Device.ManagementServer.ConnectionRequestUsername"] || {}
+    ).value || [""])[0];
+    password = ((
+      device["Device.ManagementServer.ConnectionRequestPassword"] || {}
+    ).value || [""])[0];
+  }
+  let remoteAddress;
+  try {
+    remoteAddress = new URL(connectionRequestUrl).hostname;
+  } catch (err) {
+    return "Invalid connection request URL";
+  }
+
+  const evalCallback = (exp): Expression => {
+    if (!Array.isArray(exp)) return exp;
+    if (exp[0] === "PARAM" && typeof exp[1] === "string") {
+      let name = exp[1];
+      if (name === "id") name = "DeviceID.ID";
+      else if (name === "serialNumber") name = "DeviceID.SerialNumber";
+      else if (name === "productClass") name = "DeviceID.ProductClass";
+      else if (name === "oui") name = "DeviceID.OUI";
+      else if (name === "remoteAddress") return remoteAddress;
+      else if (name === "username") return username;
+      else if (name === "password") return password;
+
+      const p = device[name];
+      if (p?.value) return p.value[0];
+    } else if (exp[0] === "FUNC") {
+      if (exp[1] === "REMOTE_ADDRESS") return remoteAddress;
+      else if (exp[1] === "USERNAME") return username;
+      else if (exp[1] === "PASSWORD") return password;
+    }
+    return exp;
+  };
+
+  const snapshot = await getRevision();
+  const now = Date.now();
+  const UDP_CONNECTION_REQUEST_PORT = +getConfig(
+    snapshot,
+    "cwmp.udpConnectionRequestPort",
+    {},
+    now,
+    evalCallback,
+  );
+  const CONNECTION_REQUEST_TIMEOUT = +getConfig(
+    snapshot,
+    "cwmp.connectionRequestTimeout",
+    {},
+    now,
+    evalCallback,
+  );
+  const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = !!getConfig(
+    snapshot,
+    "cwmp.connectionRequestAllowBasicAuth",
+    {},
+    now,
+    evalCallback,
+  );
+  let authExp: Expression = getConfigExpression(
+    snapshot,
+    "cwmp.connectionRequestAuth",
+  );
+
+  if (authExp === undefined) {
+    authExp = [
+      "FUNC",
+      "AUTH",
+      ["PARAM", "username"],
+      ["PARAM", "password"],
+    ] as Expression;
+  }
+
+  authExp = evaluate(authExp, {}, now, evalCallback);
+
+  const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
+
+  let udpProm = Promise.resolve(false);
+  if (udpConnectionRequestAddress && +stunEnable) {
+    try {
+      const u = new URL("udp://" + udpConnectionRequestAddress);
+      udpProm = udpConnectionRequest(
+        u.hostname,
+        parseInt(u.port || "80"),
+        authExp,
+        UDP_CONNECTION_REQUEST_PORT,
+        debug,
+        deviceId,
+      ).then(
+        () => true,
+        () => false,
+      );
+    } catch (err) {
+      // Ignore invalid address
+    }
+  }
+
+  let status;
+
+  if (connReqJabberId && XMPP_CONFIGURED) {
+    status = await xmppConnectionRequest(
+      connReqJabberId,
+      authExp,
+      CONNECTION_REQUEST_TIMEOUT,
+      debug,
+      deviceId,
+    );
+  } else {
+    status = await httpConnectionRequest(
+      connectionRequestUrl,
+      authExp,
+      CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
+      CONNECTION_REQUEST_TIMEOUT,
+      debug,
+      deviceId,
+    );
+  }
+
+  if (await udpProm) return "";
+
+  return status;
+}
+
+export async function awaitSessionStart(
+  deviceId: string,
+  lastInform: number,
+  timeout: number,
+): Promise<boolean> {
+  const now = Date.now();
+  const device = await collections.devices.findOne(
+    { _id: deviceId },
+    { projection: { _lastInform: 1 } },
+  );
+  const li = (device["_lastInform"] as Date).getTime();
+  if (li > lastInform) return true;
+  const token = await getToken(`cwmp_session_${deviceId}`);
+  if (token?.startsWith("cwmp_session_")) return true;
+  if (timeout < 500) return false;
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionStart(deviceId, lastInform, timeout);
+}
+
+export async function awaitSessionEnd(
+  deviceId: string,
+  timeout: number,
+): Promise<boolean> {
+  const now = Date.now();
+  const token = await getToken(`cwmp_session_${deviceId}`);
+  if (!token?.startsWith("cwmp_session_")) return true;
+  if (timeout < 500) return false;
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionEnd(deviceId, timeout);
+}
+
+function sanitizeTask(task): void {
+  task.timestamp = new Date(task.timestamp || Date.now());
+  if (task.expiry) {
+    if (task.expiry instanceof Date || isNaN(task.expiry))
+      task.expiry = new Date(task.expiry);
+    else task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000);
+  }
+
+  const validParamValue = (p): boolean => {
+    if (
+      !Array.isArray(p) ||
+      p.length < 2 ||
+      typeof p[0] !== "string" ||
+      !p[0].length ||
+      !["string", "boolean", "number"].includes(typeof p[1]) ||
+      (p[2] != null && typeof p[2] !== "string")
+    )
+      return false;
+    return true;
+  };
+
+  switch (task.name) {
+    case "getParameterValues":
+      if (!Array.isArray(task.parameterNames) || !task.parameterNames.length)
+        throw new Error("Missing 'parameterNames' property");
+      for (const p of task.parameterNames) {
+        if (typeof p !== "string" || !p.length)
+          throw new Error(`Invalid parameter name '${p}'`);
+      }
+      break;
+
+    case "setParameterValues":
+      if (!Array.isArray(task.parameterValues) || !task.parameterValues.length)
+        throw new Error("Missing 'parameterValues' property");
+      for (const p of task.parameterValues) {
+        if (!validParamValue(p))
+          throw new Error(`Invalid parameter value '${p}'`);
+      }
+      break;
+
+    case "refreshObject":
+      if (typeof task.objectName !== "string")
+        throw new Error("Missing 'objectName' property");
+      break;
+
+    case "deleteObject":
+      if (typeof task.objectName !== "string" || !task.objectName.length)
+        throw new Error("Missing 'objectName' property");
+      break;
+
+    case "addObject":
+      if (task.parameterValues != null) {
+        if (!Array.isArray(task.parameterValues))
+          throw new Error("Invalid 'parameterValues' property");
+        for (const p of task.parameterValues) {
+          if (!validParamValue(p))
+            throw new Error(`Invalid parameter value '${p}'`);
+        }
+      }
+      break;
+
+    case "download":
+      // genieacs-gui sends file ID instead of fileName and fileType
+      if (!task.file) {
+        if (typeof task.fileType !== "string" || !task.fileType.length)
+          throw new Error("Missing 'fileType' property");
+
+        if (typeof task.fileName !== "string" || !task.fileName.length)
+          throw new Error("Missing 'fileName' property");
+      }
+
+      if (
+        task.targetFileName != null &&
+        typeof task.targetFileName !== "string"
+      )
+        throw new Error("Invalid 'targetFileName' property");
+      break;
+
+    case "provisions":
+      if (
+        !Array.isArray(task.provisions) ||
+        !task.provisions.every((arr) =>
+          arr.every(
+            (s) =>
+              s == null || ["boolean", "number", "string"].includes(typeof s),
+          ),
+        )
+      )
+        throw new Error("Invalid 'provisions' property");
+      break;
+
+    case "reboot":
+      break;
+
+    case "factoryReset":
+      break;
+
+    default:
+      throw new Error("Invalid task name");
+  }
+
+  return task;
+}
+
+export async function insertTasks(tasks: any[]): Promise<Task[]> {
+  if (tasks && !Array.isArray(tasks)) tasks = [tasks];
+  else if (!tasks?.length) return tasks || [];
+
+  for (const task of tasks) {
+    sanitizeTask(task);
+    if (task.uniqueKey) {
+      await collections.tasks.deleteOne({
+        device: task.device,
+        uniqueKey: task.uniqueKey,
+      });
+    }
+  }
+  await collections.tasks.insertMany(tasks);
+  for (const task of tasks) task._id = task._id.toString();
+  return tasks;
+}
+
+export async function deleteDevice(deviceId: string): Promise<void> {
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+  if (!token) throw new ResourceLockedError("Device is in session");
+  try {
+    await Promise.all([
+      collections.tasks.deleteMany({ device: deviceId }),
+      collections.devices.deleteOne({ _id: deviceId }),
+      collections.faults.deleteMany({
+        _id: {
+          $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+        },
+      }),
+      collections.operations.deleteMany({
+        _id: {
+          $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+        },
+      }),
+    ]);
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
+}
+
+export async function deleteFault(id: string): Promise<void> {
+  const deviceId = id.split(":", 1)[0];
+  const channel = id.slice(deviceId.length + 1);
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+  if (!token) throw new ResourceLockedError("Device is in session");
+  try {
+    const proms = [dbDeleteFault(id)];
+    if (channel.startsWith("task_"))
+      proms.push(deleteTask(new ObjectId(channel.slice(5))));
+    await Promise.all(proms);
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
+}
+
+export async function deleteResource(
+  resource: string,
+  id: string,
+): Promise<void> {
+  if (resource === "devices") {
+    await deleteDevice(id);
+  } else if (resource === "files") {
+    await deleteFile(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "faults") {
+    await deleteFault(id);
+  } else if (resource === "provisions") {
+    await deleteProvision(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "presets") {
+    await deletePreset(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "virtualParameters") {
+    await deleteVirtualParameter(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "config") {
+    await deleteConfig(id);
+    await Promise.all([
+      cache.del("ui-local-cache-hash"),
+      cache.del("cwmp-local-cache-hash"),
+    ]);
+  } else if (resource === "permissions") {
+    await deletePermission(id);
+    await cache.del("ui-local-cache-hash");
+  } else if (resource === "users") {
+    await deleteUser(id);
+    await cache.del("ui-local-cache-hash");
+  } else {
+    throw new Error(`Unknown resource ${resource}`);
+  }
+}
+
+// TODO Implement validation
+export async function putResource(
+  resource: string,
+  id: string,
+  data: any,
+): Promise<void> {
+  if (resource === "presets") {
+    await putPreset(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "provisions") {
+    await putProvision(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "virtualParameters") {
+    await putVirtualParameter(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "config") {
+    await putConfig(id, data);
+    await Promise.all([
+      cache.del("ui-local-cache-hash"),
+      cache.del("cwmp-local-cache-hash"),
+    ]);
+  } else if (resource === "permissions") {
+    await putPermission(id, data);
+    await cache.del("ui-local-cache-hash");
+  } else if (resource === "users") {
+    delete data["password"];
+    delete data["salt"];
+    await putUser(id, data);
+    await cache.del("ui-local-cache-hash");
+  } else {
+    throw new Error(`Unknown resource ${resource}`);
+  }
+}
+
+export function authLocal(
+  snapshot: string,
+  username: string,
+  password: string,
+): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    const users = getUsers(snapshot);
+    const user = users[username];
+    if (!user?.password) return void resolve(null);
+    hashPassword(password, user.salt)
+      .then((hash) => {
+        if (hash === user.password) resolve(true);
+        else resolve(false);
+      })
+      .catch(reject);
+  });
+}

lib/auth.coffee

@@ -1,92 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-crypto = require 'crypto'
-
-
-exports.parseAuthHeader = (authHeader) ->
-  authHeader = authHeader.trim()
-  method = authHeader.split(' ', 1)[0]
-  res = {method: method}
-  parts = authHeader.slice(method.length + 1).split(',')
-
-  while (part = parts.shift())?
-    name = part.split('=', 1)[0]
-    throw new Error('Unable to parse auth header') if name.length == part.length
-    value = part.slice(name.length + 1)
-    if not /^\s*"/.test(value)
-      value = value.trim()
-    else
-      while not /[^\\]"\s*$/.test(value)
-        p = parts.shift()
-        throw new Error('Unable to parse auth header') if not p?
-        value += ',' + p
-
-      try
-        value = JSON.parse(value)
-      catch
-        throw new Error('Unable to parse auth header')
-
-    res[name.trim()] = value
-
-  return res
-
-
-exports.basic = (username, password) ->
-  "Basic #{new Buffer("#{username}:#{password}").toString('base64')}"
-
-
-exports.digest = (username, password, uri, httpMethod, body, authHeader) ->
-  cnonce = '0a4f113b'
-  nc = '00000001'
-
-  if authHeader.qop?
-    if authHeader.qop.indexOf(',') != -1
-      qop = 'auth' # either auth or auth-int. prefer auth
-    else
-      qop = authHeader.qop
-
-  ha1 = crypto.createHash('md5')
-  ha1.update(username).update(':').update(authHeader.realm).update(':').update(password)
-  # TODO support "MD5-sess" algorithm directive
-  ha1 = ha1.digest('hex')
-
-  ha2 = crypto.createHash('md5')
-  ha2.update(httpMethod).update(':').update(uri)
-  if qop == 'auth-int'
-    ha2.update(':').update(body)
-  ha2 = ha2.digest('hex')
-
-  response = crypto.createHash('md5')
-  response.update(ha1).update(':').update(authHeader.nonce)
-
-  if qop?
-    response.update(':').update(nc).update(':').update(cnonce).update(':').update(qop)
-  response.update(':').update(ha2)
-  response = response.digest('hex')
-
-  authString = "Digest username=\"#{username}\""
-  authString += ",realm=\"#{authHeader.realm}\""
-  authString += ",nonce=\"#{authHeader.nonce}\""
-  authString += ",uri=\"#{uri}\""
-  authString += ",algorithm=#{authHeader.algorithm}" if authHeader.algorithm?
-  authString += ",qop=#{qop},nc=#{nc},cnonce=\"#{cnonce}\"" if qop?
-  authString += ",response=\"#{response}\""
-  authString += ",opaque=\"#{authHeader.opaque}\"" if authHeader.opaque?
-  return authString

lib/auth.ts

@@ -0,0 +1,178 @@
+import { createHash, randomBytes, pbkdf2 } from "node:crypto";
+
+function parseHeaderFeilds(str: string): Record<string, string> {
+  const res = {};
+  const parts = str.split(",");
+
+  let part: string;
+  while ((part = parts.shift()) != null) {
+    const name = part.split("=", 1)[0];
+    if (name.length === part.length) {
+      if (!part.trim()) continue;
+      throw new Error("Unable to parse auth header");
+    }
+
+    let value = part.slice(name.length + 1);
+    if (!/^\s*"/.test(value)) {
+      value = value.trim();
+    } else {
+      while (!/[^\\]"\s*$/.test(value)) {
+        const p = parts.shift();
+        if (p == null) throw new Error("Unable to parse auth header");
+        value += "," + p;
+      }
+
+      try {
+        value = JSON.parse(value);
+      } catch (error) {
+        throw new Error("Unable to parse auth header");
+      }
+    }
+    res[name.trim()] = value;
+  }
+  return res;
+}
+
+export function parseAuthorizationHeader(authHeader: string): {
+  method: string;
+} {
+  authHeader = authHeader.trim();
+  const method = authHeader.split(" ", 1)[0];
+  const res = { method: method };
+
+  if (method === "Basic") {
+    // Inspired by https://github.com/jshttp/basic-auth
+    const USER_PASS_REGEX = /^([^:]*):(.*)$/;
+    const creds = USER_PASS_REGEX.exec(
+      Buffer.from(authHeader.slice(method.length + 1), "base64").toString(),
+    );
+
+    if (!creds) throw new Error("Unable to parse auth header");
+    res["username"] = creds[1];
+    res["password"] = creds[2];
+  } else if (method === "Digest") {
+    Object.assign(res, parseHeaderFeilds(authHeader.slice(method.length + 1)));
+  }
+
+  return res;
+}
+
+export function parseWwwAuthenticateHeader(
+  authHeader: string,
+): Record<string, string> {
+  authHeader = authHeader.trim();
+  const method = authHeader.split(" ", 1)[0];
+  const res = { method: method };
+  Object.assign(res, parseHeaderFeilds(authHeader.slice(method.length + 1)));
+  return res;
+}
+
+export function basic(username: string, password: string): string {
+  return "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
+}
+
+export function digest(
+  username: string | Buffer,
+  realm: string | Buffer,
+  password: string | Buffer,
+  nonce: string | Buffer,
+  httpMethod: string | Buffer,
+  uri: string | Buffer,
+  qop?: string | Buffer,
+  body?: string | Buffer,
+  cnonce?: string | Buffer,
+  nc?: string | Buffer,
+): string {
+  const ha1 = createHash("md5");
+  ha1.update(username).update(":").update(realm).update(":").update(password);
+  // TODO support "MD5-sess" algorithm directive
+  const ha1d = ha1.digest("hex");
+
+  const ha2 = createHash("md5");
+  ha2.update(httpMethod).update(":").update(uri);
+
+  if (qop === "auth-int") {
+    const bodyHash = createHash("md5")
+      .update(body || "")
+      .digest("hex");
+    ha2.update(":").update(bodyHash);
+  }
+
+  const ha2d = ha2.digest("hex");
+
+  const hash = createHash("md5");
+  hash.update(ha1d).update(":").update(nonce);
+  if (qop) {
+    hash
+      .update(":")
+      .update(nc)
+      .update(":")
+      .update(cnonce)
+      .update(":")
+      .update(qop);
+  }
+  hash.update(":").update(ha2d);
+
+  return hash.digest("hex");
+}
+
+export function solveDigest(
+  username: string | Buffer,
+  password: string | Buffer,
+  uri: string | Buffer,
+  httpMethod: string | Buffer,
+  body: string | Buffer,
+  authHeader: Record<string, string>,
+): string {
+  const cnonce = randomBytes(8).toString("hex");
+  const nc = "00000001";
+
+  let qop;
+  if (authHeader.qop) {
+    if (authHeader.qop.indexOf(",") !== -1) qop = "auth";
+    // Either auth or auth-int, prefer auth
+    else qop = authHeader.qop;
+  }
+
+  const hash = digest(
+    username,
+    authHeader.realm,
+    password,
+    authHeader.nonce,
+    httpMethod,
+    uri,
+    qop,
+    body,
+    cnonce,
+    nc,
+  );
+
+  let authString = `Digest username="${username}"`;
+  authString += `,realm="${authHeader.realm}"`;
+  authString += `,nonce="${authHeader.nonce}"`;
+  authString += `,uri="${uri}"`;
+  if (authHeader.algorithm) authString += `,algorithm=${authHeader.algorithm}`;
+  if (qop) authString += `,qop=${qop},nc=${nc},cnonce="${cnonce}"`;
+  authString += `,response="${hash}"`;
+  if (authHeader.opaque) authString += `,opaque="${authHeader.opaque}"`;
+
+  return authString;
+}
+
+export function generateSalt(length: number): Promise<string> {
+  return new Promise((resolve, reject) => {
+    randomBytes(length, (err, rand) => {
+      if (err) return void reject(err);
+      resolve(rand.toString("hex"));
+    });
+  });
+}
+
+export function hashPassword(pass: string, salt: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    pbkdf2(pass, salt, 10000, 128, "sha512", (err, hash) => {
+      if (err) return void reject(err);
+      resolve(hash.toString("hex"));
+    });
+  });
+}

lib/cache.js

@@ -1,257 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const config = require("./config");
-
-const MAX_CACHE_TTL = config.get("MAX_CACHE_TTL");
-
-let redisClient;
-let mongoDb;
-let mongoCollection;
-let mongoTimeOffset = 0;
-
-
-function connect(callback) {
-  const REDIS_HOST = config.get("REDIS_HOST");
-
-  if (REDIS_HOST) {
-    const redis = require("redis");
-    const REDIS_PORT = config.get("REDIS_PORT");
-    const REDIS_DB = config.get("REDIS_DB");
-    redisClient = redis.createClient(REDIS_PORT, REDIS_HOST);
-    redisClient.select(REDIS_DB, function(err) {
-      exports.get = redisGet;
-      exports.set = redisSet;
-      exports.del = redisDel;
-      exports.pop = redisPop;
-      exports.lock = redisLock;
-      callback(err);
-    });
-  }
-  else {
-    const mongodb = require("mongodb");
-    const MONGODB_CONNECTION_URL = config.get("MONGODB_CONNECTION_URL");
-
-    mongodb.MongoClient.connect(MONGODB_CONNECTION_URL, function(err, db) {
-      if (err)
-        return callback(err);
-
-      mongoDb = db;
-      mongoCollection = db.collection("cache");
-      mongoCollection.ensureIndex({"expire": 1}, {expireAfterSeconds: 0});
-
-      const now = Date.now();
-      mongoDb.command({hostInfo: 1}, function(err, res) {
-        if (err)
-          return callback(err);
-        mongoTimeOffset = res.system.currentTime.getTime() - now;
-        exports.get = mongoGet;
-        exports.set = mongoSet;
-        exports.del = mongoDel;
-        exports.pop = mongoPop;
-        exports.lock = mongoLock;
-        callback();
-      });
-    });
-  }
-}
-
-
-function disconnect() {
-  if (redisClient)
-    redisClient.quit();
-
-  if (mongoDb)
-    mongoDb.close();
-}
-
-
-function redisGet(key, callback) {
-  if (Array.isArray(key))
-    redisClient.mget(key, callback);
-  else
-    redisClient.get(key, callback);
-}
-
-
-function redisDel(key, callback) {
-  redisClient.del(key, callback);
-}
-
-
-function redisSet(key, value, ttl, callback) {
-  if (!callback && typeof v === "function") {
-    callback = ttl;
-    ttl = null;
-  }
-
-  if (!ttl)
-    ttl = MAX_CACHE_TTL;
-
-  redisClient.setex(key, ttl, value, callback);
-}
-
-
-function redisPop(key, callback) {
-  const POP_SCRIPT = "local v=redis.call(\"get\",KEYS[1]);redis.call(\"del\",KEYS[1]);return v;";
-  redisClient.eval(POP_SCRIPT, 1, key, callback);
-}
-
-
-function redisLock(lockName, ttl, callback) {
-  const UNLOCK_SCRIPT = "if redis.call(\"get\", KEYS[1]) == ARGV[1] then return redis.call(\"del\", KEYS[1]) else return 0 end";
-  const EXTEND_SCRIPT = "if redis.call(\"get\", KEYS[1]) == ARGV[1] then return redis.call(\"pexpire\", KEYS[1], ARGV[2]) else return 0 end";
-  const token = Math.random().toString(36).slice(2);
-
-  function unlockOrExtend(ttl) {
-    if (!ttl) {
-      redisClient.eval(UNLOCK_SCRIPT, 1, lockName, token, function(err, res) {
-        if (err || !res)
-          throw err || new Error("Lock expired");
-      });
-    }
-    else {
-      redisClient.eval(EXTEND_SCRIPT, 1, lockName, token, ttl, function(err, res) {
-        if (err || !res)
-          throw err || new Error("Lock expired");
-      });
-    }
-  }
-
-  redisClient.set(lockName, token, "NX", "EX", ttl, function(err, res) {
-    if (err || res)
-      return callback(err, unlockOrExtend);
-
-    setTimeout(function() {
-      redisLock(lockName, ttl, callback);
-    }, 200);
-  });
-}
-
-
-function mongoGet(key, callback) {
-  let expire = new Date(Date.now() - mongoTimeOffset);
-  if (Array.isArray(key))
-    mongoCollection.find({"_id": {"$in": key}}).toArray(function(err, res) {
-      if (err)
-        return callback(err);
-
-      let indices = {};
-      key.forEach(function(v, i) {
-        indices[v] = i;
-      });
-
-      let values = [];
-      res.forEach(function(r) {
-        if (r["expire"] > expire)
-          values[indices[r["_id"]]] = r["value"];
-      });
-      callback(null, values);
-    });
-  else
-    mongoCollection.findOne({"_id": {"$in": [key]}}, function(err, res) {
-      if (err || !res)
-        return callback(err);
-
-      if (res["expire"] > expire)
-        return callback(null, res["value"]);
-
-      callback();
-    });
-}
-
-
-function mongoDel(key, callback) {
-  if (Array.isArray(key))
-    mongoCollection.remove({"_id": {"$in": key}}, callback);
-  else
-    mongoCollection.remove({"_id": key}, callback);
-}
-
-
-function mongoSet(key, value, ttl, callback) {
-  if (!callback && typeof v === "function") {
-    callback = ttl;
-    ttl = null;
-  }
-
-  if (!ttl)
-    ttl = MAX_CACHE_TTL;
-
-  let expire = new Date((Date.now() - mongoTimeOffset) + (ttl * 1000));
-  mongoCollection.save({"_id": key, "value": value, "expire": expire}, callback);
-}
-
-
-function mongoPop(key, callback) {
-  mongoCollection.findAndModify({"_id": key}, null, null, {remove: true}, function(err, res) {
-    if (err || !res["value"])
-      return callback(err);
-
-    if (res["value"]["expire"] > new Date(Date.now() - mongoTimeOffset))
-      return callback(null, res["value"]["value"]);
-
-    callback();
-  });
-}
-
-
-function mongoLock(lockName, ttl, callback) {
-  const token = Math.random().toString(36).slice(2);
-
-  function unlockOrExtend(ttl) {
-    if (!ttl) {
-      mongoCollection.remove({"_id": lockName, "value": token}, function(err, res) {
-        if (err || res["result"]["n"] !== 1)
-          throw err || new Error("Lock expired");
-      });
-    }
-    else {
-      let expire = new Date((Date.now() - mongoTimeOffset) + (ttl * 1000));
-      mongoCollection.update(
-        {"_id": lockName, "value": token},
-        {"expire": expire},
-        function(err, res) {
-          if (err || res["result"]["n"] !== 1)
-            throw err | new Error("Lock expired");
-        }
-      );
-    }
-  }
-
-  let expireTest = new Date(Date.now() - mongoTimeOffset);
-  let expireSet = new Date((Date.now() - mongoTimeOffset) + (ttl * 1000));
-
-  mongoCollection.update(
-    {"_id": lockName, "expire": {"$lte": expireTest}},
-    {"$set": {"value" : token, "expire": expireSet}},
-    {upsert: true},
-    function(err, res) {
-      if (err && err.code === 11000)
-        return setTimeout(function() {mongoLock(lockName, ttl, callback)}, 200);
-
-      return callback(err, unlockOrExtend);
-    }
-  );
-}
-
-
-exports.connect = connect;
-exports.disconnect = disconnect;

lib/cache.ts

@@ -0,0 +1,35 @@
+import { collections } from "./db/db.ts";
+import * as config from "./config.ts";
+
+const CLOCK_SKEW_TOLERANCE = 30000;
+const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
+
+export async function get(key: string): Promise<string> {
+  const res = await collections.cache.findOne({ _id: key });
+  return res?.value;
+}
+
+export async function del(key: string): Promise<void> {
+  await collections.cache.deleteOne({ _id: key });
+}
+
+export async function set(
+  key: string,
+  value: string,
+  ttl: number = MAX_CACHE_TTL,
+): Promise<void> {
+  const timestamp = new Date();
+  const expire = new Date(
+    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000,
+  );
+  await collections.cache.replaceOne(
+    { _id: key },
+    { value, expire, timestamp },
+    { upsert: true },
+  );
+}
+
+export async function pop(key: string): Promise<string> {
+  const res = await collections.cache.findOneAndDelete({ _id: key });
+  return res.value?.value;
+}

lib/cluster.js

@@ -1,177 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const cluster = require("cluster");
-const logger = require("./logger");
-const config = require("./config");
-
-const VERSION = require("../package.json").version;
-
-let respawnTimestamp = 0;
-let tooManyCrashesTimestamp = 0;
-
-function getDependencyVersions() {
-  let dependencies = [`node@${process.versions.node}`];
-  try {
-    let npmls = JSON.parse(require("child_process").execSync("npm ls --prod --json --depth 0"));
-
-    for (let k in npmls.dependencies)
-      dependencies.push(`${k}@${npmls.dependencies[k].version}`);
-  } catch (err) {}
-
-  return dependencies.join(",");
-}
-
-
-function getConfig() {
-  let conf = [];
-  const keys = Object.keys(process.env);
-  for (let k of keys)
-    if (k.startsWith("GENIEACS_")) {
-      const name = k.slice(9);
-      const value = process.env[k];
-      const def = config.getDefault(name);
-      if (def == null || value !== `${def}`)
-        conf.push(`${name}=${value}`);
-    }
-
-  return conf.join(",");
-}
-
-
-function restartWorker(worker, code, signal) {
-  let msg = {
-    message: "Worker died",
-    pid: worker.process.pid
-  };
-
-  if (code != null)
-    msg.exitCode = code;
-
-  if (signal != null)
-    msg.signal = signal;
-
-  logger.error(msg);
-
-  const now = Date.now();
-  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
-  if (respawnTimestamp === now) {
-    tooManyCrashesTimestamp = now;
-    cluster.fork();
-    return
-  }
-
-  if (now - tooManyCrashesTimestamp > 60000) {
-    process.exitCode = 1;
-    cluster.removeListener("exit", restartWorker);
-    for (let pid in cluster.workers)
-      cluster.workers[pid].kill();
-
-    logger.error({
-      message: "Too many crashes, exiting",
-      pid: process.pid
-    });
-    return;
-  }
-
-  setTimeout(function() {
-    if (process.exitCode) return;
-    cluster.fork();
-  }, respawnTimestamp - now);
-}
-
-
-function start(service) {
-  logger.init(service, VERSION);
-  const config = require("./config");
-  const SERVICE_ADDRESS = config.get(`${service.toUpperCase()}_INTERFACE`)
-  const SERVICE_PORT = config.get(`${service.toUpperCase()}_PORT`)
-
-  logger.info({
-    message: `GenieACS (genieacs-${service}) starting`,
-    pid: process.pid,
-    version: VERSION,
-    dependencies: getDependencyVersions(),
-    config: getConfig()
-  });
-
-  cluster.on("listening", function(worker, address) {
-    if ((address.addressType === 4 || address.addressType === 6) &&
-        address.address === SERVICE_ADDRESS && address.port === SERVICE_PORT) {
-      logger.info({
-        message: "Worker listening",
-        pid: worker.process.pid,
-        address: address.address,
-        port: address.port
-      });
-    }
-  });
-
-  cluster.on("exit", restartWorker);
-
-  process.on("SIGINT", function() {
-    logger.info({
-      message: "Received signal SIGINT, exiting",
-      pid: process.pid
-    });
-
-    cluster.removeListener("exit", restartWorker);
-  });
-
-  process.on("SIGTERM", function() {
-    logger.info({
-      message: "Received signal SIGTERM, exiting",
-      pid: process.pid
-    });
-
-    cluster.removeListener("exit", restartWorker);
-    for (let pid in cluster.workers)
-      cluster.workers[pid].kill();
-  });
-
-  cluster.setupMaster({
-    exec: "lib/server",
-    args: [service]
-  });
-
-  let workerCount = config.get(`${service.toUpperCase()}_WORKER_PROCESSES`);
-
-  if (!workerCount)
-    workerCount = Math.max(2, require("os").cpus().length);
-
-  for (let i = 0; i < workerCount; ++ i)
-    cluster.fork();
-}
-
-
-process.on("uncaughtException", function(err) {
-  logger.error({
-    message: "Uncaught exception in master process, exiting",
-    exception: err,
-    pid: process.pid
-  });
-  process.exitCode = 1;
-  cluster.removeListener("exit", restartWorker);
-  for (let pid in cluster.workers)
-    cluster.workers[pid].kill();
-});
-
-
-exports.start = start;

lib/cluster.ts

@@ -0,0 +1,105 @@
+import cluster, { Worker } from "node:cluster";
+import { cpus } from "node:os";
+import * as logger from "./logger.ts";
+
+let respawnTimestamp = 0;
+let crashes: number[] = [];
+
+function fork(): Worker {
+  const w = cluster.fork();
+  w.on("error", (err: NodeJS.ErrnoException) => {
+    // Avoid exception when attempting to kill the process just as it's exiting
+    if (err.code !== "EPIPE") throw err;
+    setTimeout(() => {
+      if (!w.isDead()) throw err;
+    }, 50);
+  });
+  return w;
+}
+
+function restartWorker(worker, code, signal): void {
+  const msg = {
+    message: "Worker died",
+    pid: worker.process.pid,
+    exitCode: null,
+    signal: null,
+  };
+
+  if (code != null) msg.exitCode = code;
+
+  if (signal != null) msg.signal = signal;
+
+  logger.error(msg);
+
+  const now = Date.now();
+  crashes.push(now);
+
+  let min1 = 0,
+    min2 = 0,
+    min3 = 0;
+
+  crashes = crashes.filter((n) => {
+    if (n > now - 60000) ++min1;
+    else if (n > now - 120000) ++min2;
+    else if (n > now - 180000) ++min3;
+    else return false;
+    return true;
+  });
+
+  if (min1 > 5 && min2 > 5 && min3 > 5) {
+    process.exitCode = 1;
+    cluster.removeListener("exit", restartWorker);
+    for (const pid in cluster.workers) cluster.workers[pid].kill();
+
+    logger.error({
+      message: "Too many crashes, exiting",
+      pid: process.pid,
+    });
+    return;
+  }
+
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    fork();
+    return;
+  }
+
+  setTimeout(() => {
+    if (process.exitCode) return;
+    fork();
+  }, respawnTimestamp - now);
+}
+
+export function start(
+  workerCount: number,
+  servicePort: number,
+  serviceAddress: string,
+): void {
+  cluster.on("listening", (worker, address) => {
+    if (
+      (address.addressType === 4 || address.addressType === 6) &&
+      address.address === serviceAddress &&
+      address.port === servicePort
+    ) {
+      logger.info({
+        message: "Worker listening",
+        pid: worker.process.pid,
+        address: address.address,
+        port: address.port,
+      });
+    }
+  });
+
+  cluster.on("exit", restartWorker);
+
+  if (!workerCount) workerCount = Math.max(2, cpus().length);
+
+  for (let i = 0; i < workerCount; ++i) fork();
+}
+
+export function stop(): void {
+  cluster.removeListener("exit", restartWorker);
+  for (const pid in cluster.workers) cluster.workers[pid].kill();
+}
+
+export const worker = cluster.worker;

lib/common.coffee

@@ -1,229 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-buffer = require 'buffer'
-
-UNDEFINED_TYPE = '[object Undefined]'
-NULL_TYPE = '[object Null]'
-BOOLEAN_TYPE = '[object Boolean]'
-NUMBER_TYPE = '[object Number]'
-STRING_TYPE = '[object String]'
-ARRAY_TYPE = '[object Array]'
-OBJECT_TYPE = '[object Object]'
-REGEXP_TYPE = '[object RegExp]'
-DATE_TYPE = '[object Date]'
-
-
-typeOf = (obj) ->
-  Object.prototype.toString.call(obj)
-
-
-exports.generateDeviceId = (deviceIdStruct) ->
-  # Percent escaping function. Escapes everything except alphanumerics and underscore
-  esc = (str) ->
-    str.replace(/[^A-Za-z0-9_]/g, (chr) ->
-      buf = new buffer.Buffer(chr, 'utf8')
-      rep = ''
-      rep += "%#{b.toString(16).toUpperCase()}" for b in buf
-      return rep
-    )
-
-  # Guaranteeing globally unique id as defined in TR-069
-  if deviceIdStruct['ProductClass']
-    return "#{esc(deviceIdStruct['OUI'])}-#{esc(deviceIdStruct['ProductClass'])}-#{esc(deviceIdStruct['SerialNumber'])}"
-
-  return "#{esc(deviceIdStruct['OUI'])}-#{esc(deviceIdStruct['SerialNumber'])}"
-
-
-exports.getParamValueFromPath = (obj, path) ->
-  pp = path.split('.')
-  ref = obj
-  try
-    for p in pp
-      ref = ref[p]
-    return ref
-  catch err
-    return undefined
-
-
-pathOverlap = (a, b, start) ->
-  if a.length == b.length
-    res = 3
-  else if a.length > b.length
-    res = 2
-  else
-    res = 1
-
-  for i in [(start ? 0)...Math.min(a.length, b.length)] by 1
-    if a[i]?
-      if not b[i]?
-        res &= 2
-      else if a[i] != b[i]
-        return 0
-    else if b[i]?
-      res &= 1
-
-    return 0 if not res
-
-  return res
-
-
-parseAlias = (pattern, start, res) ->
-  aliases = []
-  i = start
-  while i < pattern.length and pattern[i] != ']'
-    alias = []
-    i = j = parsePath(pattern, i, alias) + 1
-
-    while pattern[j] not in [']', ',']
-      if pattern[j] == '"' and i == j
-        ++ j
-        while pattern[j] != '"' or pattern[j - 1] == '\\'
-          if ++ j >= pattern.length
-            throw new Error('Invalid alias expression')
-      if ++ j >= pattern.length
-        throw new Error('Invalid alias expression')
-    value = pattern[i...j].trim()
-    i = j
-    if value[0] == '"'
-      try
-        value = JSON.parse(value)
-      catch err
-        throw new Error('Invalid alias expression')
-    alias.push(value)
-    aliases.push(alias)
-    ++ i if pattern[i] == ','
-
-  # Need to sort to ensure identical expressions have idential string representation
-  srt = (a, b) ->
-    for j in [0...Math.min(a.length, b.length)] by 2
-      for k in [0...Math.min(a[j].length, b[j].length)] by 1
-        if Array.isArray(a[j][k])
-          if Array.isArray(b[j][k])
-            return srt(a[j][k], b[j][k])
-          else if not b[j][k]?
-            return -1
-          else
-            return 1
-        else if not a[j][k]?
-          if not b[j][k]?
-            return 0
-          else
-            return 1
-        else if not b[j][k]? or Array.isArray(b[j][k])
-            return -1
-        else if a[j][k] > b[j][k]
-          return 1
-        else if a[j][k] < b[j][k]
-          return -1
-
-      if a[j].length > b[j].length
-        return -1
-      else if a[j].length < b[j].length
-        return 1
-
-      if a[j + 1] > b[j + 1]
-        return -1
-      else if a[j + 1] < b[j + 1]
-        return 1
-
-    if a.length > b.length
-      return -1
-    else if a.length < b.length
-      return 1
-
-    return 0
-
-  aliases.sort(srt)
-  res.push([].concat.apply([], aliases))
-  return i
-
-
-parsePath = (pattern, start, res) ->
-  path = []
-  path.wildcard = 0
-  path.alias = 0
-  i = start ? 0
-
-  # Colon separator is needed for parseAlias
-  if i < pattern.length and pattern[i] != ':'
-    while true
-      if pattern[i] == '['
-        path.alias |= 1 << path.length
-        i = parseAlias(pattern, i + 1, path) + 1
-      else
-        j = i
-        while i < pattern.length and pattern[i] != ':' and pattern[i] != '.'
-          ++ i
-        n = pattern.slice(j, i)
-        path.wildcard |= 1 << path.length if n == '*'
-        path.push(n)
-
-      if i >= pattern.length or pattern[i] == ':'
-        break
-      else if pattern[i] != '.'
-        throw new Error('Invalid alias expression')
-
-      ++ i
-
-  Object.freeze(path)
-
-  if not res?
-    return path
-
-  res.push(path)
-  return i
-
-
-addPathMeta = (path) ->
-  return path if path.alias? or path.wildcard?
-
-  path.alias = 0
-  path.wildcard = 0
-
-  for p, i in path
-    if typeOf(p) is ARRAY_TYPE
-      path.alias |= 1 << i
-      for j in [0...p.length] by 2
-        addPathMeta(p[j])
-    else if p == '*'
-      path.wildcard |= 1 << i
-
-  Object.freeze(path)
-  return path
-
-
-# Source: http://stackoverflow.com/a/6969486
-escapeRegExp = (str) ->
-  return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&")
-
-
-exports.UNDEFINED_TYPE = UNDEFINED_TYPE
-exports.NULL_TYPE = NULL_TYPE
-exports.NUMBER_TYPE = NUMBER_TYPE
-exports.STRING_TYPE = STRING_TYPE
-exports.ARRAY_TYPE = ARRAY_TYPE
-exports.OBJECT_TYPE = OBJECT_TYPE
-exports.REGEXP_TYPE = REGEXP_TYPE
-
-exports.typeOf = typeOf
-exports.pathOverlap = pathOverlap
-exports.parsePath = parsePath
-exports.addPathMeta = addPathMeta
-exports.escapeRegExp = escapeRegExp

lib/common/authorizer.ts

@@ -0,0 +1,129 @@
+import { PermissionSet, Expression } from "../types.ts";
+import { evaluate, or } from "./expression/util.ts";
+
+export default class Authorizer {
+  declare private permissionSets: PermissionSet[];
+  declare private validatorCache: WeakMap<
+    any,
+    (mutationType, mutation, any) => boolean
+  >;
+  declare private hasAccessCache: Map<string, boolean>;
+  declare private getFilterCache: Map<string, Expression>;
+
+  public constructor(permissionSets: PermissionSet[]) {
+    this.permissionSets = permissionSets;
+    this.validatorCache = new WeakMap();
+    this.hasAccessCache = new Map();
+    this.getFilterCache = new Map();
+  }
+
+  public hasAccess(resourceType: string, access: number): boolean {
+    const cacheKey = `${resourceType}-${access}`;
+    if (this.hasAccessCache.has(cacheKey))
+      return this.hasAccessCache.get(cacheKey);
+
+    let has = false;
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (perm[resourceType]) {
+          if (perm[resourceType].access >= access) {
+            has = true;
+            break;
+          }
+        }
+      }
+    }
+
+    this.hasAccessCache.set(cacheKey, has);
+    return has;
+  }
+
+  public getFilter(resourceType: string, access: number): Expression {
+    const cacheKey = `${resourceType}-${access}`;
+    if (this.getFilterCache.has(cacheKey))
+      return this.getFilterCache.get(cacheKey);
+
+    let filter: Expression = null;
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (perm[resourceType]) {
+          if (perm[resourceType].access >= access)
+            filter = or(filter, perm[resourceType].filter);
+        }
+      }
+    }
+
+    this.getFilterCache.set(cacheKey, filter);
+    return filter;
+  }
+
+  public getValidator(
+    resourceType: string,
+    resource: unknown,
+  ): (mutationType: string, mutation?: any, args?: any) => boolean {
+    if (this.validatorCache.has(resource))
+      return this.validatorCache.get(resource);
+
+    const validators: Expression[] = [];
+
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (
+          perm[resourceType] &&
+          perm[resourceType].access >= 3 &&
+          perm[resourceType].validate
+        )
+          validators.push(perm[resourceType].validate);
+      }
+    }
+
+    const validator = (
+      mutationType: string,
+      mutation: any,
+      any: any,
+    ): boolean => {
+      if (!validators.length) return false;
+
+      const object = {
+        mutationType,
+        mutation,
+        resourceType,
+        object: resource,
+        options: any,
+      };
+
+      const valueFunction = (paramName): any => {
+        const entry = paramName.split(".", 1)[0];
+        paramName = paramName.slice(entry.length + 1);
+        let value = null;
+        if (["mutation", "options"].includes(entry)) {
+          value = object[entry];
+          for (const seg of paramName.split(".")) {
+            if (value == null) break;
+            if (typeof value !== "object") value = null;
+            else value = value[seg];
+          }
+        } else if (object[entry]) {
+          if (paramName) value = object[entry][paramName];
+          else value = object[entry];
+        }
+
+        return value;
+      };
+
+      const res = evaluate(
+        validators.length > 1 ? ["OR", validators] : validators[0],
+        valueFunction,
+        Date.now(),
+      );
+      return !Array.isArray(res) && !!res;
+    };
+
+    this.validatorCache.set(resource, validator);
+    return validator;
+  }
+
+  public getPermissionSets(): PermissionSet[] {
+    return this.permissionSets;
+  }
+}

lib/common/debounce.ts

@@ -0,0 +1,16 @@
+export default function debounce<T>(
+  func: (args: T[]) => void,
+  timeout: number,
+): (arg: T) => void {
+  let timer: ReturnType<typeof setTimeout>;
+  let args: T[] = [];
+  return (arg: T) => {
+    args.push(arg);
+    clearTimeout(timer);
+    timer = setTimeout(() => {
+      const argscopy = args;
+      args = [];
+      func(argscopy);
+    }, timeout);
+  };
+}

lib/common/errors.ts

@@ -0,0 +1,6 @@
+export class ResourceLockedError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ResourceLockedError";
+  }
+}

lib/common/expression/bigint.ts

@@ -0,0 +1,87 @@
+export function and(a: bigint, b: bigint): bigint {
+  return a & b;
+}
+
+export function or(a: bigint, b: bigint): bigint {
+  return a | b;
+}
+
+export function xor(a: bigint, b: bigint): bigint {
+  return a ^ b;
+}
+
+export function not(a: bigint): bigint {
+  return ~a;
+}
+
+export function lshift(a: bigint, b: bigint): bigint {
+  return a << b;
+}
+
+export function rshift(a: bigint, b: bigint): bigint {
+  return a >> b;
+}
+
+export function add(a: bigint, b: bigint): bigint {
+  return a + b;
+}
+
+export function sub(a: bigint, b: bigint): bigint {
+  return a - b;
+}
+
+export function mul(a: bigint, b: bigint): bigint {
+  return a * b;
+}
+
+export function div(a: bigint, b: bigint): bigint {
+  return a / b;
+}
+
+export function exp(a: bigint, b: bigint): bigint {
+  return a ** b;
+}
+
+export function rem(a: bigint, b: bigint): bigint {
+  return a % b;
+}
+
+export function toNumber(a: bigint): number {
+  return Number(a);
+}
+
+export function eq(a: bigint, b: bigint): boolean {
+  return a === b;
+}
+
+export function ne(a: bigint, b: bigint): boolean {
+  return a !== b;
+}
+
+export function lt(a: bigint, b: bigint): boolean {
+  return a < b;
+}
+
+export function lte(a: bigint, b: bigint): boolean {
+  return a <= b;
+}
+
+export function gt(a: bigint, b: bigint): boolean {
+  return a > b;
+}
+
+export function gte(a: bigint, b: bigint): boolean {
+  return a >= b;
+}
+
+export function asUintN(a: number, b: bigint): bigint {
+  return BigInt.asUintN(a, b);
+}
+
+export function asIntN(a: number, b: bigint): bigint {
+  return BigInt.asIntN(a, b);
+}
+
+const _BigInt = BigInt;
+type _bigint = bigint;
+export { _BigInt as BigInt, type _bigint as bigint };

lib/common/expression/normalize.ts

@@ -0,0 +1,492 @@
+import * as BI from "./bigint.ts";
+import { Expression } from "../../types.ts";
+import { map } from "./parser.ts";
+import { and, evaluateCallback } from "./util.ts";
+
+const ZERO = BI.BigInt(0);
+const ONE = BI.BigInt(1);
+const TWO = BI.BigInt(2);
+const NEGATIVE_ONE = BI.BigInt(-1);
+
+class Indeterminates {
+  declare public map: Map<string, number>;
+  declare public sortedKeys: string[];
+
+  public constructor(str?: string) {
+    this.map = new Map();
+    if (str) {
+      this.map.set(str, 1);
+      this.sortedKeys = [str];
+    } else {
+      this.sortedKeys = [];
+    }
+  }
+
+  public reciprocal(): Indeterminates {
+    const res = new Indeterminates();
+    res.sortedKeys = this.sortedKeys;
+    res.map = new Map();
+    for (const [k, v] of this.map) res.map.set(k, 0 - v);
+    return res;
+  }
+
+  public static multiply(
+    indeterminates1: Indeterminates,
+    indeterminates2: Indeterminates,
+  ): Indeterminates {
+    const res = new Indeterminates();
+    res.sortedKeys = indeterminates1.sortedKeys.slice();
+    res.map = new Map(indeterminates1.map);
+
+    for (const [key, val] of indeterminates2.map) {
+      const v = res.map.get(key);
+      if (!v) {
+        res.map.set(key, val);
+        res.sortedKeys.push(key);
+      } else {
+        const v2 = val + v;
+        if (!v2) {
+          res.map.delete(key);
+          res.sortedKeys = res.sortedKeys.filter((s) => s !== key);
+        } else {
+          res.map.set(key, v2);
+        }
+      }
+    }
+
+    res.sortedKeys.sort((a, b) => {
+      if (a.length !== b.length) return b.length - a.length;
+      else if (a > b) return 1;
+      else if (a < b) return -1;
+      return 0;
+    });
+
+    return res;
+  }
+
+  public static compare(a: Indeterminates, b: Indeterminates): number {
+    if (a.sortedKeys.length !== b.sortedKeys.length)
+      return b.sortedKeys.length - a.sortedKeys.length;
+    for (let i = 0; i < a.sortedKeys.length; ++i) {
+      const k1 = a.sortedKeys[i];
+      const w1 = a.map.get(k1);
+      const k2 = b.sortedKeys[i];
+      const w2 = b.map.get(k2);
+      if (w1 !== w2) return w2 - w1;
+      if (k1.length > k2.length) return -1;
+      else if (k1.length < k2.length) return 1;
+      else if (k1 > k2) return 1;
+      else if (k1 < k2) return -1;
+    }
+    return 0;
+  }
+}
+
+interface Term {
+  indeterminates: Indeterminates;
+  coefficientNumerator: BI.bigint;
+  coefficientDenominator: BI.bigint;
+}
+
+function findGcd(a: BI.bigint, b: BI.bigint): BI.bigint {
+  while (BI.ne(b, ZERO)) {
+    const t = b;
+    b = BI.rem(a, b);
+    a = t;
+  }
+  return a;
+}
+
+class Polynomial {
+  declare public terms: Term[];
+
+  public constructor(terms: Term[]) {
+    this.terms = terms;
+  }
+
+  public static simplifyTerms(terms: Term[]): Term[] {
+    const ts = terms
+      .slice()
+      .sort((a: Term, b: Term) =>
+        Indeterminates.compare(a.indeterminates, b.indeterminates),
+      );
+
+    for (let i = 1; i < ts.length; ++i) {
+      const t1 = ts[i - 1];
+      const t2 = ts[i];
+      if (Indeterminates.compare(t1.indeterminates, t2.indeterminates) === 0) {
+        const numerator = BI.add(
+          BI.mul(t1.coefficientNumerator, t2.coefficientDenominator),
+          BI.mul(t2.coefficientNumerator, t1.coefficientDenominator),
+        );
+
+        const denominator = BI.mul(
+          t1.coefficientDenominator,
+          t2.coefficientDenominator,
+        );
+
+        const gcd = findGcd(numerator, denominator);
+
+        ts[i] = {
+          indeterminates: t2.indeterminates,
+          coefficientNumerator: BI.div(numerator, gcd),
+          coefficientDenominator: BI.div(denominator, gcd),
+        };
+        ts[i - 1] = {
+          indeterminates: t1.indeterminates,
+          coefficientNumerator: ZERO,
+          coefficientDenominator: t1.coefficientDenominator,
+        };
+      }
+    }
+    return ts.filter((v) => BI.ne(v.coefficientNumerator, ZERO));
+  }
+
+  public static fromIndeterminate(indeterminate: Expression): Polynomial {
+    const indeterminates = new Indeterminates(JSON.stringify(indeterminate));
+    const terms = [
+      {
+        indeterminates: indeterminates,
+        coefficientNumerator: ONE,
+        coefficientDenominator: ONE,
+      },
+    ];
+    return new Polynomial(terms);
+  }
+
+  public static fromConstant(constant: number): Polynomial {
+    const [int, frac] = Math.abs(constant).toString(2).split(".", 2);
+    let numerator = BI.BigInt("0b" + int);
+    if (constant < 0) numerator = BI.mul(numerator, NEGATIVE_ONE);
+    let denominator = ONE;
+    if (frac) {
+      denominator = BI.exp(TWO, BI.BigInt(frac.length));
+      numerator = BI.add(
+        BI.mul(numerator, denominator),
+        BI.BigInt("0b" + frac),
+      );
+    }
+
+    const terms = [
+      {
+        indeterminates: new Indeterminates(),
+        coefficientNumerator: numerator,
+        coefficientDenominator: denominator,
+      },
+    ];
+
+    return new Polynomial(terms);
+  }
+
+  public negation(): Polynomial {
+    const terms = this.terms.map((t) => ({
+      indeterminates: t.indeterminates,
+      coefficientNumerator: BI.mul(t.coefficientNumerator, NEGATIVE_ONE),
+      coefficientDenominator: t.coefficientDenominator,
+    }));
+
+    return new Polynomial(terms);
+  }
+
+  public reciprocal(): Polynomial {
+    const terms = this.terms.map((t) => ({
+      indeterminates: t.indeterminates.reciprocal(),
+      coefficientNumerator: t.coefficientDenominator,
+      coefficientDenominator: t.coefficientNumerator,
+    }));
+    return new Polynomial(terms);
+  }
+
+  public constant(): Polynomial {
+    const terms = this.terms.filter((t) => !t.indeterminates.sortedKeys.length);
+    return new Polynomial(terms);
+  }
+
+  public add(rhs: Polynomial): Polynomial {
+    return new Polynomial(
+      Polynomial.simplifyTerms(this.terms.concat(rhs.terms)),
+    );
+  }
+
+  public subtract(rhs: Polynomial): Polynomial {
+    return this.add(rhs.negation());
+  }
+
+  public multiply(rhs: Polynomial): Polynomial {
+    const terms: Term[] = [];
+
+    for (const t1 of this.terms) {
+      for (const t2 of rhs.terms) {
+        const numerator = BI.mul(
+          t1.coefficientNumerator,
+          t2.coefficientNumerator,
+        );
+        const denominator = BI.mul(
+          t1.coefficientDenominator,
+          t2.coefficientDenominator,
+        );
+        const gcd = findGcd(numerator, denominator);
+
+        terms.push({
+          indeterminates: Indeterminates.multiply(
+            t1.indeterminates,
+            t2.indeterminates,
+          ),
+          coefficientNumerator: BI.div(numerator, gcd),
+          coefficientDenominator: BI.div(denominator, gcd),
+        });
+      }
+    }
+
+    return new Polynomial(Polynomial.simplifyTerms(terms));
+  }
+
+  public divide(rhs: Polynomial): Polynomial {
+    return this.multiply(rhs.reciprocal());
+  }
+
+  public toString(): string {
+    const add: string[] = [];
+    for (const t of this.terms) {
+      const coefficient =
+        BI.toNumber(t.coefficientNumerator) /
+        BI.toNumber(t.coefficientDenominator);
+
+      const mul: string[] = [];
+      if (t.indeterminates.sortedKeys.length) {
+        for (const k of t.indeterminates.sortedKeys) {
+          const w = t.indeterminates.map.get(k);
+          for (let i = Math.abs(w); i > 0; --i) {
+            if (w > 0) mul.push(k);
+            else mul.push(`["/",1,${k}]`);
+          }
+        }
+
+        if (coefficient !== 1) mul.push(coefficient.toString());
+
+        if (mul.length > 1) add.push(`["*",${mul.join(",")}]`);
+        else add.push(mul["0"]);
+      } else {
+        add.push(coefficient.toString());
+      }
+    }
+
+    if (!add.length) return "0";
+    else if (add.length === 1) return add[0];
+    else return `["+",${add.join(",")}]`;
+  }
+}
+
+const ADDITIVE_IDENTITY = Polynomial.fromConstant(0);
+const MULTIPLICATIVE_IDENTITY = Polynomial.fromConstant(1);
+
+const SWAPPED_OPS = {
+  "=": "=",
+  "<>": "<>",
+  ">": "<",
+  ">=": "<=",
+  "<": ">",
+  "<=": ">=",
+};
+
+function normalizeCallback(exp: Expression): Expression {
+  if (!Array.isArray(exp)) return exp;
+  const op = exp[0];
+
+  if (op === "FUNC" && exp[1] === "COALESCE") {
+    const res: Expression[] = ["CASE"];
+    for (let i = 2; i < exp.length; ++i)
+      res.push(normalizeCallback(["IS NOT NULL", exp[i]]), exp[i]);
+    return normalizeCallback(res);
+  }
+
+  if (op === "CASE") {
+    const res = [] as [Expression, Expression][];
+    for (let i = 1; i < exp.length; i += 2) {
+      let w = exp[i];
+      if (w instanceof Polynomial) w = JSON.parse(w.toString());
+      if (!Array.isArray(w) && !w) continue;
+      const t = exp[i + 1];
+      if (!Array.isArray(t) || t[0] !== "CASE") {
+        res.push([w, t]);
+        continue;
+      }
+      for (let j = 1; j < t.length; j += 2) res.push([and(w, t[j]), t[j + 1]]);
+      res.push([w, null]);
+      if (!Array.isArray(w) && w) break;
+    }
+    while (res[res.length - 1][1] == null) res.pop();
+    return ["CASE", ...res.flat()];
+  }
+
+  const permutations: Map<number, [Expression, Expression][]> = new Map();
+  for (const [i, e] of exp.entries()) {
+    if (!Array.isArray(e)) continue;
+    if (e[0] !== "CASE") continue;
+    const perms: [Expression, Expression][] = [];
+    for (let j = 1; j < e.length; j += 2) perms.push([e[j], e[j + 1]]);
+    perms.push([true, null]);
+    permutations.set(i, perms);
+  }
+
+  if (permutations.size) {
+    let res: [Expression, Expression][] = [[true, exp]];
+    for (const [i, perms] of permutations) {
+      const res2: [Expression, Expression][] = [];
+      for (const [w, t] of perms) {
+        res2.push(
+          ...res.map((r) => {
+            const e = (r[1] as Expression[]).slice();
+            e[i] = t;
+            return [and(w, r[0]), e] as [Expression, Expression];
+          }),
+        );
+        if (!Array.isArray(w) && w) break;
+      }
+      res = res2;
+    }
+
+    for (const r of res) r[1] = normalizeCallback(r[1]);
+
+    if (res[0][0] === true) return res[0][1];
+    while (res[res.length - 1][1] == null) res.pop();
+
+    return ["CASE", ...res.flat()];
+  }
+
+  function toPolynomial(e: Expression): Polynomial {
+    if (e == null) return null;
+    if (e instanceof Polynomial) return e;
+    if (typeof e === "number") return Polynomial.fromConstant(e);
+    if (typeof e === "string")
+      return Polynomial.fromConstant(parseFloat(e) || 0);
+    if (typeof e === "boolean") return Polynomial.fromConstant(+e);
+    return Polynomial.fromIndeterminate(e);
+  }
+
+  if (op === "+") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce(
+      (previousValue, currentValue) => previousValue.add(currentValue),
+      ADDITIVE_IDENTITY,
+    ) as unknown as Expression;
+  } else if (op === "*") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce(
+      (previousValue, currentValue) => previousValue.multiply(currentValue),
+      MULTIPLICATIVE_IDENTITY,
+    ) as unknown as Expression;
+  } else if (op === "-") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce((previousValue, currentValue) =>
+      previousValue.subtract(currentValue),
+    ) as unknown as Expression;
+  } else if (op === "/") {
+    const args: Polynomial[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const p = toPolynomial(exp[i]);
+      if (p == null) return null;
+      args.push(p);
+    }
+    return args.reduce((previousValue, currentValue) =>
+      previousValue.divide(currentValue),
+    ) as unknown as Expression;
+  } else if (["=", "<>", ">", ">=", "<", "<="].includes(op)) {
+    if (exp[1] == null || exp[2] == null) return null;
+    let lhs: Polynomial, rhs: Polynomial;
+    if (exp[1] instanceof Polynomial) lhs = exp[1];
+    else if (typeof exp[1] === "number") lhs = Polynomial.fromConstant(exp[1]);
+
+    if (exp[2] instanceof Polynomial) rhs = exp[2];
+    else if (typeof exp[2] === "number") rhs = Polynomial.fromConstant(exp[2]);
+
+    if (lhs || rhs) {
+      if (!lhs) lhs = Polynomial.fromIndeterminate(exp[1]);
+      if (!rhs) rhs = Polynomial.fromIndeterminate(exp[2]);
+
+      lhs = lhs.subtract(rhs);
+      rhs = lhs.constant().negation();
+      lhs = lhs.add(rhs);
+
+      if (!lhs.terms.length) {
+        exp = [op, JSON.parse(lhs.toString()), JSON.parse(rhs.toString())];
+
+        if (op === "=") return exp[1] === exp[2];
+        else if (op === "<>") return exp[1] !== exp[2];
+        else if (op === ">") return exp[1] > exp[2];
+        else if (op === ">=") return exp[1] >= exp[2];
+        else if (op === "<") return exp[1] < exp[2];
+        else if (op === "<=") return exp[1] <= exp[2];
+      } else {
+        let flipOp = 1;
+
+        const n = lhs.terms[0].coefficientNumerator;
+        const d = lhs.terms[0].coefficientDenominator;
+
+        if (BI.lt(n, ZERO) || BI.lt(d, ZERO)) flipOp *= -1;
+
+        const reciprocal = new Polynomial([
+          {
+            indeterminates: new Indeterminates(),
+            coefficientNumerator: d,
+            coefficientDenominator: n,
+          },
+        ]);
+
+        lhs = lhs.multiply(reciprocal);
+        rhs = rhs.multiply(reciprocal);
+
+        const keys = lhs.terms[0].indeterminates.sortedKeys;
+        let invert = lhs.terms[0].indeterminates.map.get(keys[0]) < 0 ? -1 : 0;
+
+        for (const t of lhs.terms)
+          for (const v of t.indeterminates.map.values()) invert += v;
+
+        if (invert < 0) {
+          flipOp *= -1;
+          lhs = lhs.reciprocal();
+          rhs = rhs.reciprocal();
+        }
+
+        if (flipOp < 0) exp = [SWAPPED_OPS[op], lhs, rhs];
+        else exp = [op, lhs, rhs];
+      }
+    }
+  }
+
+  // Restore polynomial expressions
+  exp = exp.map((e) =>
+    e instanceof Polynomial ? JSON.parse(e.toString()) : e,
+  );
+
+  exp = evaluateCallback(exp);
+
+  return exp;
+}
+
+export default function normalize(expr: Expression): Expression {
+  expr = map(expr, normalizeCallback);
+  if (expr instanceof Polynomial) {
+    expr = JSON.parse(expr.toString());
+  } else if (Array.isArray(expr) && expr[0] === "CASE") {
+    expr = expr.map((e) =>
+      e instanceof Polynomial ? JSON.parse(e.toString()) : e,
+    );
+  }
+  return expr;
+}

lib/common/expression/pagination.ts

@@ -0,0 +1,148 @@
+import { complement } from "espresso-iisojs";
+import { Expression } from "../../types";
+import normalize from "./normalize";
+import { Clause, SynthContext } from "./synth";
+import { and, or } from "./util";
+
+type Bookmark = Record<string, null | boolean | number | string>;
+
+type Minterm = number[];
+
+export function toBookmark(
+  sort: Record<string, number>,
+  row: unknown,
+): Bookmark {
+  const bookmark: Bookmark = {};
+  for (const param of Object.keys(sort)) {
+    let v = row[param];
+    if (v != null && typeof v === "object") v = v.value?.[0];
+    bookmark[param] = v;
+  }
+  return bookmark;
+}
+
+export function bookmarkToExpression(
+  bookmark: Bookmark,
+  sort: Record<string, number>,
+): Expression {
+  return Object.entries(sort)
+    .reverse()
+    .reduce((cur, kv) => {
+      const [param, asc] = kv;
+      if (asc < 0) {
+        if (bookmark[param] == null) {
+          return or(
+            ["IS NOT NULL", ["PARAM", param]],
+            and(["IS NULL", ["PARAM", param]], cur),
+          );
+        }
+        let f = null;
+        f = or(f, [">", ["PARAM", param], bookmark[param]]);
+        return or(f, and(["=", ["PARAM", param], bookmark[param]], cur));
+      } else {
+        let f: Expression = ["IS NULL", ["PARAM", param]];
+        if (bookmark[param] == null) return and(f, cur);
+        f = or(f, ["<", ["PARAM", param], bookmark[param]]);
+        return or(f, and(["=", ["PARAM", param], bookmark[param]], cur));
+      }
+    }, true as Expression);
+}
+
+function getCover(
+  context: SynthContext,
+  minterm: Minterm,
+  allSort: [string, number][],
+): Minterm[] {
+  if (!allSort.length) return [[]];
+  const [param, sort] = allSort[0];
+
+  const cov: Minterm = [];
+  const nextCov: Minterm = [];
+
+  if (sort > 0) {
+    const lhs = new Clause.Exp(["PARAM", param]);
+    const isNull = context.getVar(new Clause.IsNull(lhs));
+    cov.push((isNull << 2) ^ 2);
+  }
+
+  for (const m of minterm) {
+    const clause = context.getClause(m >>> 2);
+    if (clause instanceof Clause.IsNull) {
+      const lhs = clause.operand.expression();
+      if (!Array.isArray(lhs) || lhs[0] !== "PARAM") continue;
+      if (lhs[1] !== param) continue;
+      nextCov.push(m);
+      if (sort < 0 && m & 1) cov.push(m, m ^ 1);
+    } else if (clause instanceof Clause.Compare) {
+      const lhs = clause.lhs.expression();
+      if (!Array.isArray(lhs) || lhs[0] !== "PARAM") continue;
+      if (lhs[1] !== param) continue;
+      if (!(m & 1) && sort > 0) continue;
+      nextCov.push(m);
+
+      const negate = (m ^ (m >> 1)) & 1;
+
+      if (sort > 0) {
+        if (
+          (clause.op === "=" && !negate) ||
+          (clause.op === ">" && !negate) ||
+          (clause.op === "<" && negate)
+        ) {
+          const c = new Clause.Compare(clause.lhs, ">", clause.rhs);
+          const v = context.getVar(c);
+          cov.push((v << 2) ^ 3);
+        }
+      } else if (sort < 0) {
+        if (
+          (clause.op === "=" && !negate) ||
+          (clause.op === ">" && negate) ||
+          (clause.op === "<" && !negate)
+        ) {
+          const c = new Clause.Compare(clause.lhs, "<", clause.rhs);
+          const v = context.getVar(c);
+          cov.push((v << 2) ^ 0);
+        }
+      }
+    }
+  }
+
+  const next = getCover(context, minterm, allSort.slice(1));
+
+  return [cov, ...next.map((n) => [...nextCov, ...n])];
+}
+
+export function paginate(
+  fetched: Expression,
+  toFetch: Expression,
+  sort: Record<string, number>,
+): [Expression, Expression] {
+  fetched = normalize(fetched);
+  if (!fetched) return [false, toFetch];
+
+  toFetch = normalize(toFetch);
+  if (!toFetch) return [false, toFetch];
+
+  const synth1 = Clause.fromExpression(fetched);
+  const synth2 = Clause.fromExpression(toFetch);
+
+  const context = new SynthContext();
+
+  const expr1Minterms = synth1.true(context);
+  const expr2MintermsC = complement(synth2.true(context));
+
+  const gaps = context.sanitizeMinterms(
+    complement([
+      ...expr1Minterms,
+      ...expr2MintermsC,
+      ...context.getDcSet([...expr1Minterms, ...expr2MintermsC]),
+    ]),
+  );
+
+  const cover = gaps.flatMap((m) => getCover(context, m, Object.entries(sort)));
+  const minterms1 = context.minimize(complement([...cover, ...expr2MintermsC]));
+  const minterms2 = context.minimize(
+    complement([...minterms1, ...expr2MintermsC]),
+  );
+
+  return [context.toExpression(minterms1), context.toExpression(minterms2)];
+}

lib/common/expression/parser.ts

@@ -0,0 +1,459 @@
+import parsimmon from "parsimmon";
+import { Expression } from "../../types.ts";
+
+// Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
+function interpretEscapes(str): string {
+  const escapes = {
+    b: "\b",
+    f: "\f",
+    n: "\n",
+    r: "\r",
+    t: "\t",
+  };
+  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/g, (_, escape) => {
+    const type = escape.charAt(0);
+    const hex = escape.slice(1);
+    if (type === "u") return String.fromCharCode(parseInt(hex, 16));
+
+    if (escapes.hasOwnProperty(type)) return escapes[type];
+
+    return type;
+  });
+}
+
+export function map<T>(exp: Expression, callback: (e: Expression) => T): T {
+  if (!Array.isArray(exp)) return callback(exp);
+
+  let clone;
+  for (let i = 1; i < exp.length; ++i) {
+    const sub = map(exp[i], callback);
+    if (sub !== exp[i]) {
+      clone = clone || exp.slice();
+      clone[i] = sub;
+    }
+  }
+
+  return callback(clone || exp);
+}
+
+export async function mapAsync(
+  exp: Expression,
+  callback: (e: Expression) => Promise<Expression>,
+): Promise<Expression> {
+  if (!Array.isArray(exp)) return callback(exp);
+
+  let clone;
+  for (let i = 1; i < exp.length; ++i) {
+    const sub = await mapAsync(exp[i], callback);
+    if (sub !== exp[i]) {
+      clone = clone || exp.slice();
+      clone[i] = sub;
+    }
+  }
+
+  return callback(clone || exp);
+}
+
+function binaryLeft(
+  operatorsParser: parsimmon.Parser<string>,
+  nextParser: parsimmon.Parser<Expression>,
+): parsimmon.Parser<Expression> {
+  return parsimmon.seqMap(
+    nextParser,
+    parsimmon.seq(operatorsParser, nextParser).many(),
+    (first, rest) =>
+      rest.reduce((acc, ch) => {
+        const [op, another] = ch;
+        if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
+        return [op, acc, another];
+      }, first),
+  );
+}
+
+const lang = parsimmon.createLanguage({
+  ComparisonOperator: function () {
+    return parsimmon
+      .alt(
+        parsimmon.string(">="),
+        parsimmon.string("<>"),
+        parsimmon.string("<="),
+        parsimmon.string("="),
+        parsimmon.string(">"),
+        parsimmon.string("<"),
+      )
+      .skip(parsimmon.optWhitespace);
+  },
+  LikeOperator: function () {
+    return parsimmon
+      .alt(
+        parsimmon.regexp(/like/i).result("LIKE").desc("LIKE"),
+        parsimmon
+          .regexp(/not\s+like/i)
+          .result("NOT LIKE")
+          .desc("NOT LIKE"),
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  IsNullOperator: function () {
+    return parsimmon
+      .alt(
+        parsimmon
+          .regexp(/is\s+null/i)
+          .result("IS NULL")
+          .desc("IS NULL"),
+        parsimmon
+          .regexp(/is\s+not\s+null/i)
+          .result("IS NOT NULL")
+          .desc("IS NOT NULL"),
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  NotOperator: function () {
+    return parsimmon
+      .regexp(/not/i)
+      .result("NOT")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("NOT");
+  },
+  AndOperator: function () {
+    return parsimmon
+      .regexp(/and/i)
+      .result("AND")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("AND");
+  },
+  OrOperator: function () {
+    return parsimmon
+      .regexp(/or/i)
+      .result("OR")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("OR");
+  },
+  Parameter: function (r) {
+    return parsimmon
+      .alt(
+        parsimmon.regexp(/[a-zA-Z0-9_.*-]+/),
+        r.Expression.wrap(
+          parsimmon.string("{").skip(parsimmon.optWhitespace),
+          parsimmon.string("}"),
+        ),
+      )
+      .atLeast(1)
+      .map((x) => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
+      .skip(parsimmon.optWhitespace)
+      .desc("parameter");
+  },
+  StringValueSql: function () {
+    return parsimmon
+      .regexp(/'([^']*)'/, 1)
+      .atLeast(1)
+      .skip(parsimmon.optWhitespace)
+      .map((s) => s.join("'"))
+      .desc("string");
+  },
+  StringValueJs: function () {
+    return parsimmon
+      .regexp(/"((?:\\.|.)*?)"/, 1)
+      .skip(parsimmon.optWhitespace)
+      .map(interpretEscapes)
+      .desc("string");
+  },
+  NumberValue: function () {
+    return parsimmon
+      .regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/)
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .map(Number)
+      .desc("number");
+  },
+  BooleanValue: function () {
+    return parsimmon
+      .alt(
+        parsimmon.regexp(/true/i).result(true).desc("TRUE"),
+        parsimmon.regexp(/false/i).result(false).desc("FALSE"),
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  NullValue: function () {
+    return parsimmon
+      .regexp(/null/i)
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .result(null)
+      .desc("NULL");
+  },
+  FuncValue: function (r) {
+    return parsimmon.seqMap(
+      parsimmon
+        .regexp(/([a-zA-Z0-9_]+)/, 1)
+        .skip(parsimmon.optWhitespace)
+        .desc("function"),
+      r.ExpressionList.wrap(
+        parsimmon.string("(").skip(parsimmon.optWhitespace),
+        parsimmon.string(")").skip(parsimmon.optWhitespace),
+      ),
+      (f, args) => ["FUNC", f.toUpperCase()].concat(args),
+    );
+  },
+  WhenPair: function (r) {
+    return parsimmon.seq(
+      parsimmon
+        .regexp(/when/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("WHEN")
+        .then(r.Expression),
+      parsimmon
+        .regexp(/then/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("THEN")
+        .then(r.Expression),
+    );
+  },
+  CaseStatement: function (r) {
+    return parsimmon.seqMap(
+      parsimmon
+        .regexp(/case/i)
+        .result("CASE")
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("CASE"),
+      r.WhenPair.many(),
+      parsimmon
+        .regexp(/else/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("ELSE")
+        .then(r.Expression)
+        .map((e) => [[true, e]])
+        .fallback(null)
+        .skip(
+          parsimmon
+            .regex(/end/i)
+            .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/)),
+        )
+        .skip(parsimmon.optWhitespace),
+      (...arr) => arr.flat(2),
+    );
+  },
+  Value: function (r) {
+    return parsimmon.alt(
+      r.NullValue,
+      r.BooleanValue,
+      r.NumberValue,
+      r.StringValueSql,
+      r.StringValueJs,
+      r.FuncValue,
+      r.CaseStatement,
+    );
+  },
+  ValueExpression: function (r) {
+    return binaryLeft(
+      parsimmon.string("||").skip(parsimmon.optWhitespace),
+      binaryLeft(
+        parsimmon
+          .alt(parsimmon.string("+"), parsimmon.string("-"))
+          .skip(parsimmon.optWhitespace),
+        binaryLeft(
+          parsimmon
+            .alt(
+              parsimmon.string("*"),
+              parsimmon.string("/"),
+              parsimmon.string("%"),
+            )
+            .skip(parsimmon.optWhitespace),
+          parsimmon.alt(
+            r.Value,
+            r.Parameter,
+            r.Expression.wrap(
+              parsimmon.string("(").skip(parsimmon.optWhitespace),
+              parsimmon.string(")").skip(parsimmon.optWhitespace),
+            ),
+          ),
+        ),
+      ),
+    );
+  },
+  Comparison: function (r) {
+    return parsimmon.alt(
+      parsimmon.seqMap(r.ValueExpression, r.IsNullOperator, (p, o) => [o, p]),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.ComparisonOperator,
+        r.ValueExpression,
+        (p, o, v) => [o, p, v],
+      ),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.LikeOperator,
+        r.ValueExpression.skip(
+          parsimmon
+            .regexp(/escape/i)
+            .result("ESCAPE")
+            .skip(parsimmon.whitespace)
+            .desc("ESCAPE"),
+        ),
+        r.ValueExpression,
+        (a, b, c, d) => [b, a, c, d],
+      ),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.LikeOperator,
+        r.ValueExpression,
+        (a, b, c) => [b, a, c],
+      ),
+    );
+  },
+  ExpressionList: function (r) {
+    return r.Expression.sepBy(
+      parsimmon.string(",").skip(parsimmon.optWhitespace),
+    );
+  },
+  Expression: function (r) {
+    function unary(
+      operatorsParser: parsimmon.Parser<string>,
+      nextParser: parsimmon.Parser<Expression>,
+    ): parsimmon.Parser<Expression> {
+      return parsimmon.seq(operatorsParser, nextParser).or(nextParser);
+    }
+
+    return binaryLeft(
+      r.OrOperator,
+      binaryLeft(
+        r.AndOperator,
+        unary(r.NotOperator, r.Comparison.or(r.ValueExpression)),
+      ),
+    ).trim(parsimmon.optWhitespace);
+  },
+});
+
+export function parse(str: string): Expression {
+  if (!str) return null;
+  return lang.Expression.tryParse(str);
+}
+
+export function parseList(str: string): Expression[] {
+  if (!str) return [];
+  return lang.ExpressionList.tryParse(str);
+}
+
+export function stringify(exp: Expression, level = 0): string {
+  if (!Array.isArray(exp)) return JSON.stringify(exp);
+
+  const opLevels = {
+    OR: 10,
+    AND: 11,
+    NOT: 12,
+    "=": 20,
+    "<>": 20,
+    ">": 20,
+    ">=": 20,
+    "<": 20,
+    "<=": 20,
+    LIKE: 20,
+    "NOT LIKE": 20,
+    "IS NULL": 20,
+    "IS NOT NULL": 20,
+    "||": 30,
+    "+": 31,
+    "-": 31,
+    "*": 32,
+    "/": 32,
+    "%": 32,
+  };
+
+  const op = exp[0].toUpperCase();
+
+  function wrap(e): string {
+    if (opLevels[op] <= level) return `(${e})`;
+    else return e;
+  }
+
+  if (op === "FUNC") {
+    return wrap(
+      `${exp[1]}(${exp
+        .slice(2)
+        .map((e) => stringify(e))
+        .join(", ")})`,
+    );
+  } else if (op === "PARAM") {
+    if (typeof exp[1] === "string") {
+      return wrap(exp[1]);
+    } else if (Array.isArray(exp[1]) && exp[1][0] === "||") {
+      return wrap(
+        exp[1]
+          .slice(1)
+          .map((p) => {
+            if (typeof p === "string") return p;
+            else return `{${stringify(p)}}`;
+          })
+          .join(""),
+      );
+    } else {
+      return wrap(`{${stringify(exp[1])}}`);
+    }
+  } else if (op === "IS NULL" || op === "IS NOT NULL") {
+    return wrap(`${stringify(exp[1], opLevels[op])} ${op}`);
+  } else if (op === "LIKE" || op === "NOT LIKE") {
+    if (exp[3]) {
+      return wrap(
+        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
+          exp[2],
+          opLevels[op],
+        )} ESCAPE ${stringify(exp[3], opLevels[op])}`,
+      );
+    } else {
+      return wrap(
+        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
+          exp[2],
+          opLevels[op],
+        )}`,
+      );
+    }
+  } else if (op === "CASE") {
+    const parts: string[] = ["CASE"];
+    for (let i = 1; i < exp.length - 1; i += 2) {
+      if (!Array.isArray(exp[i]) && exp[i]) {
+        if (exp[i + 1] != null) parts.push("ELSE", stringify(exp[i + 1]));
+        break;
+      }
+      parts.push("WHEN", stringify(exp[i]), "THEN", stringify(exp[i + 1]));
+    }
+    parts.push("END");
+    return parts.join(" ");
+  } else if (op in opLevels) {
+    const parts = exp.slice(1).map((e, i) => {
+      return stringify(e, opLevels[exp[0]] + Math.min(i - 1, 0));
+    });
+
+    if (op === "NOT") return wrap(`${op} ${parts[0]}`);
+    else return wrap(parts.join(` ${op} `));
+  } else {
+    throw new Error(`Unrecognized operator ${exp[0]}`);
+  }
+}
+
+export function parseLikePattern(pat: string, esc: string): string[] {
+  const chars = pat.split("");
+
+  for (let i = 0; i < chars.length; ++i) {
+    const c = chars[i];
+    if (c === esc) {
+      chars[i] = chars[i + 1] || "";
+      chars[i + 1] = "";
+    } else if (c === "_") {
+      chars[i] = "\\_";
+    } else if (c === "%") {
+      chars[i] = "\\%";
+      while (chars[i + 1] === "%") chars[++i] = "";
+    }
+  }
+  return chars.filter((c) => c);
+}

lib/common/expression/synth.ts

@@ -0,0 +1,972 @@
+import { espresso, complement, tautology } from "espresso-iisojs";
+import { Expression } from "../../types.ts";
+import normalize from "./normalize.ts";
+import { map, parseLikePattern } from "./parser.ts";
+
+type Minterm = number[];
+
+export abstract class SynthContextBase<
+  T extends { toString: () => string } = unknown,
+  U = unknown,
+> {
+  public variables = new Map<string, number>();
+  protected clauses = new Map<number, U>();
+
+  public getVar(c: U): number {
+    const str = c.toString();
+    let idx = this.variables.get(str);
+    if (idx == null) {
+      idx = this.variables.size;
+      this.variables.set(str, idx);
+      this.clauses.set(idx, c);
+    }
+    return idx;
+  }
+
+  public getClause(v: number): U {
+    return this.clauses.get(v);
+  }
+
+  abstract getMinterms(exp: T, res: true | false | null): Minterm[];
+  abstract getDcSet(minterms: Minterm[]): Minterm[];
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  canRaise(idx: number, set: Set<number>): boolean {
+    return true;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  canLower(idx: number, set: Set<number>): boolean {
+    return true;
+  }
+
+  bias(a: number, b: number): number {
+    // Bias towards 1 then lower index
+    return (b ^ 1) - (a ^ 1);
+  }
+
+  sanitizeMinterms(minterms: Minterm[]): Minterm[] {
+    return minterms;
+  }
+
+  minimize(minterms: Minterm[], dcSet: Minterm[] = []): Minterm[] {
+    minterms = this.sanitizeMinterms(minterms);
+    const canRaise = this.canRaise.bind(this);
+    const canLower = this.canLower.bind(this);
+    const bias = this.bias.bind(this);
+
+    return espresso(
+      minterms,
+      [...this.getDcSet([...minterms, ...dcSet]), ...dcSet],
+      {
+        canRaise,
+        canLower,
+        bias,
+      },
+    );
+  }
+}
+
+function* findIsNullDeps(exp: Expression): IterableIterator<Expression> {
+  if (!Array.isArray(exp)) return;
+  const op = exp[0];
+  if (op === "IS NULL" || op === "IS NOT NULL") return;
+
+  if (op === "FUNC") {
+    if (exp[1] === "NOW") {
+      return;
+    } else if (exp[1] === "LOWER" || exp[1] === "UPPER") {
+      yield* findIsNullDeps(exp[2]);
+    } else if (exp[1] === "ROUND") {
+      for (const e of exp.slice(2, 4)) yield* findIsNullDeps(e);
+      return;
+    }
+  } else if (op !== "PARAM") {
+    for (const e of exp.slice(1)) yield* findIsNullDeps(e);
+    return;
+  }
+  yield exp;
+}
+
+export abstract class Clause {
+  private _isNullable: Set<string>;
+  protected _expression: Expression;
+  abstract true(context: SynthContextBase<Clause>): Minterm[];
+  abstract false(context: SynthContextBase<Clause>): Minterm[];
+  abstract null(context: SynthContextBase<Clause>): Minterm[];
+  expression(): Expression {
+    if (this._expression !== undefined) return this._expression;
+    const context = createSynthContext();
+    const minterms = this.true(context);
+    const minimized = context.minimize(minterms);
+    this._expression = context.toExpression(minimized) as Expression;
+    return this._expression;
+  }
+  isBoolean(): boolean {
+    return true;
+  }
+  isNullable(c: Clause.IsNull): boolean {
+    if (!this._isNullable) {
+      this._isNullable = new Set(
+        [...this.getNullables()].map((n) => n.toString()),
+      );
+    }
+    return this._isNullable.has(c.toString());
+  }
+  *getNullables(): IterableIterator<Clause.IsNull> {
+    const exp = this.expression();
+    for (const e of findIsNullDeps(exp)) {
+      if (e === exp) yield new Clause.IsNull(this);
+      else yield new Clause.IsNull(new Clause.Exp(e));
+    }
+  }
+  toString(): string {
+    return JSON.stringify(this.expression());
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/no-namespace
+export namespace Clause {
+  export class Not extends Clause {
+    constructor(public operand: Clause) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.false(context);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.true(context);
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.null(context);
+    }
+  }
+
+  export class Or extends Clause {
+    constructor(public operands: Clause[]) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      for (const s of this.operands) {
+        const m = s.true(context);
+        if (m.length === 1 && !m[0].length) return [[]];
+        minterms.push(...m);
+      }
+      return minterms;
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[][] = [];
+      const operands = [...this.operands];
+      for (const e of operands) {
+        if (e instanceof Or) {
+          operands.push(...e.operands);
+          continue;
+        }
+        const m = e.false(context);
+        if (!m.length) return [];
+        minterms.push(m);
+      }
+      if (!minterms.length) return [[]];
+      if (minterms.length === 1) return minterms[0];
+      return complement(minterms.map((m) => complement(m)).flat());
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      const trueMinterms: Minterm[] = [];
+      const operands = [...this.operands];
+      for (const s of operands) {
+        if (s instanceof Or) {
+          operands.push(...s.operands);
+          continue;
+        }
+        const t = s.true(context);
+        if (t.length === 1 && !t[0].length) return [];
+        trueMinterms.push(...t);
+      }
+
+      const nullMinterms: Minterm[] = [];
+      for (const s of operands) {
+        const n = s.null(context);
+        if (n.length === 1 && !n[0].length && !trueMinterms.length) return [[]];
+        nullMinterms.push(...n);
+      }
+
+      if (!trueMinterms.length) return nullMinterms;
+      return complement([...complement(nullMinterms), ...trueMinterms]);
+    }
+  }
+
+  export class And extends Clause {
+    constructor(public operands: Clause[]) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[][] = [];
+      const operands = [...this.operands];
+      for (const s of operands) {
+        if (s instanceof And) {
+          operands.push(...s.operands);
+          continue;
+        }
+        const m = s.true(context);
+        if (!m.length) return [];
+        if (m.length === 1 && !m[0].length) continue;
+        minterms.push(m);
+      }
+      if (!minterms.length) return [[]];
+      if (minterms.length === 1) return minterms[0];
+      return complement(minterms.map((m) => complement(m)).flat());
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      for (const s of this.operands) {
+        const m = s.false(context);
+        if (m.length === 1 && !m[0].length) return [[]];
+        minterms.push(...m);
+      }
+      return minterms;
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      const falseMinterms: Minterm[] = [];
+      const operands = [...this.operands];
+      for (const s of operands) {
+        if (s instanceof And) {
+          operands.push(...s.operands);
+          continue;
+        }
+        const f = s.false(context);
+        if (f.length === 1 && !f[0].length) return [];
+        falseMinterms.push(...f);
+      }
+
+      const nullMinterms: Minterm[] = [];
+      for (const s of operands) {
+        const n = s.null(context);
+        if (n.length === 1 && !n[0].length && !falseMinterms.length)
+          return [[]];
+        nullMinterms.push(...n);
+      }
+
+      if (!falseMinterms.length) return nullMinterms;
+      return complement([...complement(nullMinterms), ...falseMinterms]);
+    }
+  }
+
+  export class Case extends Clause {
+    constructor(public clauses: Clause[]) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      const cumulative: Minterm[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        const w = this.clauses[i].true(context);
+        const t = this.clauses[i + 1].true(context);
+        minterms.push(
+          ...complement([...cumulative, ...complement(w), ...complement(t)]),
+        );
+        if (i < this.clauses.length - 2) {
+          cumulative.push(
+            ...complement([
+              ...this.clauses[i].false(context),
+              ...this.clauses[i].null(context),
+            ]),
+          );
+        }
+      }
+      return minterms;
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      const cumulative: Minterm[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        const w = this.clauses[i].true(context);
+        const t = this.clauses[i + 1].false(context);
+        minterms.push(
+          ...complement([...cumulative, ...complement(w), ...complement(t)]),
+        );
+        if (i < this.clauses.length - 2) {
+          cumulative.push(
+            ...complement([
+              ...this.clauses[i].false(context),
+              ...this.clauses[i].null(context),
+            ]),
+          );
+        }
+      }
+      return minterms;
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      const minterms: Minterm[] = [];
+      const cumulative: Minterm[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        const w = this.clauses[i].true(context);
+        const t = this.clauses[i + 1].null(context);
+        minterms.push(
+          ...complement([...cumulative, ...complement(w), ...complement(t)]),
+        );
+        cumulative.push(
+          ...complement([
+            ...this.clauses[i].false(context),
+            ...this.clauses[i].null(context),
+          ]),
+        );
+      }
+      minterms.push(...complement([...cumulative]));
+      return minterms;
+    }
+    expression(): Expression {
+      if (this._expression != null) return this._expression;
+      if (this.isBoolean()) {
+        this._expression = new Clause.Not(new Clause.Not(this)).expression();
+        return this._expression;
+      }
+      const context = createSynthContext();
+      const cases: { when: Minterm[]; then: Expression }[] = [];
+      for (let i = 0; i < this.clauses.length; i += 2) {
+        let minterms = this.clauses[i].true(context);
+        const then = this.clauses[i + 1].expression();
+        if (
+          cases.length &&
+          JSON.stringify(then) === JSON.stringify(cases[cases.length - 1].then)
+        )
+          minterms.push(...cases.pop().when);
+        minterms = context.minimize(
+          minterms,
+          cases.flatMap((c) => c.when),
+        );
+        if (!minterms.length) continue;
+        cases.push({ when: minterms, then });
+        if (minterms.length === 1 && !minterms[0].length) break;
+      }
+      while (cases[cases.length - 1].then == null) cases.pop();
+      if (!cases.length) return null;
+      this._expression = [
+        "CASE",
+        ...cases.flatMap((c) => [context.toExpression(c.when), c.then]),
+      ];
+      return this._expression;
+    }
+
+    isBoolean(): boolean {
+      for (let i = 1; i < this.clauses.length; i += 2)
+        if (!this.clauses[i].isBoolean()) return false;
+      return true;
+    }
+  }
+
+  export class IsNull extends Clause {
+    private _boolean: Clause;
+    constructor(public operand: Clause) {
+      super();
+      this._boolean = new Not(new Not(this));
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return this.operand.null(context);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return complement(this.true(context));
+    }
+    null(): Minterm[] {
+      return [];
+    }
+    isBoolean(): boolean {
+      return true;
+    }
+    *getNullables(): IterableIterator<IsNull> {
+      // Never returns null
+    }
+    expression(): Expression {
+      const nullables = [...this.operand.getNullables()];
+      if (nullables.length === 1 && nullables[0].operand === this.operand)
+        return ["IS NULL", this.operand.expression()];
+      return super.expression();
+    }
+  }
+
+  export class Exp extends Clause {
+    constructor(public exp: Expression) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      if (Array.isArray(this.exp)) return context.getMinterms(this, true);
+      if (this.exp) return [[]];
+      return [];
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      if (Array.isArray(this.exp)) return context.getMinterms(this, false);
+      if (!this.exp && this.exp != null) return [[]];
+      return [];
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      if (Array.isArray(this.exp)) return context.getMinterms(this, null);
+      if (this.exp == null) return [[]];
+      return [];
+    }
+    expression(): Expression {
+      return this.exp;
+    }
+    isBoolean(): boolean {
+      return this.exp === true || this.exp === false || this.exp == null;
+    }
+  }
+
+  export class Compare extends Clause {
+    constructor(
+      public lhs: Clause,
+      public op: ">" | "<" | "=",
+      public rhs: boolean | number | string,
+    ) {
+      super();
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, true);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, false);
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      return this.lhs.null(context);
+    }
+    *getNullables(): IterableIterator<IsNull> {
+      yield* this.lhs.getNullables();
+    }
+    isBoolean(): boolean {
+      return true;
+    }
+    expression(): Expression {
+      return [this.op, this.lhs.expression(), this.rhs];
+    }
+  }
+
+  export class Like extends Clause {
+    public readonly caseSensitive: boolean;
+    public readonly contradiction: boolean;
+    public readonly pattern: string[];
+    public readonly lhs: Clause;
+
+    constructor(
+      lhs: Clause,
+      public rhs: string,
+      public esc?: string,
+    ) {
+      super();
+      const exp = lhs.expression();
+      let caseSensitive = true;
+      let contradiction = false;
+      if (Array.isArray(exp)) {
+        if (exp[0] === "FUNC") {
+          const func = exp[1];
+          if (func === "UPPER" || func === "LOWER") {
+            const p = func === "UPPER" ? rhs.toUpperCase() : rhs.toLowerCase();
+            if (p === rhs) caseSensitive = false;
+            else contradiction = true;
+            lhs = new Exp(exp[2]);
+          }
+        }
+      }
+      this.lhs = lhs;
+      this.pattern = parseLikePattern(rhs, esc);
+      this.caseSensitive = caseSensitive;
+      this.contradiction = contradiction;
+    }
+    true(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, true);
+    }
+    false(context: SynthContextBase<Clause>): Minterm[] {
+      return context.getMinterms(this, false);
+    }
+    null(context: SynthContextBase<Clause>): Minterm[] {
+      return this.lhs.null(context);
+    }
+    isBoolean(): boolean {
+      return true;
+    }
+    isNullable(c: IsNull): boolean {
+      return this.lhs.isNullable(c);
+    }
+    getNullables(): IterableIterator<IsNull> {
+      return this.lhs.getNullables();
+    }
+    expression(): Expression {
+      let lhs = this.lhs.expression();
+      if (this.contradiction) {
+        if (this.rhs === this.rhs.toLocaleUpperCase())
+          lhs = ["FUNC", "LOWER", lhs];
+        else lhs = ["FUNC", "UPPER", lhs];
+      } else if (!this.caseSensitive) {
+        if (this.rhs === this.rhs.toLocaleUpperCase())
+          lhs = ["FUNC", "UPPER", lhs];
+        else lhs = ["FUNC", "LOWER", lhs];
+      }
+      return ["LIKE", lhs, this.rhs, ...(this.esc ? [this.esc] : [])];
+    }
+  }
+
+  export function fromExpression(exp: Expression): Clause {
+    const res = map(exp, (e) => {
+      if (!Array.isArray(e)) return new Clause.Exp(e);
+      let op = e[0];
+      let negate = true;
+      if (op === "NOT LIKE") op = "LIKE";
+      else if (op === "IS NOT NULL") op = "IS NULL";
+      else if (op === "<>") op = "=";
+      else if (op === ">=") op = "<";
+      else if (op === "<=") op = ">";
+      else negate = false;
+
+      let clause: Clause;
+      if (op === "IS NULL") {
+        clause = new Clause.IsNull(e[1]);
+      } else if (op === "NOT") {
+        clause = new Clause.Not(e[1]);
+      } else if (op === "OR") {
+        clause = new Clause.Or(e.slice(1));
+      } else if (op === "AND") {
+        clause = new Clause.And(e.slice(1));
+      } else if (op === "CASE") {
+        clause = new Clause.Case(e.slice(1));
+      } else if (op === "LIKE") {
+        const rhs = e[2] instanceof Exp ? e[2].expression() : null;
+        const esc = e[3] instanceof Exp ? e[3].expression() : null;
+        if (typeof rhs === "string" && typeof esc === "string")
+          clause = new Clause.Like(e[1], rhs, esc);
+        else if (typeof rhs === "string" && esc == null)
+          clause = new Clause.Like(e[1], rhs);
+      } else if (op === ">" || op === "<" || op === "=") {
+        const rhs = e[2] instanceof Exp ? e[2].expression() : null;
+        if (["boolean", "number", "string"].includes(typeof rhs)) {
+          clause = new Clause.Compare(
+            e[1],
+            op,
+            rhs as boolean | number | string,
+          );
+        }
+      }
+
+      if (!clause) {
+        const args = e.slice(1).map((a) => (a as Clause).expression());
+        clause = new Clause.Exp([op, ...args]);
+      }
+
+      if (negate) clause = new Clause.Not(clause);
+      return clause;
+    });
+
+    return res;
+  }
+}
+
+function groupBy<T, K>(
+  input: T[],
+  callback: (item: T) => K,
+): Iterable<[K, T[]]> {
+  const groups = new Map<K, T[]>();
+  for (const item of input) {
+    const key = callback(item);
+    let arr = groups.get(key);
+    if (!arr) groups.set(key, (arr = []));
+    arr.push(item);
+  }
+
+  return groups.entries();
+}
+
+export class SynthContext extends SynthContextBase<Clause, Clause> {
+  constructor() {
+    super();
+  }
+
+  getMinterms(clause: Clause, res: true | false | null): number[][] {
+    const v = this.getVar(clause);
+    if (res === true) return [[(v << 2) ^ 3]];
+    else if (res === false) return [[(v << 2) ^ 1]];
+    else return [[v << 2, (v << 2) ^ 2]];
+  }
+
+  getDcSet(minterms: Minterm[]): number[][] {
+    const dcSet: number[][] = [];
+
+    const whitelist = new Set([...minterms.flat()].map((v) => v >> 2));
+
+    const allClauses = [...whitelist].map((v) => this.getClause(v));
+    const comparisons = allClauses.filter(
+      (c) => c instanceof Clause.Compare,
+    ) as Clause.Compare[];
+
+    // Comparisons
+    for (const [, clauses] of groupBy(comparisons, (c) =>
+      JSON.stringify(c.lhs),
+    )) {
+      const lhs = clauses[0].lhs;
+      const values = new Set(clauses.map((c) => c.rhs));
+      const valuesSorted = [...values].sort((a, b) => {
+        const ta = typeof a;
+        const tb = typeof b;
+        if (ta === tb) return a > b ? 1 : -1;
+        else if (ta === "string") return 1;
+        else if (tb === "string") return -1;
+        return +a - +b;
+      });
+
+      for (const [i, v] of valuesSorted.entries()) {
+        const eq = this.getVar(new Clause.Compare(lhs, "=", v));
+        const gt = this.getVar(new Clause.Compare(lhs, ">", v));
+        const lt = this.getVar(new Clause.Compare(lhs, "<", v));
+
+        dcSet.push([(eq << 2) ^ 3, (gt << 2) ^ 3]);
+        dcSet.push([(lt << 2) ^ 3, (gt << 2) ^ 3]);
+        dcSet.push([(lt << 2) ^ 3, (eq << 2) ^ 3]);
+        dcSet.push([(lt << 2) ^ 1, (eq << 2) ^ 1, (gt << 2) ^ 1]);
+
+        const negEquivOp = [lt, eq, gt].filter((o) => !whitelist.has(o));
+        if (negEquivOp.length === 1) whitelist.add(negEquivOp[0]);
+
+        for (let j = 0; j < i; j++) {
+          const eq2 = this.getVar(
+            new Clause.Compare(lhs, "=", valuesSorted[j]),
+          );
+          const gt2 = this.getVar(
+            new Clause.Compare(lhs, ">", valuesSorted[j]),
+          );
+          const lt2 = this.getVar(
+            new Clause.Compare(lhs, "<", valuesSorted[j]),
+          );
+
+          // This is the minimum clauses required if all relavent vars
+          // were included in the DC set.
+          // dcSet.push([(eq << 2) ^ 3, (eq2 << 2) ^ 3]);
+          // dcSet.push([(lt << 2) ^ 1, (gt2 << 2) ^ 1]);
+
+          // But we use non-minimal set because intermediate vars
+          // between any two may not be present in the DC set.
+          dcSet.push([(gt2 << 2) ^ 1, (lt << 2) ^ 1]);
+          dcSet.push([(eq2 << 2) ^ 3, (lt << 2) ^ 1]);
+          dcSet.push([(lt2 << 2) ^ 3, (lt << 2) ^ 1]);
+          dcSet.push([(gt2 << 2) ^ 1, (gt << 2) ^ 3]);
+          dcSet.push([(gt2 << 2) ^ 1, (eq << 2) ^ 3]);
+          dcSet.push([(eq2 << 2) ^ 3, (gt << 2) ^ 3]);
+          dcSet.push([(eq2 << 2) ^ 3, (eq << 2) ^ 3]);
+          dcSet.push([(lt2 << 2) ^ 3, (gt << 2) ^ 3]);
+          dcSet.push([(lt2 << 2) ^ 3, (eq << 2) ^ 3]);
+        }
+      }
+    }
+
+    // LIKE
+    const likes = allClauses.filter(
+      (c) => c instanceof Clause.Like,
+    ) as Clause.Like[];
+
+    for (const [, clauses] of groupBy(likes, (c) => JSON.stringify(c.lhs))) {
+      for (let i1 = 0; i1 < clauses.length; ++i1) {
+        const l1 = clauses[i1];
+        if (l1.contradiction) {
+          dcSet.push([(this.getVar(l1) << 2) ^ 3]);
+          continue;
+        }
+        for (let i2 = i1 + 1; i2 < clauses.length; ++i2) {
+          const l2 = clauses[i2];
+          if (l2.contradiction) continue;
+          let p1 = l1.pattern;
+          let p2 = l2.pattern;
+          if (!l1.caseSensitive || !l2.caseSensitive) {
+            p1 = p1.map((c) => c.toLowerCase());
+            p2 = p2.map((c) => c.toLowerCase());
+          }
+          if (likeDisjoint(p1, p2)) {
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 3,
+              (this.getVar(l2) << 2) ^ 3,
+            ]);
+          } else if (
+            (!l1.caseSensitive || l2.caseSensitive) &&
+            likeImplies(p1, p2)
+          ) {
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 2,
+              (this.getVar(l2) << 2) ^ 3,
+            ]);
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 1,
+              (this.getVar(l2) << 2) ^ 0,
+            ]);
+          } else if (
+            (!l2.caseSensitive || l1.caseSensitive) &&
+            likeImplies(p2, p1)
+          ) {
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 3,
+              (this.getVar(l2) << 2) ^ 2,
+            ]);
+            dcSet.push([
+              (this.getVar(l1) << 2) ^ 0,
+              (this.getVar(l2) << 2) ^ 1,
+            ]);
+          }
+        }
+      }
+    }
+
+    for (const v of whitelist) {
+      const clause = this.getClause(v);
+      const nullables = [...clause.getNullables()].map((c) => this.getVar(c));
+      if (nullables.length) {
+        dcSet.push([
+          ...nullables.map((n) => (n << 2) ^ 2),
+          (v << 2) ^ 0,
+          (v << 2) ^ 2,
+        ]);
+        for (const n of nullables) {
+          dcSet.push([(n << 2) ^ 3, (v << 2) ^ 1]);
+          dcSet.push([(n << 2) ^ 3, (v << 2) ^ 3]);
+          whitelist.add(n);
+        }
+      }
+
+      if (clause instanceof Clause.IsNull) {
+        const str = clause.operand.toString();
+        if (str === '["PARAM","DeviceID.ID"]' || str === '["PARAM","_id"]')
+          dcSet.push([(v << 2) ^ 3]);
+      }
+    }
+    return dcSet.filter((m) => m.every((v) => whitelist.has(v >> 2)));
+  }
+
+  canRaise(idx: number, set: Set<number>): boolean {
+    const clause = this.getClause(idx >> 2);
+    if (clause instanceof Clause.IsNull) {
+      for (const i of set) {
+        if (i === idx || i & 1) continue;
+        const c = this.getClause(i >> 2);
+        if (c.isNullable(clause)) return false;
+      }
+      return true;
+    }
+    return !(idx & 1) || !set.has(idx ^ 3);
+  }
+
+  canLower(idx: number, set: Set<number>): boolean {
+    if (idx & 1) return true;
+    const clause = this.getClause(idx >> 2);
+    if (clause instanceof Clause.IsNull) return true;
+    return set.has(idx ^ 3) || set.has(idx ^ 1);
+  }
+
+  bias(a: number, b: number): number {
+    // Bias towards 1 then true
+    return ((b & 3) ^ 3) - ((a & 3) ^ 3);
+  }
+
+  sanitizeMinterms(minterms: Minterm[]): Minterm[] {
+    const res = [] as number[][];
+
+    loop: for (const m of minterms) {
+      const merged: Map<number, number> = new Map();
+      for (const i of m)
+        merged.set(i >> 2, (merged.get(i >> 2) || 0) | (1 << (i & 3)));
+      const minterm: number[] = [];
+      const perms: number[][] = [];
+      for (const [k, v] of merged) {
+        if ((v & 0b0011) === 0b0011) continue loop;
+        if ((v & 0b1100) === 0b1100) continue loop;
+        const clause = this.clauses.get(k);
+        if (!clause) throw new Error("Invalid literal");
+        if (clause instanceof Clause.IsNull) {
+          if (v === 0b0100) minterm.push((k << 2) ^ 2);
+          else if (v === 0b1000) minterm.push((k << 2) ^ 3);
+          else throw new Error("Invalid literal");
+          continue;
+        }
+        if ((v & 0b1010) === 0b1010) continue loop;
+        const isNullVars = [...clause.getNullables()].map((c) =>
+          this.getVar(c),
+        );
+        const t = k << 2;
+        if (v === 0b0101) {
+          if (isNullVars.length === 1) minterm.push((isNullVars[0] << 2) ^ 3);
+          else perms.push(isNullVars.map((n) => (n << 2) ^ 3));
+        } else if (v === 0b0001) {
+          perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 3]);
+        } else if (v === 0b0100) {
+          perms.push([...isNullVars.map((n) => (n << 2) ^ 3), t ^ 1]);
+        } else if (v & 0b1000) {
+          minterm.push(t ^ 3);
+        } else if (v & 0b0010) {
+          minterm.push(t ^ 1);
+        }
+      }
+      let ms = [minterm];
+      while (perms.length) {
+        const newMs: number[][] = [];
+        const perm = perms.pop();
+        for (const p of perm) newMs.push(...ms.map((mm) => [...mm, p]));
+        ms = newMs;
+      }
+      res.push(...ms);
+    }
+    return res;
+  }
+
+  toExpression(sop: number[][]): Expression {
+    if (!sop.length) return false;
+    const res: Expression[] = [];
+    for (const s of sop) {
+      if (!s.length) return true;
+      const conjs: Expression[] = [];
+      for (const i of s) {
+        const clause = this.getClause(i >>> 2);
+        if (!clause) throw new Error("Invalid literal");
+        if (clause instanceof Clause.IsNull) {
+          if (!(i & 2)) throw new Error("Invalid literal");
+        } else if (!(i & 1)) {
+          // Should never be reached if minimized correctly
+          const isNullVars = [...clause.getNullables()].map((c) =>
+            this.getVar(c),
+          );
+          conjs.push(
+            this.toExpression([
+              [i ^ 3],
+              ...isNullVars.map((n) => [(n << 2) ^ 3]),
+            ]),
+          );
+          continue;
+        }
+
+        let expr = clause.expression() as Expression;
+        if (!(i & 1) !== !(i & 2)) expr = ["NOT", expr];
+        if (
+          Array.isArray(expr) &&
+          expr[0] === "NOT" &&
+          Array.isArray(expr[1])
+        ) {
+          const e: Expression[] = expr[1];
+          if (e[0] === "IS NULL") expr = ["IS NOT NULL", ...e.slice(1)];
+          else if (e[0] === "LIKE") expr = ["NOT LIKE", ...e.slice(1)];
+          else if (e[0] === "=") expr = ["<>", ...e.slice(1)];
+          else if (e[0] === "<>") expr = ["=", ...e.slice(1)];
+          else if (e[0] === ">") expr = ["<=", ...e.slice(1)];
+          else if (e[0] === ">=") expr = ["<", ...e.slice(1)];
+          else if (e[0] === "<") expr = [">=", ...e.slice(1)];
+          else if (e[0] === "<=") expr = [">", ...e.slice(1)];
+          else if (e[0] === "NOT") expr = e[1];
+        }
+        conjs.push(expr);
+      }
+      if (conjs.length > 1) res.push(["AND", ...conjs]);
+      else res.push(conjs[0]);
+    }
+    if (res.length > 1) return ["OR", ...res];
+    return res[0];
+  }
+}
+
+// Classes aren't hoisted in JS but functions are. This function is used to
+// create a new SynthContext instance from inside the Clause class.
+export function createSynthContext(): SynthContext {
+  return new SynthContext();
+}
+
+export function likeImplies(pat1: string[], pat2: string[]): boolean {
+  let backtrack: [number, number] = null;
+
+  for (let i1 = 0, i2 = 0; ; ++i1, ++i2) {
+    while (i1 < pat1.length && pat1[i1] === "\\%") backtrack = [i1++, i2];
+
+    if (i2 >= pat2.length) return i1 >= pat1.length;
+
+    const c = i1 < pat1.length ? pat1[i1] : null;
+
+    if (c !== pat2[i2] && c !== "\\_") {
+      if (!backtrack) return false;
+      [i1, i2] = backtrack;
+      ++backtrack[1];
+    }
+  }
+}
+
+export function likeDisjoint(pat1: string[], pat2: string[]): boolean {
+  const left1Idx = pat1.indexOf("\\%");
+  const left2Idx = pat2.indexOf("\\%");
+  const right1Idx = pat1.lastIndexOf("\\%");
+  const right2Idx = pat2.lastIndexOf("\\%");
+
+  const left1 = pat1.slice(0, left1Idx !== -1 ? left1Idx : pat1.length);
+  const left2 = pat2.slice(0, left2Idx !== -1 ? left2Idx : pat2.length);
+  const right1 = pat1.slice(right1Idx !== -1 ? right1Idx + 1 : 0).reverse();
+  const right2 = pat2.slice(right2Idx !== -1 ? right2Idx + 1 : 0).reverse();
+
+  for (let i = 0; i < Math.min(left1.length, left2.length); ++i) {
+    if (left1[i] !== left2[i] && left1[i] !== "\\_" && left2[i] !== "\\_")
+      return true;
+  }
+
+  for (let i = 0; i < Math.min(right1.length, right2.length); ++i) {
+    if (right1[i] !== right2[i] && right1[i] !== "\\_" && right2[i] !== "\\_")
+      return true;
+  }
+
+  if (pat1.length === left1.length)
+    return pat2.filter((c) => c !== "\\%").length > pat1.length;
+  else if (pat2.length === left2.length)
+    return pat1.filter((c) => c !== "\\%").length > pat2.length;
+
+  return false;
+}
+
+export function minimize(expr: Expression, boolean = false): Expression {
+  let synth = Clause.fromExpression(normalize(expr));
+  if (boolean) synth = new Clause.Not(new Clause.Not(synth));
+  return synth.expression();
+}
+
+export function unionDiff(
+  expr1: Expression,
+  expr2: Expression,
+): [Expression, Expression] {
+  expr2 = normalize(expr2);
+
+  if (!expr2) return [expr1, false];
+
+  const synth2 = Clause.fromExpression(expr2);
+
+  if (!expr1) {
+    const e = synth2.expression();
+    return [e, e];
+  }
+
+  expr1 = normalize(expr1);
+  const synth1 = Clause.fromExpression(expr1);
+
+  const context = new SynthContext();
+
+  const expr2Minterms = synth2.true(context);
+  const expr1Minterms = synth1.true(context);
+
+  const union = context.minimize([...expr1Minterms, ...expr2Minterms]);
+
+  const diff = context.minimize(
+    complement([...expr1Minterms, ...complement(expr2Minterms)]),
+  );
+
+  return [context.toExpression(union), context.toExpression(diff)];
+}
+
+export function covers(expr1: Expression, expr2: Expression): boolean {
+  expr2 = normalize(expr2);
+  if (!expr2) return true;
+  expr1 = normalize(expr1);
+  if (!Array.isArray(expr1) && expr1) return true;
+
+  const synt1 = Clause.fromExpression(expr1);
+  const synt2 = Clause.fromExpression(expr2);
+
+  const context = new SynthContext();
+  const expr1Minterms = synt1.true(context);
+  const expr2Minterms = synt2.true(context);
+
+  return tautology([
+    ...context.sanitizeMinterms(complement(expr2Minterms)),
+    ...context.getDcSet([...expr2Minterms, ...expr1Minterms]),
+    ...context.sanitizeMinterms(expr1Minterms),
+  ]);
+}

lib/common/expression/util.ts

@@ -0,0 +1,393 @@
+import { map, mapAsync, parseLikePattern } from "./parser.ts";
+import { Expression } from "../../types.ts";
+
+const isArray = Array.isArray;
+
+const regExpCache = new WeakMap<any, RegExp>();
+
+const REDUCE_SKIP = {};
+function reduce(exp, callback): Expression {
+  let loop = true;
+  while (loop) {
+    loop = false;
+    for (let i = 2; i < exp.length; ++i) {
+      const res = callback(exp[i - 1], exp[i], i - 2);
+      if (res !== REDUCE_SKIP) {
+        loop = true;
+        exp = exp.slice();
+        exp.splice(i - 1, 2, res);
+      }
+    }
+  }
+  if (exp.length === 2) return exp[1];
+  return exp;
+}
+
+export function likePatternToRegExp(pat: string, esc = "", flags = ""): RegExp {
+  const convChars = {
+    "-": "\\-",
+    "/": "\\/",
+    "\\": "\\/",
+    "^": "\\^",
+    $: "\\$",
+    "*": "\\*",
+    "+": "\\+",
+    "?": "\\?",
+    ".": "\\.",
+    "(": "\\(",
+    ")": "\\)",
+    "|": "\\|",
+    "[": "\\[",
+    "]": "\\]",
+    "{": "\\{",
+    "}": "\\}",
+    "\\%": ".*",
+    "\\_": ".",
+  };
+  let chars = parseLikePattern(pat, esc);
+  if (!chars.length) return new RegExp("^$", flags);
+  chars = chars.map((c) => convChars[c] || c);
+  chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
+  const l = chars.length - 1;
+  chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
+  return new RegExp(chars.join(""), flags);
+}
+
+function compare(
+  a: boolean | number | string,
+  b: boolean | number | string,
+): number {
+  if (typeof a === "boolean") a = +a;
+  if (typeof b === "boolean") b = +b;
+  if (typeof a !== typeof b) return typeof a === "string" ? 1 : -1;
+  return a > b ? 1 : a < b ? -1 : 0;
+}
+
+function toNumber(a: boolean | number | string): number {
+  switch (typeof a) {
+    case "number":
+      return a;
+    case "boolean":
+      return +a;
+    case "string":
+      return parseFloat(a) || 0;
+  }
+}
+
+function toString(a: boolean | number | string): string {
+  switch (typeof a) {
+    case "string":
+      return a;
+    case "number":
+      return a.toString();
+    case "boolean":
+      return (+a).toString();
+  }
+}
+
+export function evaluateCallback(exp: Expression): Expression {
+  if (!Array.isArray(exp)) return exp;
+  if (exp[0] === "CASE") {
+    for (let i = 1; i < exp.length; i += 2) {
+      if (Array.isArray(exp[i])) return exp;
+      if (exp[i]) return exp[i + 1];
+    }
+    return null;
+  } else if (exp[0] === "FUNC") {
+    if (exp[1] === "COALESCE") {
+      const args: Expression[] = [];
+      for (let i = 2; i < exp.length; ++i) {
+        const e = exp[i];
+        if (e == null) continue;
+        args.push(e);
+        if (!Array.isArray(e)) break;
+      }
+      if (!args.length) return null;
+      if (args.length === 1) return args[0];
+      return ["FUNC", "COALESCE", ...args];
+    } else if (exp[1] === "UPPER") {
+      if (exp[2] == null) return null;
+      if (!isArray(exp[2])) return toString(exp[2]).toUpperCase();
+    } else if (exp[1] === "LOWER") {
+      if (exp[2] == null) return null;
+      if (!isArray(exp[2])) return toString(exp[2]).toLowerCase();
+    } else if (exp[1] === "ROUND") {
+      const n = exp[2];
+      const p = exp.length > 3 ? exp[3] : 0;
+      if (n == null || p == null) return null;
+      if (!isArray(n) && !isArray(p)) {
+        const d = 10 ** p;
+        const m = n * d * (1 + Number.EPSILON);
+        return Math.round(m) / d;
+      }
+    }
+  } else if (exp[0] === "PARAM") {
+    if (exp[1] == null) return null;
+  } else if (exp[0] === "AND") {
+    for (let i = 1; i < exp.length; ++i)
+      if (!Array.isArray(exp[i]) && exp[i] != null && !exp[i]) return false;
+    const args: Expression[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const ee = exp[i];
+      if (ee == null) return null;
+      if (Array.isArray(ee)) {
+        if (ee[0] === "AND") args.push(...ee.slice(1));
+        else args.push(ee);
+      }
+    }
+    if (!args.length) return true;
+    if (args.length === 1) args.push(true);
+    return ["AND", ...args];
+  } else if (exp[0] === "OR") {
+    const args: Expression[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const ee = exp[i];
+      if (Array.isArray(ee)) {
+        if (ee[0] === "OR") args.push(...ee.slice(1));
+        else args.push(ee);
+      } else if (ee) {
+        return true;
+      }
+    }
+    if (!args.length) return exp.some((ee) => ee == null) ? null : false;
+    if (args.length === 1) args.push(false);
+    return ["OR", ...args];
+  } else if (exp[0] === "NOT") {
+    if (exp[1] == null) return null;
+    if (!isArray(exp[1])) return !exp[1];
+    else if (exp[1][0] === "NOT") return exp[1][1];
+  } else if (exp[0] === "IS NULL") {
+    if (isArray(exp[1])) return exp;
+    else if (exp[1] == null) return true;
+    else return false;
+  } else if (exp[0] === "IS NOT NULL") {
+    if (isArray(exp[1])) return exp;
+    else if (exp[1] != null) return true;
+    else return false;
+  } else if (exp[0] === "LIKE") {
+    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
+    else if (
+      exp[1] == null ||
+      exp[2] == null ||
+      ((exp as any[]).length >= 4 && exp[3] == null)
+    )
+      return null;
+    let r = regExpCache.get(exp as any[]);
+    if (!r) {
+      r = likePatternToRegExp(exp[2], exp[3]);
+      regExpCache.set(exp as any[], r);
+    }
+    return r.test(exp[1]);
+  } else if (exp[0] === "NOT LIKE") {
+    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
+    else if (
+      exp[1] == null ||
+      exp[2] == null ||
+      ((exp as any[]).length >= 4 && exp[3] == null)
+    )
+      return null;
+    let r = regExpCache.get(exp as any[]);
+    if (!r) {
+      r = likePatternToRegExp(exp[2], exp[3]);
+      regExpCache.set(exp as any[], r);
+    }
+    return !r.test(exp[1]);
+  } else if (exp[0] === "=") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) === 0;
+  } else if (exp[0] === "<>") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) !== 0;
+  } else if (exp[0] === ">") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) > 0;
+  } else if (exp[0] === ">=") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) >= 0;
+  } else if (exp[0] === "<") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) < 0;
+  } else if (exp[0] === "<=") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) <= 0;
+  } else if (exp[0] === "*") {
+    return reduce(exp, (a, b) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b)) return toNumber(a) * toNumber(b);
+      return REDUCE_SKIP;
+    });
+  } else if (exp[0] === "/") {
+    return reduce(exp, (a, b, i) => {
+      if (a == null || b == null) return null;
+      if (isArray(a) || isArray(b)) return REDUCE_SKIP;
+      const n1 = toNumber(a);
+      const n2 = toNumber(b);
+      if (i !== 0) return n1 * n2;
+      if (n2 === 0) return null;
+      return n1 / n2;
+    });
+  } else if (exp[0] === "+") {
+    return reduce(exp, (a, b) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b)) return toNumber(a) + toNumber(b);
+      return REDUCE_SKIP;
+    });
+  } else if (exp[0] === "-") {
+    return reduce(exp, (a, b, i) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b))
+        return i === 0 ? toNumber(a) - toNumber(b) : toNumber(a) + toNumber(b);
+      return REDUCE_SKIP;
+    });
+  } else if (exp[0] === "%") {
+    return reduce(exp, (a, b, i) => {
+      if (a == null || b == null) return null;
+      if (isArray(a) || isArray(b) || i !== 0) return REDUCE_SKIP;
+      const n1 = toNumber(a);
+      const n2 = Math.trunc(toNumber(b));
+      if (n2 === 0) return null;
+      return n1 % n2;
+    });
+  } else if (exp[0] === "||") {
+    return reduce(exp, (a, b) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b)) return toString(a) + toString(b);
+      return REDUCE_SKIP;
+    });
+  }
+  return exp;
+}
+
+export function evaluate(
+  exp: Expression,
+  obj: Record<string, unknown> | ((e: string) => Expression),
+  now: number,
+  cb?: (e: Expression) => Expression,
+): string | number | boolean | null;
+export function evaluate(
+  exp: Expression,
+  obj?: Record<string, unknown> | ((e: string) => Expression),
+  now?: number,
+  cb?: (e: Expression) => Expression,
+): Expression;
+export function evaluate(
+  exp: Expression,
+  obj?: Record<string, unknown> | ((e: string) => Expression),
+  now?: number,
+  cb?: (e: Expression) => Expression,
+): Expression {
+  return map(exp, (e) => {
+    if (cb) e = cb(e);
+    if (!isArray(e)) return e;
+
+    if (e[0] === "FUNC" && e[1] === "NOW") {
+      if (now) return now;
+    } else if (e[0] === "PARAM") {
+      if (e[1] == null) return null;
+      if (obj && !isArray(e[1])) {
+        let v;
+        if (typeof obj === "function") v = obj(e[1]);
+        else v = obj[e[1]];
+
+        if (v == null) return null;
+        if (typeof v === "object") v = v.value ? v.value[0] : null;
+        return v;
+      }
+    }
+    return evaluateCallback(e);
+  });
+}
+
+export async function evaluateAsync(
+  exp: Expression,
+  obj: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Promise<Expression>,
+): Promise<string | number | boolean | null>;
+export async function evaluateAsync(
+  exp: Expression,
+  obj?: Record<string, unknown>,
+  now?: number,
+  cb?: (e: Expression) => Promise<Expression>,
+): Promise<Expression>;
+export async function evaluateAsync(
+  exp: Expression,
+  obj?: Record<string, unknown>,
+  now?: number,
+  cb?: (e: Expression) => Promise<Expression>,
+): Promise<Expression> {
+  return mapAsync(exp, async (e) => {
+    if (cb) e = await cb(e);
+    if (!isArray(e)) return e;
+
+    if (e[0] === "FUNC") {
+      if (e[1] === "NOW") if (now) return now;
+    } else if (e[0] === "PARAM") {
+      if (e[1] == null) return null;
+      if (obj && !isArray(e[1])) {
+        let v = obj[e[1]];
+        if (v == null) return null;
+        if (typeof v === "object") v = v["value"] ? v["value"][0] : null;
+        return v as Expression;
+      }
+    }
+    return evaluateCallback(e);
+  });
+}
+
+export function and(exp1: Expression, exp2: Expression): Expression {
+  if (exp1 != null && !exp1) return false;
+  if (exp2 != null && !exp2) return false;
+  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
+    if (exp1 == null || exp2 == null) return null;
+    return true;
+  }
+  if (!Array.isArray(exp2) && exp1[0] === "AND") return exp1;
+  if (!Array.isArray(exp1) && exp2[0] === "AND") return exp2;
+
+  const res: Expression = ["AND"];
+
+  if (Array.isArray(exp1) && exp1[0] === "AND") res.push(...exp1.slice(1));
+  else res.push(exp1);
+
+  if (Array.isArray(exp2) && exp2[0] === "AND") res.push(...exp2.slice(1));
+  else res.push(exp2);
+
+  return res;
+}
+
+export function or(exp1: Expression, exp2: Expression): Expression {
+  if (!Array.isArray(exp1) && exp1) return true;
+  if (!Array.isArray(exp2) && exp2) return true;
+  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
+    if (exp1 == null || exp2 == null) return null;
+    return false;
+  }
+  if (!Array.isArray(exp2) && exp1[0] === "OR") return exp1;
+  if (!Array.isArray(exp1) && exp2[0] === "OR") return exp2;
+
+  const res: Expression = ["OR"];
+
+  if (Array.isArray(exp1) && exp1[0] === "OR") res.push(...exp1.slice(1));
+  else res.push(exp1);
+
+  if (Array.isArray(exp2) && exp2[0] === "OR") res.push(...exp2.slice(1));
+  else res.push(exp2);
+
+  return res;
+}
+
+export function extractParams(exp: Expression): Expression[] {
+  const params = new Set<Expression>();
+  map(exp, (e) => {
+    if (isArray(e) && e[0] === "PARAM") params.add(e[1]);
+    return e;
+  });
+  return Array.from(params);
+}

lib/common/memoize.ts

@@ -0,0 +1,53 @@
+let cache1 = new Map();
+let cache2 = new Map();
+const keys = new WeakMap();
+
+function getKey(obj): string {
+  if (obj === null) return "null";
+  else if (obj === undefined) return "undefined";
+
+  const t = typeof obj;
+  if (t === "number" || t === "boolean" || t === "string") return `${t}:${obj}`;
+  if (t !== "function" && t !== "object")
+    throw new Error(`Cannot memoize ${t} arguments`);
+
+  let k = keys.get(obj);
+  if (!k) {
+    const rnd = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER);
+    k = `${t}:${rnd.toString(36)}`;
+    keys.set(obj, k);
+  }
+  return k;
+}
+
+export default function memoize<T extends (...args: any[]) => any>(func: T): T {
+  const funcKey = getKey(func);
+  return ((...args) => {
+    const key = JSON.stringify(args.map(getKey)) + funcKey;
+
+    if (cache1.has(key)) return cache1.get(key);
+
+    let r;
+    if (cache2.has(key)) {
+      cache1.set(key, (r = cache2.get(key)));
+    } else {
+      cache1.set(key, (r = func(...args)));
+      // Evict rejected promises
+      if (r instanceof Promise) {
+        r.catch(() => {
+          cache1.delete(key);
+          cache2.delete(key);
+        });
+      }
+    }
+    return r;
+  }) as any;
+}
+
+const interval = setInterval(() => {
+  cache2 = cache1;
+  cache1 = new Map();
+}, 120000);
+
+// Don't hold Node.js process
+if (interval.unref) interval.unref();

lib/common/path-set.ts

@@ -0,0 +1,111 @@
+import Path from "./path.ts";
+
+export default class PathSet {
+  declare private lengthIndex: Set<Path>[];
+  declare private fragmentIndex: Map<string, Set<Path>>[];
+  declare private stringIndex: Map<string, Path>;
+
+  public constructor() {
+    this.lengthIndex = [];
+    this.fragmentIndex = [];
+    this.stringIndex = new Map();
+  }
+
+  public get depth(): number {
+    return this.lengthIndex.length;
+  }
+
+  public add(path: Path): Path {
+    if (path.alias) throw new Error("PathSet does not support aliased paths");
+    const p = this.get(path);
+
+    if (p) return p;
+
+    this.stringIndex.set(path.toString(), path);
+
+    while (this.lengthIndex.length <= path.length) {
+      this.lengthIndex.push(new Set());
+      // fragmentIndex is one less than lengthIndex
+      if (this.lengthIndex.length > 1) this.fragmentIndex.push(new Map());
+    }
+
+    const lengthIndex = this.lengthIndex[path.length];
+    lengthIndex.add(path);
+
+    for (let i = 0; i < path.length; ++i) {
+      const fragment = path.segments[i] as string;
+      const fragmentIndex = this.fragmentIndex[i];
+
+      let fragmentIndexSet = fragmentIndex.get(fragment);
+      if (!fragmentIndexSet) {
+        fragmentIndexSet = new Set<Path>();
+        fragmentIndex.set(fragment, fragmentIndexSet);
+      }
+
+      fragmentIndexSet.add(path);
+    }
+
+    return path;
+  }
+
+  public get(path: Path): Path {
+    return this.stringIndex.get(path.toString()) || null;
+  }
+
+  public find(
+    path: Path,
+    superset = false,
+    subset = false,
+    depth: number = path.length,
+  ): Path[] {
+    if (path.alias) throw new Error("PathSet does not support aliased paths");
+
+    const len = path.length;
+
+    if (!superset && depth === len && (!subset || !path.wildcard)) {
+      const p = this.get(path);
+      return p ? [p] : [];
+    }
+
+    const lengthIndex = this.lengthIndex.slice(len, depth + 1);
+    if (!lengthIndex.length) return [];
+
+    let res;
+    for (let i = len - 1; i >= 0; --i) {
+      let fragmentIndexSet2;
+      const fragmentIndex = this.fragmentIndex[i];
+
+      if ((path.wildcard >> i) & 1) {
+        if (subset) continue;
+      } else if (superset) {
+        fragmentIndexSet2 = fragmentIndex.get("*");
+      }
+
+      const fragment = path.segments[i] as string;
+      const fragmentIndexSet1 = fragmentIndex.get(fragment);
+
+      if (!fragmentIndexSet1) {
+        if (!fragmentIndexSet2) return [];
+        if (!res) res = [...fragmentIndexSet2];
+        else res = res.filter((r) => fragmentIndexSet2.has(r));
+      } else if (!fragmentIndexSet2) {
+        if (!res) res = [...fragmentIndexSet1];
+        else res = res.filter((r) => fragmentIndexSet1.has(r));
+      } else {
+        if (!res) {
+          res = [...fragmentIndexSet1, ...fragmentIndexSet2];
+        } else {
+          res = res.filter(
+            (r) => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r),
+          );
+        }
+      }
+      if (!res.length) return res;
+    }
+
+    if (!res) res = [].concat(...lengthIndex.map((a) => [...a]));
+    else res = res.filter((r) => lengthIndex.some((a) => a.has(r)));
+
+    return res;
+  }
+}

lib/common/path.ts

@@ -0,0 +1,274 @@
+type Segments = (string | Alias)[];
+type Alias = [Path, string][];
+
+let cache1 = new Map<string, Path>();
+let cache2 = new Map<string, Path>();
+
+const CHAR_SPACE = 32;
+const CHAR_BACKSLASH = 92;
+const CHAR_COLON = 58;
+const CHAR_QUOTE = 34;
+const CHAR_OPEN_BRACKET = 91;
+const CHAR_CLOSE_BRACKET = 93;
+const CHAR_ASTERISK = 42;
+const CHAR_COMMA = 44;
+const CHAR_DOT = 46;
+
+function charCodeAt(str: string, index: number): number {
+  if (index >= str.length) return 0;
+  return str.charCodeAt(index);
+}
+
+function legalChar(c: number): boolean {
+  return (
+    (c >= 97 && c <= 122) ||
+    (c >= 65 && c <= 90) ||
+    (c >= 48 && c <= 57) ||
+    c === 95 ||
+    c === 45
+  );
+}
+
+export default class Path {
+  declare public readonly segments: Segments;
+  declare public readonly wildcard: number;
+  declare public readonly alias: number;
+  declare protected _string: string;
+  declare protected _stringIndex: number[];
+
+  protected static parseAliasValue(
+    pattern: string,
+    index: number,
+  ): { index: number; value: string } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) === CHAR_QUOTE) {
+      for (let j = i + 1; j < pattern.length; ++j) {
+        if (
+          pattern.charCodeAt(j) === CHAR_QUOTE &&
+          pattern.charCodeAt(j - 1) !== CHAR_BACKSLASH
+        ) {
+          try {
+            ++j;
+            const v = JSON.parse(pattern.slice(i, j));
+            return { index: j, value: v };
+          } catch {
+            return { index, value: null };
+          }
+        }
+      }
+      return { index, value: null };
+    }
+
+    for (; i < pattern.length; ++i) {
+      const c = pattern.charCodeAt(i);
+      if (c === CHAR_CLOSE_BRACKET || c === CHAR_COMMA) break;
+    }
+
+    return { index: i, value: pattern.slice(index, i).trim() };
+  }
+
+  protected static parseAliasPath(
+    pattern: string,
+    index: number,
+  ): { index: number; path: Path } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    const { index: idx, segments } = Path.parsePath(pattern, i);
+    if (idx === i) return { index, path: null };
+    i = idx;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) !== CHAR_COLON) return { index, path: null };
+    return { index: i + 1, path: new Path(segments) };
+  }
+
+  protected static parseAlias(
+    pattern: string,
+    index: number,
+  ): { index: number; alias: Alias } {
+    const alias: Alias = [];
+    let i = index;
+    for (;;) {
+      let path: Path;
+      let value: string;
+      ({ index: i, path } = Path.parseAliasPath(pattern, i));
+      if (!path) break;
+      ({ index: i, value } = Path.parseAliasValue(pattern, i));
+      if (value == null) break;
+      alias.push([path, value]);
+      index = i;
+
+      if (charCodeAt(pattern, index) !== CHAR_COMMA) break;
+      ++i;
+    }
+
+    // Ensure identical expressions have idential string representation
+    alias.sort((a, b) => {
+      if (a[0].toString() > b[0].toString()) return 1;
+      else if (a[0].toString() < b[0].toString()) return -1;
+      else if (a[1] > b[1]) return 1;
+      else if (a[1] < b[1]) return -1;
+      else return 0;
+    });
+
+    Object.freeze(alias);
+    return { index, alias };
+  }
+
+  protected static parsePath(
+    pattern: string,
+    index: number,
+  ): { index: number; segments: Segments } {
+    const segments = [];
+
+    let segStart = index;
+    let wildcard = -1;
+    for (let i = index; i <= pattern.length; ++i) {
+      const c = charCodeAt(pattern, i);
+      if (c === CHAR_OPEN_BRACKET && i === segStart) {
+        const { index: idx, alias } = Path.parseAlias(pattern, i + 1);
+        if (charCodeAt(pattern, idx) !== CHAR_CLOSE_BRACKET) break;
+        segments.push(alias);
+        index = i = idx + 1;
+        segStart = i + 1;
+      } else if (c === CHAR_ASTERISK && i === segStart) {
+        wildcard = i + 1;
+      } else if (!legalChar(c) || i === wildcard) {
+        if (i === segStart) break;
+        const s = pattern.slice(segStart, i);
+        segments.push(s);
+        index = i;
+        if (c !== CHAR_DOT) break;
+        segStart = i + 1;
+      }
+    }
+
+    Object.freeze(segments);
+    return { index, segments };
+  }
+
+  protected constructor(segments: Segments) {
+    let alias = 0;
+    let wildcard = 0;
+    const arr = segments.map((s, i) => {
+      if (Array.isArray(s)) {
+        alias |= 1 << i;
+        const parts = s.map(
+          (al) => `${al[0].toString()}:${JSON.stringify(al[1])}`,
+        );
+        return `[${parts.join(",")}]`;
+      } else if (s === "*") {
+        wildcard |= 1 << i;
+      }
+      return s;
+    });
+
+    let offset = 0;
+    const stringIndex = arr.map((s, i) => (offset += s.length) + i);
+
+    this.segments = segments;
+    this.wildcard = wildcard;
+    this.alias = alias;
+    this._string = arr.join(".");
+    this._stringIndex = stringIndex;
+  }
+
+  public static parse(str: string): Path {
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const { index, segments } = Path.parsePath(str, 0);
+        if (index < str.length) throw new Error("Invalid parameter path");
+        path = new Path(segments);
+        if (path.toString() !== str) cache1.set(path.toString(), path);
+      }
+      cache1.set(str, path);
+    }
+    return path;
+  }
+
+  public get length(): number {
+    return this.segments.length;
+  }
+
+  public toString(): string {
+    return this._string;
+  }
+
+  public slice(start = 0, end: number = this.segments.length): Path {
+    if (start < 0) start = Math.max(0, this.segments.length + start);
+    if (end < 0) end = Math.max(0, this.segments.length + end);
+
+    let str;
+    if (start >= end) {
+      str = "";
+    } else {
+      const i1 = start > 0 ? this._stringIndex[start - 1] + 1 : 0;
+      const i2 =
+        end <= this.segments.length
+          ? this._stringIndex[end - 1]
+          : this._string.length;
+      str = this._string.slice(i1, i2);
+    }
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const segments = this.segments.slice(start, end);
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+
+  public concat(path2: Path): Path {
+    if (!path2._string) return this;
+    else if (!this._string) return path2;
+
+    const str = `${this._string}.${path2._string}`;
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const segments = this.segments.concat(path2.segments);
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+
+  public stripAlias(): Path {
+    if (!this.alias) return this;
+    const segments = this.segments.map((s) => (Array.isArray(s) ? "*" : s));
+    const str = segments.join(".");
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+}
+
+const interval = setInterval(() => {
+  cache2 = cache1;
+  cache1 = new Map();
+}, 120000);
+
+// Don't hold Node.js process
+if (interval.unref) interval.unref();

lib/common/yaml.ts

@@ -0,0 +1,218 @@
+const LINE_WIDTH = 80;
+const INDENTATION = "  ";
+
+const STRING_RESERVED = new Set([
+  "true",
+  "True",
+  "TRUE",
+  "false",
+  "False",
+  "FALSE",
+  "null",
+  "Null",
+  "NULL",
+]);
+
+function isPrintable(str: string): boolean {
+  return !/[^\t\n\x20-\x7e\x85\u{a0}-\u{d7ff}\u{e000}-\u{fffd}\u{10000}-\u{10ffff}]/u.test(
+    str,
+  );
+}
+
+function stringifyKey(str: string): string {
+  if (!str || !isPrintable(str)) return JSON.stringify(str);
+  if (/^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|[\n,[\]{}]|\s$/.test(str))
+    return JSON.stringify(str);
+  return str;
+}
+
+function foldString(str: string): string[] {
+  if (str.length <= LINE_WIDTH) return [str];
+  if (str.startsWith(" ")) return [str];
+  const lines: string[] = [];
+
+  let idx = 0;
+  let cand = 0;
+  for (let i = 1; i < str.length - 1; ++i) {
+    if (str[i] !== " ") continue;
+
+    if (str[i + 1] === " ") {
+      i += 2;
+      while (str[i] === " ") ++i;
+      continue;
+    }
+
+    if (i <= idx + LINE_WIDTH) {
+      cand = i;
+      continue;
+    }
+
+    const c = cand > idx ? cand : i;
+    lines.push(str.slice(idx, c));
+    idx = c + 1;
+    cand = i;
+  }
+
+  if (cand > idx && str.length > idx + LINE_WIDTH) {
+    lines.push(str.slice(idx, cand));
+    idx = cand + 1;
+  }
+
+  lines.push(str.slice(idx));
+
+  return lines;
+}
+
+function stringifyString(str: string, res: string[], prefix1, prefix2): void {
+  if (/^\s*$/.test(str) || STRING_RESERVED.has(str) || !isPrintable(str)) {
+    res.push(prefix1 + JSON.stringify(str));
+    return;
+  }
+
+  if (!prefix2) prefix2 = INDENTATION;
+
+  const lines = str.split("\n");
+  if (lines.length > 1) {
+    let idt = "";
+    let chmp = "-";
+    if ((lines.find((l) => l) || "").startsWith(" "))
+      idt = `${INDENTATION.length}`;
+
+    if (!lines[lines.length - 1]) {
+      lines.pop();
+      if (lines[lines.length - 1]) chmp = "";
+      else chmp = "+";
+    }
+
+    if (/^\s+$/.test(lines[lines.length - 1])) {
+      res.push(prefix1 + JSON.stringify(str));
+      return;
+    }
+
+    let isFolded = false;
+    const folded = lines.map((l) => {
+      const ls = foldString(l);
+      if (ls.length > 1) isFolded = true;
+      return ls;
+    });
+
+    if (!isFolded) {
+      res.push(
+        `${prefix1}|${idt}${chmp}`,
+        ...lines.map((l) => (l ? prefix2 + l : l)),
+      );
+      return;
+    }
+
+    res.push(`${prefix1}>${idt}${chmp}`);
+    res.push(...folded[0].map((f) => prefix2 + f));
+    for (let i = 1; i < folded.length; ++i) {
+      const prevLine = folded[i - 1][0];
+      if (prevLine && !folded[i - 1][0].startsWith(" ")) res.push("");
+      res.push(...folded[i].map((f) => prefix2 + f));
+    }
+    return;
+  }
+
+  if (
+    /^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|\s$/.test(str) ||
+    parseFloat(str) === +str
+  ) {
+    res.push(prefix1 + JSON.stringify(str));
+    return;
+  }
+
+  res.push(prefix1 + str);
+}
+
+function stringifyAny(
+  obj: unknown,
+  res: string[],
+  prefix1 = "",
+  prefix2 = "",
+): void {
+  if (obj == null) {
+    res.push(`${prefix1}null`);
+    return;
+  }
+  if (typeof obj === "number" || typeof obj === "boolean") {
+    res.push(`${prefix1}${JSON.stringify(obj)}`);
+    return;
+  }
+  if (obj instanceof Date) {
+    res.push(`${prefix1}${obj.toJSON()}`);
+    return;
+  }
+
+  if (typeof obj === "string") {
+    stringifyString(obj, res, prefix1, prefix2);
+    return;
+  }
+
+  if (Array.isArray(obj)) {
+    if (!obj.length) {
+      res.push(prefix1 + "[]");
+      return;
+    }
+
+    if (!prefix1 || prefix1.endsWith("- ")) {
+      stringifyAny(obj[0], res, prefix1 + "- ", prefix2 + INDENTATION);
+      prefix1 = prefix2 + "- ";
+      prefix2 = prefix2 + INDENTATION;
+      for (let i = 1; i < obj.length; ++i)
+        stringifyAny(obj[i], res, prefix1, prefix2);
+    } else {
+      res.push(prefix1);
+      prefix1 = prefix2 + "- ";
+      prefix2 = prefix2 + INDENTATION;
+      for (let i = 0; i < obj.length; ++i)
+        stringifyAny(obj[i], res, prefix1, prefix2);
+    }
+    return;
+  }
+
+  const entries = Object.entries(obj).filter((e) => e[1] !== undefined);
+
+  if (!entries.length) {
+    res.push(prefix1 + "{}");
+    return;
+  }
+
+  if (!prefix1 || prefix1.endsWith("- ")) {
+    stringifyAny(
+      entries[0][1],
+      res,
+      prefix1 + `${stringifyKey(entries[0][0])}: `,
+      prefix2 + INDENTATION,
+    );
+    prefix1 = prefix2;
+    prefix2 = prefix2 + INDENTATION;
+    for (let i = 1; i < entries.length; ++i) {
+      stringifyAny(
+        entries[i][1],
+        res,
+        prefix1 + `${stringifyKey(entries[i][0])}: `,
+        prefix2,
+      );
+    }
+  } else {
+    res.push(prefix1);
+    prefix1 = prefix2;
+    prefix2 = prefix2 + INDENTATION;
+    for (let i = 0; i < entries.length; ++i) {
+      stringifyAny(
+        entries[i][1],
+        res,
+        prefix1 + `${stringifyKey(entries[i][0])}: `,
+        prefix2,
+      );
+    }
+  }
+}
+
+export function stringify(obj: unknown): string {
+  if (obj === undefined) return undefined;
+  const lines: string[] = [];
+  stringifyAny(obj, lines);
+  return lines.join("\n") + "\n";
+}

lib/config.coffee

@@ -1,208 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-path = require 'path'
-common = require './common'
-
-options = {
-  CONFIG_DIR : {type : 'path', default : 'config'},
-  MONGODB_CONNECTION_URL : {type : 'string', default : 'mongodb://127.0.0.1/genieacs'},
-  REDIS_PORT : {type : 'int', default : 6379},
-  REDIS_HOST : {type : 'string', default : ''},
-  REDIS_DB : {type : 'int', default : 0},
-
-  CWMP_WORKER_PROCESSES : {type : 'int', default : 0},
-  CWMP_PORT : {type : 'int', default : 7547},
-  CWMP_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  CWMP_SSL : {type : 'bool', default : false},
-  CWMP_LOG_FILE : {type: 'path', default : ''},
-  CWMP_ACCESS_LOG_FILE : {type : 'path', default : ''},
-
-  NBI_WORKER_PROCESSES : {type : 'int', default : 0},
-  NBI_PORT : {type : 'int', default : 7557},
-  NBI_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  NBI_SSL : {type : 'bool', default : false},
-  NBI_LOG_FILE : {type: 'path', default : ''},
-  NBI_ACCESS_LOG_FILE : {type : 'path', default : ''},
-
-  FS_WORKER_PROCESSES : {type : 'int', default : 0},
-  FS_PORT : {type : 'int', default : 7567},
-  FS_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  FS_SSL : {type : 'bool', default : false},
-  FS_HOSTNAME : {type : 'string', default : 'acs.example.com'},
-  FS_LOG_FILE : {type: 'path', default : ''},
-  FS_ACCESS_LOG_FILE : {type : 'path', default : ''},
-
-  UDP_CONNECTION_REQUEST_PORT : {type : 'int', default : 0},
-
-  DOWNLOAD_TIMEOUT: {type : 'int', default : 3600},
-  EXT_TIMEOUT: {type: 'int', default: 3000},
-  MAX_CACHE_TTL : {type : 'int', default : 86400},
-  DEBUG : {type : 'bool', default : false},
-  RETRY_DELAY : {type : 'int', default : 300},
-  SESSION_TIMEOUT : {type : 'int', default : 30},
-  CONNECTION_REQUEST_TIMEOUT : {type : 'int', default: 2000},
-  GPN_NEXT_LEVEL : {type : 'int', default : 0},
-  GPV_BATCH_SIZE : {type : 'int', default : 32},
-  MAX_DEPTH : {type: 'int', default : 16},
-  COOKIES_PATH : {type : 'string'},
-  LOG_FORMAT : {type : 'string', default : 'simple'},
-  ACCESS_LOG_FORMAT : {type : 'string', default : ''},
-  MAX_CONCURRENT_REQUESTS : {type : 'int', default: 20},
-  DATETIME_MILLISECONDS : {type: 'bool', default: true},
-  BOOLEAN_LITERAL : {type: 'bool', default: true},
-  CONNECTION_REQUEST_ALLOW_BASIC_AUTH: {type: 'bool', default: false}
-
-  # XML configuration
-  XML_RECOVER : {type : 'bool', default : false},
-  XML_IGNORE_ENC : {type : 'bool', default : false},
-  XML_FORMAT : {type : 'bool', default : false},
-  XML_NO_DECL : {type : 'bool', default : false},
-  XML_NO_EMPTY : {type : 'bool', default : false},
-  XML_IGNORE_NAMESPACE : {type : 'bool', default : false},
-
-  # Should probably never be changed
-  DEVICE_ONLINE_THRESHOLD : {type : 'int', default : 4000}
-}
-
-allConfig = {}
-
-
-setConfig = (name, value, commandLineArgument) ->
-  return true if allConfig[name]?
-
-  # For compatibility with v1.0
-  if name in ['PRESETS_CACHE_DURATION', 'presets-cache-duration']
-    setConfig('MAX_CACHE_TTL', value)
-
-  if name in ['GET_PARAMETER_NAMES_DEPTH_THRESHOLD', 'get-parameter-names-depth-threshold']
-    setConfig('GPN_NEXT_LEVEL', value)
-
-  if name in ['TASK_PARAMETERS_BATCH_SIZE', 'task-parameters-batch-size']
-    setConfig('GPV_BATCH_SIZE', value)
-
-  if name in ['IGNORE_XML_NAMESPACES', 'ignore-xml-namespaces']
-    setConfig('XML_IGNORE_NAMESPACE', value)
-
-  if name in ['XML_PARSE_IGNORE_ENC', 'xml-parse-ignore-enc']
-    setConfig('XML_IGNORE_ENC', value)
-
-  if name in ['XML_PARSE_RECOVER', 'xml-parse-recover']
-    setConfig('XML_RECOVER', value)
-
-  if name in ['FS_IP', 'fs-ip']
-    setConfig('FS_HOSTNAME', value)
-
-  cast = (val, type) ->
-    switch type
-      when 'int'
-        Number(val)
-      when 'bool'
-        String(val).trim().toLowerCase() in ['true', 'on', 'yes', '1']
-      when 'string'
-        String(val)
-      when 'path'
-        if val then path.resolve(val) else ''
-      else
-        null
-
-  _value = null
-
-  for optionName, optionDetails of options
-    n = optionName
-    if commandLineArgument
-      n = n.toLowerCase().replace(/_/g, '-')
-
-    if name == n
-      _value = cast(value, optionDetails.type)
-      n = optionName
-    else if name.startsWith("#{n}-")
-      _value = cast(value, optionDetails.type)
-      n = "#{optionName}-#{name[optionName.length+1..]}"
-
-    if _value?
-      allConfig[n] = _value
-      # Save as environmnet variable to pass on to any child processes
-      process.env["GENIEACS_#{n}"] = _value
-      return true
-
-  return false
-
-
-# Command line arguments
-argv = process.argv[2..]
-while argv.length
-  arg = argv.shift()
-  if arg[0] == '-'
-    v = argv.shift()
-    setConfig(arg[2..], v, true)
-
-
-# Environment variable
-for k, v of process.env
-  continue if k.lastIndexOf('GENIEACS_', 0) != 0
-  k = k[9..] # remove "GENIEACS_" prefix
-  setConfig(k, v)
-
-
-# Use default config dir if none defined
-setConfig('CONFIG_DIR', options['CONFIG_DIR'].default)
-
-# Configuration file
-for k, v of require(path.resolve(allConfig.CONFIG_DIR, 'config'))
-  if not setConfig(k, v)
-    # Pass as environment variable to be accessable by extensions
-    process.env["GENIEACS_#{k}"] = v
-
-
-# Defaults
-for k, v of options
-  setConfig(k, v.default) if v.default?
-
-
-get = (option, deviceId) ->
-  if deviceId?
-    name = "#{option}-#{deviceId}"
-    v = allConfig[name]
-    return v if v?
-    i = name.lastIndexOf('-')
-    v = allConfig[name[0...i]]
-    return v if v?
-    i = name.lastIndexOf('-', i-1)
-    v = allConfig[name[0...i]]
-    return v if v? or i == -1
-
-  return allConfig[option]
-
-
-getDefault = (optionName) ->
-  option = options[optionName]
-  return null if not option
-  val = option.default
-  if val and option.type == 'path'
-    val = path.resolve(val)
-  return val
-
-
-# load authentication scripts
-try
-  exports.auth = require(path.resolve(allConfig.CONFIG_DIR, 'auth'))
-
-exports.get = get
-exports.getDefault = getDefault

lib/config.ts

@@ -0,0 +1,279 @@
+import { resolve } from "node:path";
+import { readFileSync, existsSync } from "node:fs";
+
+// Find project root directory
+export let ROOT_DIR = resolve(__dirname, "..");
+while (!existsSync(`${ROOT_DIR}/package.json`)) {
+  const d = resolve(ROOT_DIR, "..");
+  if (d === ROOT_DIR) {
+    ROOT_DIR = process.cwd();
+    break;
+  }
+  ROOT_DIR = d;
+}
+
+// For compatibility with v1.1
+let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl, fsHostname;
+
+const options = {
+  EXT_DIR: { type: "path", default: resolve(ROOT_DIR, "config/ext") },
+  MONGODB_CONNECTION_URL: {
+    type: "string",
+    default: "mongodb://127.0.0.1/genieacs",
+  },
+
+  CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
+  CWMP_PORT: { type: "int", default: 7547 },
+  CWMP_INTERFACE: { type: "string", default: "::" },
+  CWMP_SSL_CERT: { type: "string", default: "" },
+  CWMP_SSL_KEY: { type: "string", default: "" },
+  CWMP_LOG_FILE: { type: "path", default: "" },
+  CWMP_ACCESS_LOG_FILE: { type: "path", default: "" },
+
+  NBI_WORKER_PROCESSES: { type: "int", default: 0 },
+  NBI_PORT: { type: "int", default: 7557 },
+  NBI_INTERFACE: { type: "string", default: "::" },
+  NBI_SSL_CERT: { type: "string", default: "" },
+  NBI_SSL_KEY: { type: "string", default: "" },
+  NBI_LOG_FILE: { type: "path", default: "" },
+  NBI_ACCESS_LOG_FILE: { type: "path", default: "" },
+
+  FS_WORKER_PROCESSES: { type: "int", default: 0 },
+  FS_PORT: { type: "int", default: 7567 },
+  FS_INTERFACE: { type: "string", default: "::" },
+  FS_SSL_CERT: { type: "string", default: "" },
+  FS_SSL_KEY: { type: "string", default: "" },
+  FS_URL_PREFIX: { type: "string", default: "" },
+  FS_LOG_FILE: { type: "path", default: "" },
+  FS_ACCESS_LOG_FILE: { type: "path", default: "" },
+
+  UI_WORKER_PROCESSES: { type: "int", default: 0 },
+  UI_PORT: { type: "int", default: 3000 },
+  UI_INTERFACE: { type: "string", default: "::" },
+  UI_SSL_CERT: { type: "string", default: "" },
+  UI_SSL_KEY: { type: "string", default: "" },
+  UI_LOG_FILE: { type: "path", default: "" },
+  UI_ACCESS_LOG_FILE: { type: "path", default: "" },
+  UI_JWT_SECRET: { type: "string", default: "" },
+
+  UDP_CONNECTION_REQUEST_PORT: { type: "int", default: 0 },
+  FORWARDED_HEADER: { type: "string", default: "" },
+
+  DOWNLOAD_TIMEOUT: { type: "int", default: 3600 },
+  EXT_TIMEOUT: { type: "int", default: 3000 },
+  MAX_CACHE_TTL: { type: "int", default: 86400 },
+  DEBUG_FILE: { type: "path", default: "" },
+  DEBUG_FORMAT: { type: "string", default: "yaml" },
+  DEBUG: { type: "bool", default: false },
+  RETRY_DELAY: { type: "int", default: 300 },
+  SESSION_TIMEOUT: { type: "int", default: 30 },
+  CONNECTION_REQUEST_TIMEOUT: { type: "int", default: 2000 },
+  GPN_NEXT_LEVEL: { type: "int", default: 0 },
+  GPV_BATCH_SIZE: { type: "int", default: 32 },
+  MAX_DEPTH: { type: "int", default: 16 },
+  COOKIES_PATH: { type: "string" },
+  LOG_FORMAT: { type: "string", default: "simple" },
+  ACCESS_LOG_FORMAT: { type: "string", default: "" },
+  MAX_CONCURRENT_REQUESTS: { type: "int", default: 20 },
+  DATETIME_MILLISECONDS: { type: "bool", default: true },
+  BOOLEAN_LITERAL: { type: "bool", default: true },
+  CONNECTION_REQUEST_ALLOW_BASIC_AUTH: { type: "bool", default: false },
+  MAX_COMMIT_ITERATIONS: { type: "int", default: 32 },
+
+  // Should probably never be changed
+  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
+
+  XMPP_JID: { type: "string", default: "" },
+  XMPP_PASSWORD: { type: "string", default: "" },
+};
+
+const allConfig: { [name: string]: string | number } = {};
+
+function setConfig(name, value, commandLineArgument = false): boolean {
+  if (allConfig[name] != null) return true;
+
+  // For compatibility with v1.1
+  if (name === "CONFIG_DIR" || name === "config-dir")
+    configDir = configDir || resolve(ROOT_DIR, value);
+
+  if (name === "CWMP_SSL" || name === "cwmp-ssl")
+    cwmpSsl = cwmpSsl || String(value).toLowerCase().trim();
+
+  if (name === "NBI_SSL" || name === "nbi-ssl")
+    nbiSsl = nbiSsl || String(value).toLowerCase().trim();
+
+  if (name === "FS_SSL" || name === "fs-ssl")
+    fsSsl = fsSsl || String(value).toLowerCase().trim();
+
+  if (name === "UI_SSL" || name === "ui-ssl")
+    uiSsl = uiSsl || String(value).toLowerCase().trim();
+
+  if (name === "FS_HOSTNAME" || name === "fs-hostname")
+    fsHostname = fsHostname || String(value).trim();
+
+  // For compatibility with v1.0
+  if (name === "PRESETS_CACHE_DURATION" || name === "presets-cache-duration")
+    setConfig("MAX_CACHE_TTL", value);
+
+  if (
+    name === "GET_PARAMETER_NAMES_DEPTH_THRESHOLD" ||
+    name === "get-parameter-names-depth-threshold"
+  )
+    setConfig("GPN_NEXT_LEVEL", value);
+
+  if (
+    name === "TASK_PARAMETERS_BATCH_SIZE" ||
+    name === "task-parameters-batch-size"
+  )
+    setConfig("GPV_BATCH_SIZE", value);
+
+  if (name === "FS_IP" || name === "fs-ip") setConfig("FS_HOSTNAME", value);
+
+  function cast(val, type): string | number | boolean {
+    switch (type) {
+      case "int":
+        return Number(val);
+      case "bool":
+        return ["true", "1"].includes(String(val).trim().toLowerCase());
+      case "string":
+        return String(val);
+      case "path":
+        if (!val) return "";
+        return resolve(val);
+      default:
+        return null;
+    }
+  }
+
+  let _value = null;
+  for (const [optionName, optionDetails] of Object.entries(options)) {
+    let n = optionName;
+    if (commandLineArgument) n = n.toLowerCase().replace(/_/g, "-");
+
+    if (name === n) {
+      _value = cast(value, optionDetails.type);
+      n = optionName;
+    } else if (name.startsWith(`${n}-`)) {
+      _value = cast(value, optionDetails.type);
+      n = `${optionName}-${name.slice(optionName.length + 1)}`;
+    }
+
+    if (_value != null) {
+      allConfig[n] = _value;
+      // Save as environmnet variable to pass on to any child process
+      process.env[`GENIEACS_${n}`] = _value;
+      return true;
+    }
+  }
+
+  return false;
+}
+
+// Command line arguments
+const argv = process.argv.slice(2);
+while (argv.length) {
+  const arg = argv.shift();
+  if (arg[0] === "-") {
+    const v = argv.shift();
+    setConfig(arg.slice(2), v, true);
+  }
+}
+
+// Environment variable
+for (const [k, v] of Object.entries(process.env))
+  if (k.startsWith("GENIEACS_")) setConfig(k.slice(9), v);
+
+// Configuration file
+const configFilename = configDir
+  ? `${configDir}/config.json`
+  : `${ROOT_DIR}/config/config.json`;
+
+if (existsSync(configFilename)) {
+  const configFile = JSON.parse(readFileSync(configFilename).toString());
+
+  for (const [k, v] of Object.entries(configFile)) {
+    if (!setConfig(k, v))
+      // Pass as environment variable to be accessable by extensions
+      process.env[`GENIEACS_${k}`] = `${v}`;
+  }
+}
+
+if (configDir) setConfig("EXT_DIR", `${configDir}/ext`);
+
+if (["true", "1"].includes(cwmpSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("CWMP_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("CWMP_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(nbiSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("NBI_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("NBI_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(fsSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("FS_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("FS_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(uiSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("UI_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("UI_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (fsHostname) {
+  const FS_PORT = allConfig["FS_PORT"] || 7567;
+  const FS_SSL = !!allConfig["FS_SSL_CERT"];
+  setConfig(
+    "FS_URL_PREFIX",
+    (FS_SSL ? "https" : "http") + `://${fsHostname}:${FS_PORT}/`,
+  );
+}
+
+// Defaults
+for (const [k, v] of Object.entries(options))
+  if (v["default"] != null) setConfig(k, v["default"]);
+
+export function get(
+  optionName: string,
+  deviceId?: string,
+): string | number | boolean {
+  if (!deviceId) return allConfig[optionName];
+
+  optionName = `${optionName}-${deviceId}`;
+  let v = allConfig[optionName];
+  if (v != null) return v;
+
+  let i = optionName.lastIndexOf("-");
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  if (i > 0) {
+    v = allConfig[optionName.slice(0, i)];
+    if (v != null) return v;
+  }
+
+  return null;
+}
+
+export function getDefault(optionName: string): string | number | boolean {
+  const option = options[optionName];
+  if (!option) return null;
+
+  let val = option["default"];
+  if (val && option.type === "path") val = resolve(val);
+
+  return val;
+}

lib/connection-request.ts

@@ -0,0 +1,304 @@
+import * as crypto from "node:crypto";
+import * as dgram from "node:dgram";
+import * as http from "node:http";
+import { evaluateAsync } from "./common/expression/util.ts";
+import { Expression } from "./types.ts";
+import * as auth from "./auth.ts";
+import * as extensions from "./extensions.ts";
+import * as debug from "./debug.ts";
+import XmppClient from "./xmpp-client.ts";
+import * as config from "../lib/config.ts";
+import { encodeEntities, parseAttrs, Element } from "./xml-parser.ts";
+import * as logger from "../lib/logger.ts";
+
+async function extractAuth(
+  exp: Expression,
+  dflt: any,
+): Promise<[string, string, Expression]> {
+  let username, password;
+  const _exp = await evaluateAsync(
+    exp,
+    {},
+    0,
+    async (e: Expression): Promise<Expression> => {
+      if (!username && Array.isArray(e) && e[0] === "FUNC") {
+        if (e[1] === "EXT") {
+          if (typeof e[2] !== "string" || typeof e[3] !== "string") return null;
+
+          for (let i = 4; i < e.length; i++)
+            if (Array.isArray(e[i])) return null;
+
+          const { fault, value } = await extensions.run(e.slice(2));
+          return fault ? null : value;
+        } else if (e[1] === "AUTH") {
+          if (!Array.isArray(e[2]) && !Array.isArray(e[3])) {
+            username = e[2] || "";
+            password = e[3] || "";
+          }
+          return dflt;
+        }
+      }
+      return e;
+    },
+  );
+  return [username, password, _exp];
+}
+
+function httpGet(
+  url: URL,
+  options: http.RequestOptions,
+  _debug: boolean,
+  deviceId: string,
+): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
+  return new Promise((resolve, reject) => {
+    const req = http
+      .get(url, options, (res) => {
+        res.resume();
+        resolve({ statusCode: res.statusCode, headers: res.headers });
+        if (_debug) {
+          debug.outgoingHttpRequest(req, deviceId, "GET", url, null);
+          debug.incomingHttpResponse(res, deviceId, null);
+        }
+      })
+      .on("error", (err) => {
+        req.destroy();
+        reject(err);
+        if (_debug)
+          debug.outgoingHttpRequestError(req, deviceId, "GET", url, err);
+      })
+      .on("timeout", () => {
+        req.destroy();
+      });
+  });
+}
+
+export async function httpConnectionRequest(
+  address: string,
+  authExp: Expression,
+  allowBasicAuth: boolean,
+  timeout: number,
+  _debug: boolean,
+  deviceId: string,
+): Promise<string> {
+  const url = new URL(address);
+  if (url.protocol !== "http:")
+    return "Invalid connection request URL or protocol";
+
+  const options: http.RequestOptions = {
+    agent: new http.Agent({ maxSockets: 1, keepAlive: true, timeout: timeout }),
+  };
+
+  let authHeader: Record<string, string>;
+  let username: string;
+  let password: string;
+
+  while (!authHeader || (username != null && password != null)) {
+    let opts = options;
+    if (authHeader) {
+      if (authHeader["method"] === "Basic") {
+        if (!allowBasicAuth) return "Basic HTTP authentication not allowed";
+
+        opts = Object.assign(
+          {
+            headers: {
+              Authorization: auth.basic(username || "", password || ""),
+            },
+          },
+          options,
+        );
+      } else if (authHeader["method"] === "Digest") {
+        opts = Object.assign(
+          {
+            headers: {
+              Authorization: auth.solveDigest(
+                username,
+                password,
+                url.pathname + url.search,
+                "GET",
+                null,
+                authHeader,
+              ),
+            },
+          },
+          options,
+        );
+      } else {
+        return "Unrecognized auth method";
+      }
+    }
+
+    let res: { statusCode: number; headers: http.IncomingHttpHeaders };
+    try {
+      res = await httpGet(url, opts, _debug, deviceId);
+    } catch (err) {
+      // Workaround for some devices unexpectedly closing the connection
+      if (authHeader) {
+        try {
+          res = await httpGet(url, opts, _debug, deviceId);
+        } catch (err) {
+          return `Connection request error: ${err.message}`;
+        }
+      }
+
+      if (err["code"] === "ECONNRESET" || err["code"] === "ECONNREFUSED")
+        return "Device is offline";
+
+      return `Connection request error: ${err.message}`;
+    }
+
+    if (res.statusCode === 200 || res.statusCode === 204) return "";
+
+    // When a Connection Request is received for the Virtual CWMP Device and the Proxied
+    // Device is offline the CPE Proxier MUST respond with an HTTP 503 failure
+    if (res.statusCode === 503) return "Device is offline";
+
+    if (res.statusCode === 401 && res.headers["www-authenticate"]) {
+      try {
+        authHeader = auth.parseWwwAuthenticateHeader(
+          res.headers["www-authenticate"],
+        );
+      } catch (err) {
+        return "Connection request error: Error parsing www-authenticate header";
+      }
+      [username, password, authExp] = await extractAuth(authExp, false);
+    } else {
+      return `Connection request error: Unexpected status code ${res.statusCode}`;
+    }
+  }
+
+  return "Connection request error: Incorrect connection request credentials";
+}
+
+export async function udpConnectionRequest(
+  host: string,
+  port: number,
+  authExp: Expression,
+  sourcePort = 0,
+  _debug: boolean,
+  deviceId: string,
+): Promise<void> {
+  const now = Date.now();
+
+  const client = dgram.createSocket({ type: "udp4", reuseAddr: true });
+  // When a device is NAT'ed, the UDP Connection Request must originate from
+  // the same address and port used by the STUN server, in order to traverse
+  // the firewall. This does require that the Genieacs NBI and STUN server
+  // are allowed to bind to the same address and port. The STUN server needs
+  // to open its UDP port with the SO_REUSEADDR option, allowing the NBI to
+  // also bind to the same port.
+  if (sourcePort) client.bind({ port: sourcePort, exclusive: true });
+
+  let username: string;
+  let password: string;
+
+  [username, password, authExp] = await extractAuth(authExp, null);
+
+  if (username == null) username = "";
+  if (password == null) password = "";
+  while (username != null && password != null) {
+    const ts = Math.trunc(now / 1000);
+    const id = Math.trunc(Math.random() * 4294967295);
+    const cn = crypto.randomBytes(8).toString("hex");
+    const sig = crypto
+      .createHmac("sha1", password)
+      .update(`${ts}${id}${username}${cn}`)
+      .digest("hex");
+    const uri = `http://${host}:${port}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
+    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${host}:${port}\r\n\r\n`;
+    const message = Buffer.from(msg);
+
+    for (let i = 0; i < 3; ++i) {
+      await new Promise<void>((resolve, reject) => {
+        client.send(message, 0, message.length, port, host, (err: Error) => {
+          if (err) reject(err);
+          else resolve();
+          if (_debug) debug.outgoingUdpMessage(host, deviceId, port, msg);
+        });
+      });
+    }
+
+    [username, password, authExp] = await extractAuth(authExp, null);
+  }
+  client.close();
+}
+
+const XMPP_JID = config.get("XMPP_JID") as string;
+const XMPP_PASSWORD = config.get("XMPP_PASSWORD") as string;
+const XMPP_RESOURCE = crypto.randomBytes(8).toString("hex");
+
+let xmppClient: XmppClient;
+
+function xmppClientOnError(err: Error): void {
+  xmppClient = null;
+  logger.error({
+    message: "XMPP exception",
+    exception: err,
+    pid: process.pid,
+  });
+}
+
+function xmppClientOnClose(): void {
+  xmppClient = null;
+}
+
+export async function xmppConnectionRequest(
+  jid: string,
+  authExp: Expression,
+  timeout: number,
+  _debug: boolean,
+  deviceId: string,
+): Promise<string> {
+  if (!xmppClient) {
+    const [host, username] = XMPP_JID.split("@").reverse();
+    xmppClient = await XmppClient.connect({
+      host,
+      username,
+      resource: XMPP_RESOURCE,
+      password: XMPP_PASSWORD,
+      timeout: 120000,
+    });
+    xmppClient.on("error", xmppClientOnError);
+    xmppClient.on("close", xmppClientOnClose);
+    xmppClient.unref();
+  }
+
+  let username: string;
+  let password: string;
+
+  [username, password, authExp] = await extractAuth(authExp, null);
+  while (username != null && password != null) {
+    const msg = `<connectionRequest xmlns="urn:broadband-forum-org:cwmp:xmppConnReq-1-0"><username>${encodeEntities(
+      username,
+    )}</username><password>${encodeEntities(
+      password,
+    )}</password></connectionRequest>`;
+    let res: Element, rawRes: string, rawReq: string;
+    try {
+      ({ res, rawRes, rawReq } = await xmppClient.sendIqStanza(
+        XMPP_JID,
+        jid,
+        "get",
+        msg,
+        timeout,
+      ));
+    } catch (err) {
+      return err.message;
+    }
+    if (_debug) {
+      debug.outgoingXmppStanza(deviceId, rawReq);
+      debug.incomingXmppStanza(deviceId, rawRes);
+    }
+    const attrs = parseAttrs(res.attrs);
+    const type = attrs.find((a) => a.name === "type");
+    if (type && type.value === "result") return "";
+    const error = res.children.find((c) => c.name === "error");
+    if (!error || !error.children[0])
+      return "Unexpected XMPP connection request response";
+    if (error.children[0].name === "service-unavailable")
+      return "Device is offline";
+    if (error.children[0].name !== "not-authorized")
+      return "Unexpected XMPP connection request response";
+    [username, password, authExp] = await extractAuth(authExp, null);
+  }
+  return "Incorrect connection request credentials";
+}

lib/cwmp.coffee

@@ -1,961 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-util = require 'util'
-zlib = require 'zlib'
-crypto = require 'crypto'
-
-config = require './config'
-common = require './common'
-soap = require './soap'
-session = require './session'
-query = require './query'
-device = require './device'
-cache = require './cache'
-localCache = require './local-cache'
-db = require './db'
-logger = require './logger'
-scheduling = require './scheduling'
-
-
-MAX_CYCLES = 4
-
-MAX_CONCURRENT_REQUESTS = config.get('MAX_CONCURRENT_REQUESTS')
-stats = {
-  concurrentRequests: 0
-  totalRequests: 0
-  droppedRequests: 0
-  initiatedSessions: 0
-}
-
-throwError = (err, httpResponse) ->
-  if httpResponse
-    currentSessions.delete(httpResponse.connection)
-    httpResponse.writeHead(500, {'Connection' : 'close'})
-    httpResponse.end("#{err.name}: #{err.message}")
-    stats.concurrentRequests -= 1
-
-  throw err
-
-
-writeResponse = (sessionContext, res, close) ->
-  # Close connection after last request in session
-  res.headers['Connection'] = 'close' if close
-
-  if config.get('DEBUG', sessionContext.deviceId)
-    dump = "# RESPONSE #{new Date(Date.now())}\n" + JSON.stringify(res.headers) + "\n#{res.data}\n\n"
-    require('fs').appendFile("./debug/#{sessionContext.deviceId}.dump", dump, (err) ->
-      throwError(err) if err
-    )
-
-  finish = () ->
-    stats.concurrentRequests -= 1
-    if sessionContext.httpRequest.connection.destroyed
-      logger.accessError({
-        sessionContext: sessionContext
-        message: 'Connection dropped'
-      })
-    else if close
-      endSession(sessionContext, (err, isNew) ->
-        return throwError(err) if err
-
-        if isNew
-          logger.accessInfo({
-            sessionContext: sessionContext
-            message: 'New device registered'
-          })
-      )
-    else
-      sessionContext.lastActivity = Date.now()
-      currentSessions.set(sessionContext.httpRequest.connection, sessionContext)
-
-  # respond using the same content-encoding as the request
-  if sessionContext.httpRequest.headers['content-encoding']? and res.data.length > 0
-    switch sessionContext.httpRequest.headers['content-encoding']
-      when 'gzip'
-        res.headers['Content-Encoding'] = 'gzip'
-        compress = zlib.gzip
-      when 'deflate'
-        res.headers['Content-Encoding'] = 'deflate'
-        compress = zlib.deflate
-
-  if compress?
-    compress(res.data, (err, data) ->
-      return throwError(err, sessionContext.httpResponse) if err
-      res.headers['Content-Length'] = data.length
-      sessionContext.httpResponse.writeHead(res.code, res.headers)
-      sessionContext.httpResponse.end(data)
-      finish()
-    )
-  else
-    res.headers['Content-Length'] = res.data.length
-    sessionContext.httpResponse.writeHead(res.code, res.headers)
-    sessionContext.httpResponse.end(res.data)
-    finish()
-
-
-recordFault = (sessionContext, fault, provisions, channels) ->
-  if not provisions
-    provisions = sessionContext.provisions
-    channels = sessionContext.channels
-
-  faults = sessionContext.faults
-
-  for channel of channels
-    provs = sessionContext.faults[channel]?.provisions or []
-    faults[channel] = Object.assign({provisions: provs}, fault)
-
-    if sessionContext.retries[channel]?
-      ++ sessionContext.retries[channel]
-    else
-      sessionContext.retries[channel] = 0
-      if Object.keys(channels).length != 1
-        faults[channel].retryNow = true
-
-    if channels[channel] == 0
-      faults[channel].precondition = true
-
-    sessionContext.faultsTouched ?= {}
-    sessionContext.faultsTouched[channel] = true
-
-    logger.accessWarn({
-      sessionContext: sessionContext
-      message: 'Channel has faulted'
-      fault: fault
-      channel: channel
-      retries: sessionContext.retries[channel]
-    })
-
-  for provision, i in provisions
-    for channel of channels
-      if (channels[channel] >> i) & 1
-        faults[channel].provisions.push(provision)
-
-  for channel of channels
-    provs = faults[channel].provisions
-    faults[channel].provisions = []
-    appendProvisions(faults[channel].provisions, provs)
-
-  session.clearProvisions(sessionContext)
-  return
-
-
-inform = (sessionContext, rpc) ->
-  session.inform(sessionContext, rpc.cpeRequest, (err, acsResponse) ->
-    return throwError(err, sessionContext.httpResponse) if err
-    res = soap.response({
-      id: rpc.id,
-      acsResponse: acsResponse,
-      cwmpVersion : sessionContext.cwmpVersion
-    })
-
-    if !!cookiesPath = config.get('COOKIES_PATH', sessionContext.deviceId)
-      res.headers['Set-Cookie'] = "session=#{sessionContext.sessionId}; Path=#{cookiesPath}"
-    else
-      res.headers['Set-Cookie'] = "session=#{sessionContext.sessionId}"
-
-    writeResponse(sessionContext, res)
-  )
-
-
-transferComplete = (sessionContext, rpc) ->
-  session.transferComplete(sessionContext, rpc.cpeRequest, (err, acsResponse, operation, fault) ->
-    return throwError(err, sessionContext.httpResponse) if err
-
-    if not operation?
-      logger.accessWarn({
-        sessionContext: sessionContext
-        message: 'Unrecognized command key'
-        rpc: rpc
-      })
-
-    if fault
-      for k, v of operation.retries
-        sessionContext.retries[k] = v
-
-      recordFault(sessionContext, fault, operation.provisions, operation.channels)
-
-    res = soap.response({
-      id : rpc.id,
-      acsResponse : acsResponse,
-      cwmpVersion : sessionContext.cwmpVersion
-    })
-    writeResponse(sessionContext, res)
-  )
-
-
-# Append providions and remove duplicates
-appendProvisions = (original, toAppend) ->
-  modified = false
-  stringified = new WeakMap()
-  for p, i in original
-    stringified.set(p, JSON.stringify(p))
-
-  for p, i in toAppend by -1
-    s = JSON.stringify(p)
-    for pp, j in original by -1
-      ss = stringified.get(pp)
-      if s == ss
-        if (not p) or j >= original.length - (toAppend.length - i)
-          p = null
-        else
-          original.splice(j, 1)
-          modified = true
-    if p
-      original.splice(original.length - (toAppend.length - i) + 1, 0, p)
-      stringified.set(p, s)
-      modified = true
-
-  return modified
-
-
-applyPresets = (sessionContext) ->
-  localCache.getPresets((err, presetsHash, presets) ->
-    return throwError(err, sessionContext.httpResponse) if err
-
-    # Filter presets based on existing faults
-    blackList = {}
-    whiteList = null
-    whiteListProvisions = null
-
-    RETRY_DELAY = config.get('RETRY_DELAY', sessionContext.deviceId)
-
-    if sessionContext.faults?
-      for channel, fault of sessionContext.faults
-        if fault.retryNow
-          retryTimestamp = 0
-        else
-          retryTimestamp = fault.timestamp + (RETRY_DELAY * Math.pow(2, sessionContext.retries[channel])) * 1000
-
-        if retryTimestamp <= sessionContext.timestamp
-          whiteList = channel
-          whiteListProvisions = fault.provisions
-          break
-
-        blackList[channel] = if fault.precondition then 1 else 2
-
-    sessionContext.deviceData.timestamps.revision = 1
-    sessionContext.deviceData.attributes.revision = 1
-
-    deviceEvents = {}
-    for p in sessionContext.deviceData.paths.find(['Events', '*'], false, true)
-      if sessionContext.timestamp <= sessionContext.deviceData.attributes.get(p)?.value?[1][0]
-        deviceEvents[p[1]] = true
-
-    parameters = {}
-    filteredPresets = []
-
-    for preset in presets
-      if whiteList?
-        continue if preset.channel != whiteList
-      else if blackList[preset.channel] == 1
-        continue
-
-      eventsMatch = true
-      for k, v of preset.events
-        if (!v) != (!deviceEvents[k.replace(' ', '_')])
-          eventsMatch = false
-          break
-
-      continue if not eventsMatch
-
-      if preset.schedule
-        continue if not preset.schedule.schedule
-        r = scheduling.cron(sessionContext.timestamp, preset.schedule.schedule)
-        if not (r[0] + preset.schedule.duration > sessionContext.timestamp)
-          continue
-
-      filteredPresets.push(preset)
-      for k of preset.precondition
-        sessionContext.channels[preset.channel] = 0
-        p = k.split(/([^a-zA-Z0-9\-\_\.].*)/, 1)[0]
-        parameters[p] = common.parsePath(p)
-
-    declarations = []
-    for k, v of parameters
-      declarations.push([v, 1, {value: 1}])
-
-    session.rpcRequest(sessionContext, declarations, (err, fault, id, acsRequest) ->
-      return throwError(err, sessionContext.httpResponse) if err
-
-      if fault
-        recordFault(sessionContext, fault)
-        session.clearProvisions(sessionContext)
-        return applyPresets(sessionContext)
-
-      if acsRequest
-        return sendAcsRequest(sessionContext, id, acsRequest)
-
-      session.clearProvisions(sessionContext)
-
-      for k, v of parameters
-        unpacked = device.unpack(sessionContext.deviceData, v)
-        if unpacked[0] and (vv = sessionContext.deviceData.attributes.get(unpacked[0]).value?[1])?
-          parameters[k] = vv[0]
-        else
-          delete parameters[k]
-
-      if whiteList?
-        session.addProvisions(sessionContext, whiteList, whiteListProvisions)
-
-      appendProvisionsToFaults = {}
-      for p in filteredPresets
-        if query.testFilter(parameters, p.precondition)
-          if blackList[p.channel] == 2
-            appendProvisionsToFaults[p.channel] =
-              (appendProvisionsToFaults[p.channel] or []).concat(p.provisions)
-          else
-            session.addProvisions(sessionContext, p.channel, p.provisions)
-
-      for channel, provisions of appendProvisionsToFaults
-        if appendProvisions(sessionContext.faults[channel].provisions, provisions)
-          sessionContext.faultsTouched ?= {}
-          sessionContext.faultsTouched[channel] = true
-
-      # Don't increment when processing individual channels (e.g. after fault)
-      sessionContext.presetCycles = (sessionContext.presetCycles or 0) + 1 if not whiteList?
-
-      if sessionContext.presetCycles > MAX_CYCLES
-        fault = {
-          code: 'preset_loop'
-          message: 'The presets are stuck in an endless configuration loop'
-          timestamp: sessionContext.timestamp
-        }
-        recordFault(sessionContext, fault)
-        # No need to save retryNow
-        for c of sessionContext.faults
-          delete sessionContext.faults[c].retryNow
-        session.clearProvisions(sessionContext)
-        return sendAcsRequest(sessionContext)
-
-      sessionContext.deviceData.timestamps.dirty = 0;
-      sessionContext.deviceData.attributes.dirty = 0;
-
-      session.rpcRequest(sessionContext, null, (err, fault, id, acsRequest) ->
-        return throwError(err, sessionContext.httpResponse) if err
-
-        if fault
-          recordFault(sessionContext, fault)
-          session.clearProvisions(sessionContext)
-          return applyPresets(sessionContext)
-
-        if not acsRequest
-          for channel, flags of sessionContext.channels
-            if channel of sessionContext.faults
-              delete sessionContext.faults[channel]
-              sessionContext.faultsTouched ?= {}
-              sessionContext.faultsTouched[channel] = true
-
-          if whiteList?
-            return applyPresets(sessionContext)
-
-          if sessionContext.deviceData.timestamps.dirty > 1 or
-              sessionContext.deviceData.attributes.dirty > 1
-            return applyPresets(sessionContext)
-
-        sendAcsRequest(sessionContext, id, acsRequest)
-      )
-    )
-  )
-
-
-nextRpc = (sessionContext) ->
-  session.rpcRequest(sessionContext, null, (err, fault, id, acsRequest) ->
-    return throwError(err, sessionContext.httpResponse) if err
-
-    if fault
-      recordFault(sessionContext, fault)
-      session.clearProvisions(sessionContext)
-      return nextRpc(sessionContext)
-
-    if acsRequest
-      return sendAcsRequest(sessionContext, id, acsRequest)
-
-    for channel, flags of sessionContext.channels
-      if flags and channel of sessionContext.faults
-        delete sessionContext.faults[channel]
-        sessionContext.faultsTouched ?= {}
-        sessionContext.faultsTouched[channel] = true
-
-      if channel.startsWith('task_')
-        taskId = channel.slice(5)
-        sessionContext.doneTasks ?= []
-        sessionContext.doneTasks.push(taskId)
-        for t, j in sessionContext.tasks
-          if t._id == taskId
-            sessionContext.tasks.splice(j, 1)
-            break
-
-    session.clearProvisions(sessionContext)
-
-    # Clear expired tasks
-    sessionContext.tasks = sessionContext.tasks.filter((task) ->
-      return true if not (task.expiry <= sessionContext.timestamp)
-
-      logger.accessInfo({
-        sessionContext: sessionContext
-        message: 'Task expired'
-        task: task
-      })
-      sessionContext.doneTasks ?= []
-      sessionContext.doneTasks.push(task._id)
-      channel = "task_#{task._id}"
-      if channel of sessionContext.faults
-        delete sessionContext.faults[channel]
-        sessionContext.faultsTouched ?= {}
-        sessionContext.faultsTouched[channel] = true
-
-      return false
-    )
-
-    task = sessionContext.tasks.find((task) -> !sessionContext.faults["task_#{task._id}"])
-
-    return applyPresets(sessionContext) if not task
-
-    switch task.name
-      when 'getParameterValues'
-        # Set channel in case params array is empty
-        sessionContext.channels["task_#{task._id}"] = 0
-        for p in task.parameterNames
-          session.addProvisions(sessionContext, "task_#{task._id}",
-            [['refresh', p]])
-      when 'setParameterValues'
-        # Set channel in case params array is empty
-        sessionContext.channels["task_#{task._id}"] = 0
-        for p in task.parameterValues
-          session.addProvisions(sessionContext, "task_#{task._id}",
-            [['value', p[0], p[1]]])
-      when 'refreshObject'
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['refresh', task.objectName]])
-      when 'reboot'
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['reboot']])
-      when 'factoryReset'
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['reset']])
-      when 'download'
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['download', task.fileType, task.fileName, task.targetFileName]])
-      when 'addObject'
-        alias = ("#{p[0]}:#{JSON.stringify(p[1])}" for p in task.parameterValues or []).join(',')
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['instances', "#{task.objectName}.[#{alias}]", "+1"]]);
-      when 'deleteObject'
-        session.addProvisions(sessionContext, "task_#{task._id}",
-          [['instances', task.objectName, 0]]);
-      else
-        return throwError(new Error('Task name not recognized'), sessionContext.httpResponse)
-
-    return nextRpc(sessionContext)
-  )
-
-
-endSession = (sessionContext, callback) ->
-  saveCache = sessionContext.cacheUntil?
-
-  counter = 3
-
-  counter += 2
-  db.saveDevice(sessionContext.deviceId, sessionContext.deviceData, sessionContext.new, sessionContext.timestamp, (err) ->
-    if err
-      callback(err) if counter & 1
-      return counter = 0
-
-    return callback(null, sessionContext.new) if (counter -= 2) == 1
-  )
-
-  for k of sessionContext.operationsTouched
-    counter += 2
-    saveCache = true
-    if sessionContext.operations[k]?
-      db.saveOperation(sessionContext.deviceId, k, sessionContext.operations[k], (err) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        return callback(null, sessionContext.new) if (counter -= 2) == 1
-      )
-    else
-      db.deleteOperation(sessionContext.deviceId, k, (err) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        return callback(null, sessionContext.new) if (counter -= 2) == 1
-      )
-
-  if sessionContext.doneTasks?.length
-    counter += 2
-    saveCache = true
-    db.clearTasks(sessionContext.deviceId, sessionContext.doneTasks, (err) ->
-      if err
-        callback(err) if counter & 1
-        return counter = 0
-
-      return callback(null, sessionContext.new) if (counter -= 2) == 1
-    )
-
-  for k of sessionContext.faultsTouched
-    counter += 2
-    saveCache = true
-    if sessionContext.faults[k]
-      sessionContext.faults[k].retries = sessionContext.retries[k]
-      db.saveFault(sessionContext.deviceId, k, sessionContext.faults[k], (err) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        return callback(null, sessionContext.new) if (counter -= 2) == 1
-      )
-    else
-      db.deleteFault(sessionContext.deviceId, k, (err) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        return callback(null, sessionContext.new) if (counter -= 2) == 1
-      )
-
-  if saveCache
-    counter += 2
-    cacheDueTasksAndFaultsAndOperations(sessionContext.deviceId, sessionContext.tasks, sessionContext.faults, sessionContext.operations, sessionContext.cacheUntil, (err) ->
-      if err
-        callback(err) if counter & 1
-        return counter = 0
-
-      return callback(null, sessionContext.new) if (counter -= 2) == 1
-    )
-
-  callback(null, sessionContext.new) if (counter -= 2) == 1
-
-
-sendAcsRequest = (sessionContext, id, acsRequest) ->
-  if not acsRequest?
-    return writeResponse(sessionContext, soap.response(null), true)
-
-  if acsRequest.name is 'Download'
-    if not acsRequest.url?
-      FS_PORT = config.get('FS_PORT')
-      FS_HOSTNAME = config.get('FS_HOSTNAME')
-      FS_SSL = config.get('FS_SSL')
-      acsRequest.url = if FS_SSL then 'https://' else 'http://'
-      acsRequest.url += FS_HOSTNAME
-      acsRequest.url += ":#{FS_PORT}" if FS_PORT != 80
-      acsRequest.url += "/#{encodeURI(acsRequest.fileName)}"
-
-    if not acsRequest.fileSize?
-      return localCache.getFiles((err, hash, files) ->
-        return throwError(err, sessionContext.httpResponse) if err
-
-        if acsRequest.fileName of files
-          acsRequest.fileSize = files[acsRequest.fileName].length
-        else
-          acsRequest.fileSize = 0
-
-        return sendAcsRequest(sessionContext, id, acsRequest)
-      )
-
-  rpc = {
-    id : id,
-    acsRequest : acsRequest,
-    cwmpVersion : sessionContext.cwmpVersion
-  }
-
-  logger.accessInfo({
-    sessionContext: sessionContext
-    message: 'ACS request'
-    rpc: rpc
-  })
-
-  res = soap.response(rpc)
-
-  writeResponse(sessionContext, res)
-
-
-getSession = (connection, sessionId, callback) ->
-  sessionContext = currentSessions.get(connection)
-
-  if sessionContext
-    currentSessions.delete(connection)
-    return callback(null, sessionContext)
-
-  return callback() if not sessionId
-
-  setTimeout(() ->
-    cache.pop("session_#{sessionId}", (err, sessionContextString) ->
-      return callback(err) if err or not sessionContextString
-      session.deserialize(sessionContextString, callback)
-    )
-  , 100)
-
-
-currentSessions = new WeakMap()
-
-# When socket closes, store active sessions in cache
-onConnection = (socket) ->
-  # The property remoteAddress may be undefined after the connection is
-  # closed, unless we read it at least once (caching?)
-  socket.remoteAddress
-
-  socket.on('close', () ->
-    sessionContext = currentSessions.get(socket)
-    return if not sessionContext
-    currentSessions.delete(socket)
-    now = Date.now()
-
-    lastActivity = sessionContext.lastActivity
-    timeoutMsg = logger.flatten({
-      sessionContext: sessionContext
-      message: 'Session timeout'
-      sessionTimestamp: sessionContext.timestamp
-    })
-
-    timeout = (sessionContext.lastActivity + sessionContext.timeout * 1000) - now
-    if timeout <= 0
-      return logger.accessError(timeoutMsg)
-
-    setTimeout(() ->
-      cache.get("session_#{sessionContext.sessionId}", (err, sessionContextString) ->
-        return throwError(err) if err
-        return if not sessionContextString
-        session.deserialize(sessionContextString, (err, sessionContext) ->
-          return throwError(err) if err
-          if sessionContext.lastActivity == lastActivity
-            logger.accessError(timeoutMsg)
-        )
-      )
-    , timeout + 1000).unref()
-
-    session.serialize(sessionContext, (err, sessionContextString) ->
-      return throwError(err) if err
-      cache.set("session_#{sessionContext.sessionId}", sessionContextString, Math.ceil(timeout / 1000) + 3, (err) ->
-        return throwError(err) if err
-      )
-    )
-  )
-
-setInterval(() ->
-  if stats.droppedRequests
-    logger.warn({
-      message: 'Worker overloaded'
-      droppedRequests: stats.droppedRequests
-      totalRequests: stats.totalRequests
-      initiatedSessions: stats.initiatedSessions
-      pid: process.pid
-    })
-
-  stats.totalRequests = 0
-  stats.droppedRequests = 0
-  stats.initiatedSessions = 0
-, 10000).unref()
-
-
-getDueTasksAndFaultsAndOperations = (deviceId, timestamp, callback) ->
-  cache.get("#{deviceId}_tasks_faults_operations", (err, res) ->
-    return callback(err) if err
-
-    if res
-      res = JSON.parse(res)
-      return callback(null, res.tasks or [], res.faults or {}, res.operations or {})
-
-    faults = tasks = operations = cacheUntil = null
-
-    db.getFaults(deviceId, (err, _faults) ->
-      if err
-        callback?(err)
-        return callback = null
-
-      faults = _faults
-      if tasks? and operations?
-        return callback(null, tasks, faults, operations, cacheUntil)
-    )
-
-    db.getDueTasks(deviceId, timestamp, (err, dueTasks, nextTimestamp) ->
-      if err
-        callback?(err)
-        return callback = null
-
-      tasks = dueTasks
-      cacheUntil = nextTimestamp or 0
-
-      if faults? and operations?
-        return callback(null, tasks, faults, operations, cacheUntil)
-    )
-
-    db.getOperations(deviceId, (err, _operations) ->
-      if err
-        callback?(err)
-        return callback = null
-
-      operations = _operations
-
-      if faults? and tasks?
-        return callback(null, tasks, faults, operations, cacheUntil)
-    )
-
-    if faults? and tasks? and operations?
-      return callback(null, tasks, faults, operations, cacheUntil)
-  )
-
-
-cacheDueTasksAndFaultsAndOperations = (deviceId, tasks, faults, operations, cacheUntil, callback) ->
-  v = {}
-  v.tasks = tasks if tasks.length
-  v.faults = faults if Object.keys(faults).length
-  v.operations = operations if Object.keys(operations).length
-
-  if cacheUntil
-    ttl = Math.trunc((Date.now() - cacheUntil) / 1000)
-  else
-    ttl = config.get('MAX_CACHE_TTL', deviceId)
-
-  cache.set("#{deviceId}_tasks_faults_operations", JSON.stringify(v), ttl, callback)
-
-
-processRequest = (sessionContext, rpc) ->
-  if rpc.cpeRequest?
-    if rpc.cpeRequest.name is 'Inform'
-      logger.accessInfo({
-        sessionContext: sessionContext
-        message: 'Inform'
-        rpc: rpc
-      })
-      inform(sessionContext, rpc)
-    else if rpc.cpeRequest.name is 'TransferComplete'
-      logger.accessInfo({
-        sessionContext: sessionContext
-        message: 'CPE request'
-        rpc: rpc
-      })
-      transferComplete(sessionContext, rpc)
-    else if rpc.cpeRequest.name is 'GetRPCMethods'
-      logger.accessInfo({
-        sessionContext: sessionContext
-        message: 'CPE request'
-        rpc: rpc
-      })
-      res = soap.response({
-        id : rpc.id
-        acsResponse : {name: 'GetRPCMethodsResponse', methodList: ['Inform', 'GetRPCMethods', 'TransferComplete']}
-        cwmpVersion : sessionContext.cwmpVersion
-      })
-      writeResponse(sessionContext, res)
-    else
-      return throwError(new Error('ACS method not supported'), sessionContext.httpResponse)
-  else if rpc.cpeResponse
-    session.rpcResponse(sessionContext, rpc.id, rpc.cpeResponse, (err) ->
-      return throwError(err, sessionContext.httpResponse) if err
-      nextRpc(sessionContext)
-    )
-  else if rpc.cpeFault
-    logger.accessWarn({
-      sessionContext: sessionContext
-      message: 'CPE fault'
-      rpc:rpc
-    })
-    session.rpcFault(sessionContext, rpc.id, rpc.cpeFault, (err, fault) ->
-      return throwError(err, sessionContext.httpResponse) if err
-
-      if fault
-        recordFault(sessionContext, fault)
-        session.clearProvisions(sessionContext)
-      nextRpc(sessionContext)
-    )
-  else # CPE sent empty response
-    session.timeoutOperations(sessionContext, (err, faults, operations) ->
-      return throwError(err, sessionContext.httpResponse) if err
-
-      for fault, i in faults
-        for k, v of operations[i].retries
-          sessionContext.retries[k] = v
-
-        recordFault(sessionContext, fault, operations[i].provisions, operations[i].channels)
-
-      nextRpc(sessionContext)
-    )
-
-
-listener = (httpRequest, httpResponse) ->
-  stats.totalRequests += 1
-  if httpRequest.method != 'POST'
-    httpResponse.writeHead 405, {'Allow': 'POST', 'Connection': 'close'}
-    httpResponse.end('405 Method Not Allowed')
-    return
-
-  sessionId = null
-  # Separation by comma is important as some devices don't comform to standard
-  COOKIE_REGEX = /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g
-  while match = COOKIE_REGEX.exec(httpRequest.headers.cookie)
-    sessionId = match[2] if match[1] == 'session'
-
-  # If overloaded, ask CPE to retry in 60 seconds
-  if not sessionId and stats.concurrentRequests > MAX_CONCURRENT_REQUESTS
-    httpResponse.writeHead(503, {'Retry-after': 60, 'Connection': 'close'})
-    httpResponse.end('503 Service Unavailable')
-    stats.droppedRequests += 1
-    return
-
-  if httpRequest.headers['content-encoding']?
-    switch httpRequest.headers['content-encoding']
-      when 'gzip'
-        stream = httpRequest.pipe(zlib.createGunzip())
-      when 'deflate'
-        stream = httpRequest.pipe(zlib.createInflate())
-      else
-        httpResponse.writeHead(415, {'Connection': 'close'})
-        httpResponse.end('415 Unsupported Media Type')
-        return
-  else
-    stream = httpRequest
-
-  stats.concurrentRequests += 1
-  httpRequest.on('aborted', () ->
-    stats.concurrentRequests -= 1
-    # In some cases event end can be emitted after aborted event
-    httpRequest.removeAllListeners('end')
-  )
-
-  chunks = []
-  bytes = 0
-
-  stream.on('data', (chunk) ->
-    chunks.push(chunk)
-    bytes += chunk.length
-  )
-
-  stream.on('end', () ->
-    body = new Buffer(bytes)
-    offset = 0
-    chunks.forEach((chunk) ->
-      chunk.copy(body, offset, 0, chunk.length)
-      offset += chunk.length
-    )
-
-    parsedRpc = (sessionContext, rpc, parseWarnings) ->
-      for w in parseWarnings
-        w.sessionContext = sessionContext
-        w.rpc = rpc;
-        logger.accessWarn(w)
-
-      if config.get('DEBUG', sessionContext.deviceId)
-        dump = "# REQUEST #{new Date(Date.now())}\n" + JSON.stringify(httpRequest.headers) + "\n#{body}\n\n"
-        require('fs').appendFile("./debug/#{sessionContext.deviceId}.dump", dump, (err) ->
-          return throwError(err) if err
-        )
-
-      processRequest(sessionContext, rpc)
-
-    getSession(httpRequest.connection, sessionId, (err, sessionContext) ->
-      return throwError(err, httpResponse) if err
-
-      if sessionContext
-        sessionContext.httpRequest = httpRequest
-        sessionContext.httpResponse = httpResponse
-
-        if sessionContext.sessionId != sessionId or
-            sessionContext.lastActivity + sessionContext.timeout * 1000 < Date.now()
-          logger.accessError({message: 'Invalid session', sessionContext: sessionContext})
-          httpResponse.writeHead(400, {'Connection': 'close'})
-          httpResponse.end('Invalid session')
-          stats.concurrentRequests -= 1
-          return
-      else if stats.concurrentRequests > MAX_CONCURRENT_REQUESTS
-        # Check again just in case device included old session ID from the previous session
-        httpResponse.writeHead(503, {'Retry-after': 60, 'Connection': 'close'})
-        httpResponse.end('503 Service Unavailable')
-        stats.droppedRequests += 1
-        stats.concurrentRequests -= 1
-        return
-
-      parseWarnings = []
-      try
-        rpc = soap.request(body, sessionContext?.cwmpVersion, parseWarnings)
-      catch err
-        logger.accessError({
-          message: 'XML parse error'
-          parseError: err.message.trim()
-          sessionContext: sessionContext or {httpRequest: httpRequest, httpResponse: httpResponse}
-        })
-        httpResponse.writeHead(400, {'Connection': 'close'})
-        httpResponse.end(err.message)
-        stats.concurrentRequests -= 1
-        return
-
-      if sessionContext
-        if rpc.cpeRequest?.name is 'Inform' or
-            (!sessionContext.rpcRequest ^ !(rpc.cpeResponse or rpc.cpeFault))
-          logger.accessError({
-            message: 'Bad session state'
-            sessionContext: sessionContext
-          })
-          httpResponse.writeHead(400, {'Connection': 'close'})
-          httpResponse.end('Bad session state')
-          stats.concurrentRequests -= 1
-          return
-
-        return parsedRpc(sessionContext, rpc, parseWarnings)
-
-      if rpc.cpeRequest?.name isnt 'Inform'
-        logger.accessError({
-          message: 'Invalid session'
-          sessionContext: sessionContext or {httpRequest: httpRequest, httpResponse: httpResponse}
-        })
-        httpResponse.writeHead(400, {'Connection': 'close'})
-        httpResponse.end('Invalid session')
-        stats.concurrentRequests -= 1
-        return
-
-      stats.initiatedSessions += 1
-      deviceId = common.generateDeviceId(rpc.cpeRequest.deviceId)
-      return session.init(deviceId, rpc.cwmpVersion, rpc.sessionTimeout ? config.get('SESSION_TIMEOUT', deviceId), (err, sessionContext) ->
-        return throwError(err, httpResponse) if err
-
-        sessionContext.httpRequest = httpRequest
-        sessionContext.httpResponse = httpResponse
-        sessionContext.sessionId = crypto.randomBytes(8).toString('hex')
-        httpRequest.connection.setTimeout(sessionContext.timeout * 1000)
-
-        getDueTasksAndFaultsAndOperations(deviceId, sessionContext.timestamp, (err, dueTasks, faults, operations, cacheUntil) ->
-          return throwError(err, httpResponse) if err
-
-          sessionContext.tasks = dueTasks
-          sessionContext.faults = faults
-          sessionContext.retries = {}
-          for k, v of faults
-            sessionContext.retries[k] = v.retries
-          sessionContext.operations = operations
-          sessionContext.cacheUntil = cacheUntil
-
-          # Delete expired faults
-          for k, v of sessionContext.faults
-            if v.expiry >= sessionContext.timestamp
-              delete sessionContext.faults[k]
-              sessionContext.faultsTouched ?= {}
-              sessionContext.faultsTouched[k] = true
-
-          parsedRpc(sessionContext, rpc, parseWarnings)
-        )
-      )
-    )
-  )
-
-
-exports.listener = listener
-exports.onConnection = onConnection

lib/cwmp/db.ts

@@ -0,0 +1,732 @@
+import { ObjectId } from "mongodb";
+import { decodeTag, encodeTag, escapeRegExp } from "../util.ts";
+import {
+  DeviceData,
+  Attributes,
+  SessionFault,
+  Task,
+  Operation,
+} from "../types.ts";
+import Path from "../common/path.ts";
+import { collections } from "../db/db.ts";
+import { optimizeProjection } from "../db/util.ts";
+import * as MongoTypes from "../db/types.ts";
+
+const INVALID_PATH_SUFFIX = "__invalid";
+
+function compareAccessLists(list1: string[], list2: string[]): boolean {
+  if (list1.length !== list2.length) return false;
+  for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
+  return true;
+}
+
+export async function fetchDevice(
+  id: string,
+  timestamp: number,
+): Promise<[Path, number, Attributes?][]> {
+  const res: [Path, number, Attributes?][] = [
+    [
+      Path.parse("Events"),
+      timestamp,
+      { object: [timestamp, 1], writable: [timestamp, 0] },
+    ],
+    [
+      Path.parse("DeviceID"),
+      timestamp,
+      { object: [timestamp, 1], writable: [timestamp, 0] },
+    ],
+  ];
+
+  const device = await collections.devices.findOne({ _id: id });
+  if (!device) return null;
+
+  function storeParams(
+    obj,
+    path: string,
+    pathLength: number,
+    ts: number,
+  ): void {
+    if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
+    if (obj["_attributesTimestamp"])
+      obj["_attributesTimestamp"] = +obj["_attributesTimestamp"];
+
+    const attrs: Attributes = {};
+    let t = obj["_timestamp"] || 1;
+    if (ts > t) t = ts;
+
+    if (obj["_value"] != null) {
+      attrs.value = [obj["_timestamp"] || 1, [obj["_value"], obj["_type"]]];
+      if (obj["_type"] === "xsd:dateTime" && obj["_value"] instanceof Date)
+        attrs.value[1][0] = +attrs.value[1][0];
+
+      obj["_object"] = false;
+    }
+    if (obj["_writable"] != null)
+      attrs.writable = [ts || 1, obj["_writable"] ? 1 : 0];
+
+    if (obj["_object"] != null) attrs.object = [t, obj["_object"] ? 1 : 0];
+
+    if (obj["_notification"] != null) {
+      attrs.notification = [
+        obj["_attributesTimestamp"] || 1,
+        obj["_notification"],
+      ];
+    }
+
+    if (obj["_accessList"] != null)
+      attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
+
+    try {
+      res.push([Path.parse(path), t, attrs]);
+    } catch (err) {
+      // The path parser is now more strict so we might be in a situation where
+      // the database contains invalid paths from before this change So here we
+      // encode the invalid characters.
+      const splits = path.split(".");
+      splits[splits.length - 1] =
+        encodeTag(splits[splits.length - 1]) + INVALID_PATH_SUFFIX;
+      path = splits.join(".");
+      res.push([Path.parse(path), t, attrs]);
+      return;
+    }
+
+    for (const [k, v] of Object.entries(obj)) {
+      if (!k.startsWith("_")) {
+        obj["_object"] = true;
+        storeParams(v, `${path}.${k}`, pathLength + 1, obj["_timestamp"]);
+      }
+    }
+
+    if (obj["_object"] && obj["_timestamp"])
+      res.push([Path.parse(path + ".*"), obj["_timestamp"]]);
+  }
+
+  const ts: number = +device["_timestamp"] || 0;
+  if (ts) res.push([Path.parse("*"), ts]);
+
+  for (const [k, v] of Object.entries(device)) {
+    switch (k) {
+      case "_lastInform":
+        res.push([
+          Path.parse("Events.Inform"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
+        ]);
+        break;
+      case "_lastBoot":
+        res.push([
+          Path.parse("Events.1_BOOT"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
+        ]);
+        break;
+      case "_lastBootstrap":
+        res.push([
+          Path.parse("Events.0_BOOTSTRAP"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
+        ]);
+        break;
+      case "_registered":
+        // Use current timestamp for registered event attribute timestamps
+        res.push([
+          Path.parse("Events.Registered"),
+          timestamp,
+          {
+            object: [timestamp, 0],
+            writable: [timestamp, 0],
+            value: [timestamp, [+v, "xsd:dateTime"]],
+          },
+        ]);
+        break;
+      case "_id":
+        res.push([
+          Path.parse("DeviceID.ID"),
+          timestamp,
+          {
+            object: [timestamp, 0],
+            writable: [timestamp, 0],
+            value: [timestamp, [v as string, "xsd:string"]],
+          },
+        ]);
+        break;
+      case "_tags":
+        if ((v as string[]).length) {
+          res.push([
+            Path.parse("Tags"),
+            timestamp,
+            { object: [timestamp, 1], writable: [timestamp, 0] },
+          ]);
+        }
+
+        for (const t of v as string[]) {
+          res.push([
+            Path.parse("Tags." + encodeTag(t)),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 1],
+              value: [timestamp, [true, "xsd:boolean"]],
+            },
+          ]);
+        }
+        break;
+      case "_deviceId":
+        if (v["_Manufacturer"] != null) {
+          res.push([
+            Path.parse("DeviceID.Manufacturer"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_Manufacturer"], "xsd:string"]],
+            },
+          ]);
+        }
+
+        if (v["_OUI"] != null) {
+          res.push([
+            Path.parse("DeviceID.OUI"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_OUI"], "xsd:string"]],
+            },
+          ]);
+        }
+
+        if (v["_ProductClass"] != null) {
+          res.push([
+            Path.parse("DeviceID.ProductClass"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_ProductClass"], "xsd:string"]],
+            },
+          ]);
+        }
+
+        if (v["_SerialNumber"] != null) {
+          res.push([
+            Path.parse("DeviceID.SerialNumber"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_SerialNumber"], "xsd:string"]],
+            },
+          ]);
+        }
+        break;
+      default:
+        if (!k.startsWith("_")) storeParams(v, k, 1, ts);
+    }
+  }
+  return res;
+}
+
+export async function saveDevice(
+  deviceId: string,
+  deviceData: DeviceData,
+  isNew: boolean,
+  sessionTimestamp: number,
+): Promise<void> {
+  const update = { $set: {}, $unset: {}, $addToSet: {}, $pull: {} };
+
+  for (const diff of deviceData.timestamps.diff()) {
+    if (diff[0].wildcard !== 1 << (diff[0].length - 1)) continue;
+
+    if (
+      diff[0].segments[0] === "Events" ||
+      diff[0].segments[0] === "DeviceID" ||
+      diff[0].segments[0] === "Tags"
+    )
+      continue;
+
+    const parent = deviceData.paths.get(diff[0].slice(0, -1));
+
+    // Param timestamps may be greater than session timestamp to track revisions
+    if (diff[2] > sessionTimestamp) diff[2] = sessionTimestamp;
+
+    if (diff[2] == null && diff[1] != null) {
+      update["$unset"][
+        parent.length ? parent.toString() + "._timestamp" : "_timestamp"
+      ] = 1;
+    } else {
+      if (parent && (!parent.length || deviceData.attributes.has(parent))) {
+        update["$set"][
+          parent.length ? parent.toString() + "._timestamp" : "_timestamp"
+        ] = new Date(diff[2]);
+      }
+    }
+  }
+
+  for (const diff of deviceData.attributes.diff()) {
+    const path = diff[0];
+    const value1 = (((diff[1] || {}).value || [])[1] || [])[0];
+    const value2 = (((diff[2] || {}).value || [])[1] || [])[0];
+    const valueType1 = (((diff[1] || {}).value || [])[1] || [])[1];
+    const valueType2 = (((diff[2] || {}).value || [])[1] || [])[1];
+    const valueTimestamp1 = ((diff[1] || {}).value || [])[0];
+    const valueTimestamp2 = ((diff[2] || {}).value || [])[0];
+    const object1 = ((diff[1] || {}).object || [])[1];
+    const object2 = ((diff[2] || {}).object || [])[1];
+    const writable2 = ((diff[2] || {}).writable || [])[1];
+    const writable1 = ((diff[1] || {}).writable || [])[1];
+    const attributesTimestamp1 = ((diff[1] || {}).notification || [])[0];
+    const attributesTimestamp2 = ((diff[2] || {}).notification || [])[0];
+    const notification1 = ((diff[1] || {}).notification || [])[1];
+    const notification2 = ((diff[2] || {}).notification || [])[1];
+    const accessList1 = ((diff[1] || {}).accessList || [])[1];
+    const accessList2 = ((diff[2] || {}).accessList || [])[1];
+
+    switch (path.segments[0]) {
+      case "Events":
+        if (path.length === 2 && value2 !== value1) {
+          if (!diff[2]) {
+            switch (path.segments[1]) {
+              case "Inform":
+                update["$unset"]["_lastInform"] = 1;
+                break;
+              case "1_BOOT":
+                update["$unset"]["_lastBoot"] = 1;
+                break;
+              case "0_BOOTSTRAP":
+                update["$unset"]["_lastBootstrap"] = 1;
+                break;
+              case "Registered":
+                update["$unset"]["_registered"] = 1;
+            }
+          } else {
+            const t = new Date(diff[2].value[1][0] as number);
+            switch (path.segments[1]) {
+              case "Inform":
+                update["$set"]["_lastInform"] = t;
+                break;
+              case "1_BOOT":
+                update["$set"]["_lastBoot"] = t;
+                break;
+              case "0_BOOTSTRAP":
+                update["$set"]["_lastBootstrap"] = t;
+                break;
+              case "Registered":
+                update["$set"]["_registered"] = t;
+            }
+          }
+        }
+
+        break;
+      case "DeviceID":
+        if (value2 !== value1) {
+          const v = diff[2].value[1][0];
+          switch (path.segments[1]) {
+            case "ID":
+              update["$set"]["_id"] = v;
+              break;
+            case "Manufacturer":
+              update["$set"]["_deviceId._Manufacturer"] = v;
+              break;
+            case "OUI":
+              update["$set"]["_deviceId._OUI"] = v;
+              break;
+            case "ProductClass":
+              update["$set"]["_deviceId._ProductClass"] = v;
+              break;
+            case "SerialNumber":
+              update["$set"]["_deviceId._SerialNumber"] = v;
+          }
+        }
+        break;
+      case "Tags":
+        if (value2 !== value1) {
+          if (value2 != null) {
+            if (!update["$addToSet"]["_tags"])
+              update["$addToSet"]["_tags"] = { $each: [] };
+            update["$addToSet"]["_tags"]["$each"].push(
+              decodeTag(path.segments[1] as string),
+            );
+          } else {
+            if (!update["$pull"]["_tags"]) {
+              update["$pull"]["_tags"] = {
+                $in: [],
+              };
+            }
+            update["$pull"]["_tags"]["$in"].push(
+              decodeTag(path.segments[1] as string),
+            );
+          }
+        }
+
+        break;
+      default:
+        if (!diff[2]) {
+          let pathStr = path.toString();
+          // Paths with that suffix are encoded and need to be decoded
+          if (pathStr.endsWith(INVALID_PATH_SUFFIX)) {
+            const splits = pathStr.split(".");
+            splits[splits.length - 1] = decodeTag(
+              splits[splits.length - 1].slice(
+                0,
+                0 - INVALID_PATH_SUFFIX.length,
+              ),
+            );
+            pathStr = splits.join(".");
+          }
+          update["$unset"][pathStr] = 1;
+          continue;
+        }
+
+        for (const attrName of Object.keys(diff[2])) {
+          // Param timestamps may be greater than session timestamp to track revisions
+          if (diff[2][attrName][0] > sessionTimestamp)
+            diff[2][attrName][0] = sessionTimestamp;
+
+          if (diff[2][attrName][1] != null) {
+            switch (attrName) {
+              case "value":
+                if (value2 !== value1) {
+                  if (
+                    valueType2 === "xsd:dateTime" &&
+                    Number.isInteger(value2 as number)
+                  ) {
+                    update["$set"][path.toString() + "._value"] = new Date(
+                      value2 as number,
+                    );
+                  } else {
+                    update["$set"][path.toString() + "._value"] = value2;
+                  }
+                }
+
+                if (valueType2 !== valueType1)
+                  update["$set"][path.toString() + "._type"] = valueType2;
+
+                if (valueTimestamp2 !== valueTimestamp1) {
+                  update["$set"][path.toString() + "._timestamp"] = new Date(
+                    valueTimestamp2,
+                  );
+                }
+
+                break;
+              case "object":
+                if (!diff[1]?.object || object2 !== object1) {
+                  update["$set"][
+                    path.length ? path.toString() + "._object" : "_object"
+                  ] = !!object2;
+                }
+
+                break;
+              case "writable":
+                if (!diff[1]?.writable || writable2 !== writable1) {
+                  update["$set"][
+                    path.length ? path.toString() + "._writable" : "_writable"
+                  ] = !!writable2;
+                }
+
+                break;
+              case "notification":
+                if (
+                  !diff[1] ||
+                  !diff[1].notification ||
+                  notification2 !== notification1
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._notification"
+                      : "_notification"
+                  ] = notification2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][path.toString() + "._attributesTimestamp"] =
+                    new Date(attributesTimestamp2);
+                }
+
+                break;
+              case "accessList":
+                if (
+                  !diff[1] ||
+                  !diff[1].accessList ||
+                  !compareAccessLists(accessList2, accessList1)
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._accessList"
+                      : "_accessList"
+                  ] = accessList2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][path.toString() + "._attributesTimestamp"] =
+                    new Date(attributesTimestamp2);
+                }
+            }
+          }
+        }
+
+        if (diff[1]) {
+          for (const attrName of Object.keys(diff[1])) {
+            if (
+              diff[1][attrName][1] != null &&
+              diff[2]?.[attrName]?.[1] == null
+            ) {
+              const p = path.length ? path.toString() + "." : "";
+              update["$unset"][`${p}_${attrName}`] = 1;
+              if (attrName === "value") {
+                update["$unset"][p + "_type"] = 1;
+                update["$unset"][p + "_timestamp"] = 1;
+              } else if (attrName === "notification") {
+                if (accessList2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
+              } else if (attrName === "accessList") {
+                if (notification2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
+              }
+            }
+          }
+        }
+    }
+  }
+
+  update["$unset"] = optimizeProjection(update["$unset"]);
+
+  // Remove overlap possibly caused by parameters changing from objects
+  // to regular parameters or vice versa. Reason being that _timestamp
+  // represents two different things depending on whether the parameter
+  // is an object or not.
+  for (const k of Object.keys(update["$unset"]))
+    if (update["$set"][k] != null) delete update["$unset"][k];
+
+  // Remove empty keys
+  for (const [k, v] of Object.entries(update)) {
+    if (k === "$addToSet") {
+      for (const [kk, vv] of Object.entries(v))
+        if (!vv["$each"].length) delete v[kk];
+    } else if (k === "$pull") {
+      for (const [kk, vv] of Object.entries(v))
+        if (!vv["$in"].length) delete v[kk];
+    }
+    if (!Object.keys(v).length) delete update[k];
+  }
+
+  if (!Object.keys(update).length) return;
+
+  // Mongo doesn't allow $addToSet and $pull at the same time
+  let update2;
+  if (update["$addToSet"] && update["$pull"]) {
+    update2 = { $pull: update["$pull"] };
+    delete update["$pull"];
+  }
+
+  const result = await collections.devices.updateOne(
+    { _id: deviceId },
+    update,
+    {
+      upsert: isNew,
+    },
+  );
+
+  if (!result.matchedCount && !result.upsertedCount)
+    throw new Error(`Device ${deviceId} not found in database`);
+
+  if (update2) {
+    await collections.devices.updateOne({ _id: deviceId }, update2);
+    return;
+  }
+}
+
+export async function getFaults(
+  deviceId: string,
+): Promise<{ [channel: string]: SessionFault }> {
+  const res = await collections.faults
+    .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
+    .toArray();
+
+  const faults: { [channel: string]: SessionFault } = {};
+  for (const r of res) {
+    const channel = r._id.slice(deviceId.length + 1);
+    const fault: SessionFault = {
+      code: r.code,
+      message: r.message,
+      ...(r.detail && { detail: r.detail }),
+      timestamp: +r.timestamp,
+      provisions: JSON.parse(r.provisions),
+      retries: r.retries,
+      ...(r.expiry && { expiry: +r.expiry }),
+    };
+    faults[channel] = fault;
+  }
+
+  return faults;
+}
+
+export async function saveFault(
+  deviceId: string,
+  channel: string,
+  fault: SessionFault,
+): Promise<void> {
+  const id = `${deviceId}:${channel}`;
+  const f: MongoTypes.Fault = {
+    _id: id,
+    device: deviceId,
+    channel: channel,
+    timestamp: new Date(fault.timestamp),
+    code: fault.code,
+    message: fault.message,
+    ...(fault.detail && { detail: fault.detail }),
+    retries: fault.retries,
+    ...(fault.expiry && { expiry: new Date(fault.expiry) }),
+    provisions: JSON.stringify(fault.provisions),
+  };
+  await collections.faults.replaceOne({ _id: id }, f, { upsert: true });
+}
+
+export async function deleteFault(
+  deviceId: string,
+  channel: string,
+): Promise<void> {
+  await collections.faults.deleteOne({ _id: `${deviceId}:${channel}` });
+}
+
+export async function getDueTasks(
+  deviceId: string,
+  timestamp: number,
+): Promise<[Task[], number]> {
+  const cur = collections.tasks
+    .find({ device: deviceId })
+    .sort({ timestamp: 1 });
+  const tasks = [] as Task[];
+
+  for await (const t of cur) {
+    if (+t.timestamp >= timestamp) return [tasks, +t.timestamp];
+    const task: Task = {
+      _id: t._id.toString(),
+      name: t.name,
+      ...(t.timestamp && { timestamp: +t.timestamp }),
+      ...(t.expiry && { expiry: +t.expiry }),
+      ...(t.name === "getParameterValues" && {
+        parameterNames: t.parameterNames,
+      }),
+      ...(t.name === "setParameterValues" && {
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "refreshObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "download" && {
+        fileType: t.fileType,
+        fileName: t.fileName,
+        targetFileName: t.targetFileName,
+      }),
+      ...(t.name === "addObject" && {
+        objectName: t.objectName,
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "deleteObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "provisions" && {
+        provisions: t.provisions,
+      }),
+    };
+
+    tasks.push(task);
+
+    // For API compatibility
+    if (task.name === "download" && t["file"]) {
+      let q;
+      if (ObjectId.isValid(t["file"]))
+        q = { _id: { $in: [t["file"], new ObjectId(t["file"])] } };
+      else q = { _id: t["file"] };
+
+      const res = await collections.files.find(q).toArray();
+
+      if (res[0]) {
+        if (!task.fileType) task.fileType = res[0].metadata.fileType;
+
+        if (!task.fileName)
+          task.fileName = res[0].filename || res[0]._id.toString();
+      }
+    }
+  }
+  return [tasks, null];
+}
+
+export async function clearTasks(
+  deviceId: string,
+  taskIds: string[],
+): Promise<void> {
+  await collections.tasks.deleteMany({
+    _id: { $in: taskIds.map((id) => new ObjectId(id)) },
+  });
+}
+
+export async function getOperations(
+  deviceId: string,
+): Promise<{ [commandKey: string]: Operation }> {
+  const res = await collections.operations
+    .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
+    .toArray();
+
+  const operations: { [commandKey: string]: Operation } = {};
+  for (const r of res) {
+    const commandKey = r._id.slice(deviceId.length + 1);
+    // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
+    if (typeof r.provisions !== "string") {
+      delete r._id;
+      operations[commandKey] = r as unknown as Operation;
+      continue;
+    }
+    const operation: Operation = {
+      name: r.name,
+      timestamp: +r.timestamp,
+      channels:
+        typeof r.channels === "string" ? JSON.parse(r.channels) : r.channels,
+      retries: JSON.parse(r.retries),
+      provisions: JSON.parse(r.provisions),
+      ...(r.args && { args: JSON.parse(r.args) }),
+    };
+    operations[commandKey] = operation;
+  }
+  return operations;
+}
+
+export async function saveOperation(
+  deviceId: string,
+  commandKey: string,
+  operation: Operation,
+): Promise<void> {
+  const id = `${deviceId}:${commandKey}`;
+  const o: MongoTypes.Operation = {
+    _id: id,
+    name: operation.name,
+    timestamp: new Date(operation.timestamp),
+    channels: JSON.stringify(operation.channels),
+    provisions: JSON.stringify(operation.provisions),
+    retries: JSON.stringify(operation.retries),
+    args: JSON.stringify(operation.args),
+  };
+  await collections.operations.replaceOne({ _id: id }, o, {
+    upsert: true,
+  });
+}
+
+export async function deleteOperation(
+  deviceId: string,
+  commandKey: string,
+): Promise<void> {
+  await collections.operations.deleteOne({ _id: `${deviceId}:${commandKey}` });
+}

lib/cwmp/local-cache.ts

@@ -0,0 +1,357 @@
+import * as vm from "node:vm";
+import * as crypto from "node:crypto";
+import * as config from "../config.ts";
+import { collections } from "../db/db.ts";
+import { convertOldPrecondition } from "../db/util.ts";
+import * as logger from "../logger.ts";
+import * as scheduling from "../scheduling.ts";
+import * as expression from "../common/expression/util.ts";
+import { parse } from "../common/expression/parser.ts";
+import {
+  Preset,
+  Expression,
+  Provisions,
+  VirtualParameters,
+  Files,
+  Config,
+} from "../types.ts";
+import { LocalCache } from "../local-cache.ts";
+
+interface Snapshot {
+  presets: Preset[];
+  provisions: Provisions;
+  virtualParameters: VirtualParameters;
+  files: Files;
+  config: Config;
+}
+
+function flattenObject<T extends Record<keyof T, unknown>>(
+  src: T,
+  prefix = "",
+  dst = {} as T,
+): T {
+  for (const k of Object.keys(src)) {
+    const v = src[k];
+    if (typeof v === "object" && !Array.isArray(v))
+      flattenObject(v as T, `${prefix}${k}.`, dst);
+    else dst[`${prefix}${k}`] = v;
+  }
+  return dst;
+}
+
+async function fetchPresets(): Promise<[string, Preset[]]> {
+  const res = await collections.presets.find().toArray();
+  let objects = await collections.objects.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  objects.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(res))
+    .update(JSON.stringify(objects))
+    .digest("hex");
+
+  objects = objects.map((obj) => {
+    // Flatten object
+    obj = flattenObject(obj);
+
+    // If no keys are defined, consider all parameters as keys to keep the
+    // same behavior from v1.0
+    if (!obj["_keys"]?.length)
+      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
+
+    return obj;
+  });
+
+  res.sort((a, b) => {
+    if (a["weight"] === b["weight"])
+      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
+    else return a["weight"] - b["weight"];
+  });
+
+  const presets = [] as Preset[];
+  for (const preset of res) {
+    let schedule: { md5: string; duration: number; schedule: any } = null;
+    if (preset["schedule"]) {
+      const parts = preset["schedule"].trim().split(/\s+/);
+      schedule = {
+        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
+        duration: null,
+        schedule: null,
+      };
+
+      try {
+        schedule.duration = +parts.shift() * 1000;
+        schedule.schedule = scheduling.parseCron(parts.join(" "));
+      } catch (err) {
+        logger.warn({
+          message: "Invalid preset schedule",
+          preset: preset["_id"],
+          schedule: preset["schedule"],
+        });
+        schedule.schedule = false;
+      }
+    }
+
+    const events = preset["events"] || {};
+    let precondition = true as Expression;
+    if (preset["precondition"]) {
+      try {
+        precondition = parse(preset["precondition"]);
+      } catch (error) {
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"]),
+        );
+      }
+
+      // Simplify expression
+      precondition = expression.evaluate(precondition);
+    }
+
+    const _provisions: Preset["provisions"] = [];
+
+    // Generate provisions from the old configuration format
+    for (const c of preset.configurations) {
+      switch (c.type) {
+        case "age":
+          _provisions.push(["refresh", c.name, +c.age]);
+          break;
+
+        case "value":
+          _provisions.push(["value", c.name, c.value]);
+          break;
+
+        case "add_tag":
+          _provisions.push(["tag", c.tag, true]);
+          break;
+
+        case "delete_tag":
+          _provisions.push(["tag", c.tag, false]);
+          break;
+
+        case "provision":
+          _provisions.push([c.name, ...(c.args || [])]);
+          break;
+
+        case "add_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 1]);
+
+              for (const k in obj) {
+                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
+                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
+              }
+            }
+          }
+
+          break;
+
+        case "delete_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 0]);
+            }
+          }
+
+          break;
+
+        default: {
+          const exhaustiveCheck: never = c;
+          throw new Error(
+            `Unknown configuration type ${exhaustiveCheck["type"]}`,
+          );
+        }
+      }
+    }
+
+    presets.push({
+      name: preset["_id"],
+      channel: (preset["channel"] as string) || "default",
+      schedule: schedule,
+      events: events,
+      precondition: precondition,
+      provisions: _provisions,
+    });
+  }
+
+  return [h, presets];
+}
+
+async function fetchProvisions(): Promise<[string, Provisions]> {
+  const res = await collections.provisions.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+
+  const provisions = {};
+  for (const r of res) {
+    provisions[r._id] = {};
+    provisions[r._id].md5 = crypto
+      .createHash("md5")
+      .update(r.script)
+      .digest("hex");
+    provisions[r._id].script = new vm.Script(
+      `"use strict";(function(){\n${r.script}\n})();`,
+      { filename: r._id, lineOffset: -1 },
+    );
+  }
+
+  return [h, provisions];
+}
+
+async function fetchVirtualParameters(): Promise<[string, VirtualParameters]> {
+  const res = await collections.virtualParameters.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+
+  const virtualParameters = {};
+  for (const r of res) {
+    virtualParameters[r._id] = {};
+    virtualParameters[r._id].md5 = crypto
+      .createHash("md5")
+      .update(r.script)
+      .digest("hex");
+    virtualParameters[r._id].script = new vm.Script(
+      `"use strict";(function(){\n${r.script}\n})();`,
+      { filename: r._id, lineOffset: -1 },
+    );
+  }
+
+  return [h, virtualParameters];
+}
+
+async function fetchFiles(): Promise<[string, Files]> {
+  const res = await collections.files.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+  const files = {};
+
+  for (const r of res) {
+    const id = r.filename || r._id.toString();
+    files[id] = {};
+    files[id].length = r.length;
+  }
+
+  return [h, files];
+}
+
+async function fetchConfig(): Promise<[string, Config]> {
+  const conf = await collections.config.find().toArray();
+  conf.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(conf)).digest("hex");
+
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    _config[c._id] = expression.evaluate(parse(c.value));
+  }
+
+  return [h, _config];
+}
+
+const localCache = new LocalCache<Snapshot>("cwmp-local-cache-hash", refresh);
+
+async function refresh(): Promise<[string, Snapshot]> {
+  const res = await Promise.all([
+    fetchPresets(),
+    fetchProvisions(),
+    fetchVirtualParameters(),
+    fetchFiles(),
+    fetchConfig(),
+  ]);
+
+  const h = crypto.createHash("md5");
+  for (const r of res) h.update(r[0]);
+
+  const snapshot = {
+    presets: res[0][1],
+    provisions: res[1][1],
+    virtualParameters: res[2][1],
+    files: res[3][1],
+    config: res[4][1],
+  };
+
+  return [h.digest("hex"), snapshot];
+}
+
+export async function getRevision(): Promise<string> {
+  return await localCache.getRevision();
+}
+
+export function getPresets(revision: string): Preset[] {
+  return localCache.get(revision).presets;
+}
+
+export function getProvisions(revision: string): Provisions {
+  return localCache.get(revision).provisions;
+}
+
+export function getVirtualParameters(revision: string): VirtualParameters {
+  return localCache.get(revision).virtualParameters;
+}
+
+export function getFiles(revision: string): Files {
+  return localCache.get(revision).files;
+}
+
+export function getConfig(
+  snapshotKey: string,
+  key: string,
+  context: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Expression,
+): string | number | boolean | null {
+  const snapshot = localCache.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const oldOpts = {
+    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
+    "cwmp.debug": "DEBUG",
+    "cwmp.retryDelay": "RETRY_DELAY",
+    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
+    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
+    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
+    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
+    "cwmp.cookiesPath": "COOKIES_PATH",
+    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
+    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
+    "cwmp.connectionRequestAllowBasicAuth":
+      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
+    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
+    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
+  };
+
+  if (!(key in snapshot.config)) {
+    if (key in oldOpts) {
+      let id;
+      if (context?.["id"]) {
+        id = context["id"];
+      } else if (cb) {
+        id = cb(["PARAM", "DeviceID.ID"]);
+        if (Array.isArray(id)) id = null;
+      }
+      return config.get(oldOpts[key], id);
+    }
+    return null;
+  }
+
+  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+  return Array.isArray(v) ? null : v;
+}
+
+export function getConfigExpression(
+  snapshotKey: string,
+  key: string,
+): Expression {
+  const snapshot = localCache.get(snapshotKey);
+  return snapshot.config[key];
+}

lib/db.coffee

@@ -1,652 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-config = require './config'
-mongodb = require 'mongodb'
-common = require './common'
-
-tasksCollection = null
-devicesCollection = null
-presetsCollection = null
-objectsCollection = null
-provisionsCollection = null
-virtualParametersCollection = null
-faultsCollection = null
-filesCollection = null
-operationsCollection = null
-
-
-connect = (callback) ->
-  callbackCounter = 9
-  mongodb.MongoClient.connect(config.get('MONGODB_CONNECTION_URL'), (err, db) ->
-    return callback(err) if err
-    exports.mongoDb = db
-    db.collection('tasks', (err, collection) ->
-      exports.tasksCollection = tasksCollection = collection
-      collection?.ensureIndex({device: 1, timestamp: 1}, (err) ->
-      )
-
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('devices', (err, collection) ->
-      exports.devicesCollection  = devicesCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('presets', (err, collection) ->
-      exports.presetsCollection = presetsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('objects', (err, collection) ->
-      exports.objectsCollection = objectsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('fs.files', (err, collection) ->
-      exports.filesCollection = filesCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('provisions', (err, collection) ->
-      exports.provisionsCollection = provisionsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('virtualParameters', (err, collection) ->
-      exports.virtualParametersCollection = virtualParametersCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('faults', (err, collection) ->
-      exports.faultsCollection = faultsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('operations', (err, collection) ->
-      exports.operationsCollection = operationsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-  )
-
-
-disconnect = () ->
-  exports.mongoDb?.close()
-
-
-# Optimize projection by removing overlaps
-# This can modify the object
-optimizeProjection = (obj) ->
-  if obj['']
-    return {'': obj['']}
-
-  keys = Object.keys(obj).sort()
-  return obj if keys.length <= 1
-  i = 1
-  while i < keys.length
-    a = keys[i-1]
-    b = keys[i]
-    if b.startsWith(a)
-      if b.charAt(a.length) == '.' or b.charAt(a.length - 1) == '.'
-        delete obj[b]
-        keys.splice(i--, 1)
-    ++ i
-  return obj
-
-
-fetchDevice = (id, timestamp, patterns, callback) ->
-  MAX_DEPTH = config.get('MAX_DEPTH', id)
-
-  if not patterns?.length
-    patterns = [[[], (1 << MAX_DEPTH) - 1]]
-
-  projection = {}
-  projectionTree = {}
-  func = (path, pats, projTree) ->
-    children = {}
-    for pat in pats
-      fragment = pat[0][path.length] or '*'
-
-      if fragment == '*'
-        projection[path.join('.')] = 1 if pat[1] << path.length
-        return
-
-      projTree[fragment] ?= {}
-
-      if pat[1] & (1 << path.length)
-        projection[path.concat('_timestamp').join('.')] = 1
-        projection[path.concat('_object').join('.')] = 1
-        projection[path.concat('_instance').join('.')] = 1
-        projection[path.concat([fragment, '_timestamp']).join('.')] = 1
-        projection[path.concat([fragment, '_value']).join('.')] = 1
-        projection[path.concat([fragment, '_type']).join('.')] = 1
-        projection[path.concat([fragment, '_object']).join('.')] = 1
-        projection[path.concat([fragment, '_instance']).join('.')] = 1
-        projection[path.concat([fragment, '_writable']).join('.')] = 1
-        projection[path.concat([fragment, '_orig']).join('.')] = 1
-
-      if pat[1] >> (path.length + 1)
-        children[fragment] ?= []
-        children[fragment].push(pat)
-
-    for k, v of children
-      func(path.concat(k), v, projTree[k])
-
-  func([], patterns, projectionTree)
-
-  delete projectionTree['DeviceID']
-  delete projectionTree['Events']
-  delete projectionTree['Tags']
-
-  res = []
-  loaded = []
-
-  projection = optimizeProjection(projection)
-
-  for k, v of projection
-    if k == '' or k == 'Events' or k.startsWith('Events.')
-      if k == '' or k == 'Events' or k == 'Events._writable'
-        res.push([['Events'], timestamp,
-          {object: [timestamp, 1], writable: [timestamp, 0]}])
-        loaded.push([['Events'], 1]) if k == 'Events._writable'
-      if k == 'Events'
-        projection['_lastInform'] = 1
-        projection['_lastBoot'] = 1
-        projection['_lastBootstrap'] = 1
-        projection['_registered'] = 1
-        loaded.push([['Events'], ((1 << MAX_DEPTH) - 1)])
-      else if k == 'Events.Inform._writable' or k == 'Events.Inform'
-        projection['_lastInform'] = 1
-        loaded.push([['Events', 'Inform'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'Events.1_BOOT._writable' or k == 'Events.1_BOOT'
-        projection['_lastBoot'] = 1
-        loaded.push([['Events', '1_BOOT'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'Events.0_BOOTSTRAP._writable' or k == 'Events.0_BOOTSTRAP'
-        projection['_lastBootstrap'] = 1
-        loaded.push([['Events', '0_BOOTSTRAP'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'Events.Registered._writable' or k == 'Events.Registered'
-        projection['_registered'] = 1
-        loaded.push([['Events', 'Registered'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k.endsWith('._writable') and k != 'Events._writable'
-        loaded.push([k.split('.').slice(0, 2), 1 ^ ((1 << MAX_DEPTH) - 1)])
-      delete projection[k] if k != ''
-
-    if k == '' or k == 'DeviceID' or k.startsWith('DeviceID.')
-      if k == '' or k == 'DeviceID' or k == 'DeviceID._writable'
-        res.push([['DeviceID'], timestamp,
-          {object: [timestamp, 1], writable: [timestamp, 0]}])
-        loaded.push([['DeviceID'], 1]) if k == 'DeviceID._writable'
-      if k == 'DeviceID'
-        projection['_id'] = 1
-        projection['_deviceId'] = 1
-        loaded.push([['DeviceID'], ((1 << MAX_DEPTH) - 1)])
-      else if k == 'DeviceID.ID._writable' or k == 'DeviceID.ID'
-        projection['_id'] = 1
-        loaded.push([['DeviceID', 'ID'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'DeviceID.Manufacturer._writable' or k == 'DeviceID.DeManufacturer'
-        projection['_deviceId._Manufacturer'] = 1
-        loaded.push([['DeviceID', 'Manufacturer'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'DeviceID.ProductClass._writable' or k == 'DeviceID.ProductClass'
-        projection['_deviceId._ProductClass'] = 1
-        loaded.push([['DeviceID', 'ProductClass'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'DeviceID.OUI._writable' or k == 'DeviceID.OUI'
-        projection['_deviceId._OUI'] = 1
-        loaded.push([['DeviceID', 'ProductClass'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k == 'DeviceID.SerialNumber._writable' or k == 'DeviceID.SerialNumber'
-        projection['_deviceId._SerialNumber'] = 1
-        loaded.push([['DeviceID', 'SerialNumber'], 1 ^ ((1 << MAX_DEPTH) - 1)])
-      else if k.endsWith('._writable') and k != 'DeviceID._writable'
-        loaded.push([k.split('.').slice(0, 2), 1 ^ ((1 << MAX_DEPTH) - 1)])
-      delete projection[k] if k != ''
-
-    if k == 'Tags' or k.startsWith('Tags.')
-      if not projection['_tags']
-        projection['_tags'] = 1
-        loaded.push([['Tags'], ((1 << MAX_DEPTH) - 1)])
-      delete projection[k]
-
-  if projection['']
-    proj = {}
-  else if Object.keys(projection).length == 0
-    proj = {'_id': 1}
-  else
-    proj = projection
-
-  devicesCollection.findOne({'_id' : id}, proj, (err, device) ->
-    return callback(err) if err or not device?
-
-    storeParams = (obj, path, timestamp, descendantsFetched, projTree) ->
-      if descendantsFetched
-        thisFetched = true
-      else
-        if projection[path.join('.')]
-          descendantsFetched = true
-          if path.length and projection[path.slice(0, -1).concat("_timestamp").join('.')]
-            thisFetched = true
-            loaded.push([path, ((1 << (path.length - 1)) - 1) ^ ((1 << MAX_DEPTH) - 1)])
-          else
-            loaded.push([path, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
-        else if projection[path.concat('_writable').join('.')]
-          loaded.push([path, 1 << (path.length - 1)])
-          thisFetched = true
-
-      if obj['_timestamp']?
-        obj['_timestamp'] = +obj['_timestamp']
-
-      # For compatibility with v1.0 database
-      if obj['_instance'] and not obj['_object']?
-        obj['_object'] = true
-
-      if thisFetched
-        attrs = {}
-
-        t = obj['_timestamp'] or 1
-        t = timestamp if timestamp > t
-
-        if obj['_value']?
-          attrs.value = [obj['_timestamp'] or 1, [obj['_value'], obj['_type']]]
-          attrs.value[1][0] = +attrs.value[1][0] if obj['_type'] == 'xsd:dateTime'
-          obj['_object'] = false
-
-        if obj['_writable']?
-          attrs.writable = [timestamp or 1, if obj['_writable'] then 1 else 0]
-
-        if obj['_object']?
-          attrs.object = [t, if obj['_object'] then 1 else 0]
-
-        res.push([path, t, attrs])
-
-      for k, v of obj
-        if not k.startsWith('_')
-          kk = k
-          obj['_object'] = true
-          storeParams(v, path.concat(k), obj['_timestamp'], descendantsFetched, projTree?[k])
-          delete projTree[kk] if projTree
-
-      if not descendantsFetched
-        for k, v of projTree
-          p = path.concat(k)
-          loaded.push([p, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
-          if (obj['_object'] or path.length == 0) and obj['_timestamp']
-            res.push([p, obj['_timestamp']])
-      else if (obj['_object'] or path.length == 0) and obj['_timestamp']
-          res.push([path.concat('*'), obj['_timestamp']])
-
-    for k, v of device
-      switch k
-        when '_lastInform'
-          res.push([['Events', 'Inform'], +v,
-            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_lastBoot'
-          res.push([['Events', '1_BOOT'], +v,
-            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_lastBootstrap'
-          res.push([['Events', '0_BOOTSTRAP'], +v,
-            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_registered'
-          # Use current timestamp for registered event attribute timestamps
-          res.push([['Events', 'Registered'], timestamp,
-            {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_id'
-          if projection[''] or projection['_id']
-            res.push([['DeviceID', 'ID'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v, 'xsd:string']]}])
-          delete device[k]
-        when '_tags'
-          if v.length
-            res.push([['Tags'], timestamp,
-              {object: [timestamp, 1], writable: [timestamp, 0]}])
-          for t in v
-            t = t.replace(/[^a-zA-Z0-9\-]+/g, '_')
-            res.push([['Tags', t], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 1], value: [timestamp, [true, 'xsd:boolean']]}])
-          delete device[k]
-        when '_deviceId'
-          if v['_Manufacturer']?
-            res.push([['DeviceID', 'Manufacturer'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_Manufacturer'], 'xsd:string']]}])
-          if v['_OUI']?
-            res.push([['DeviceID', 'OUI'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_OUI'], 'xsd:string']]}])
-          if v['_ProductClass']?
-            res.push([['DeviceID', 'ProductClass'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_ProductClass'], 'xsd:string']]}])
-          if v['_SerialNumber']?
-            res.push([['DeviceID', 'SerialNumber'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_SerialNumber'], 'xsd:string']]}])
-          delete device[k]
-
-    storeParams(device, [], 0, false, projectionTree)
-
-    return callback(null, res, loaded)
-  )
-
-
-saveDevice = (deviceId, deviceData, isNew, sessionTimestamp, callback) ->
-  update = {'$set' : {}, '$unset' : {}, '$addToSet' : {}, '$pull' : {}}
-
-  iter = deviceData.timestamps.diff()
-  while diff = iter.next().value
-    continue if diff[0].wildcard != (1 << (diff[0].length - 1))
-    continue if diff[0][0] in ['Events', 'DeviceID', 'Tags']
-
-    # Param timestamps may be greater than session timestamp to track revisions
-    if diff[2] > sessionTimestamp
-      diff[2] = sessionTimestamp
-
-    if not diff[2]? and diff[1]?
-      update['$unset'][diff[0].slice(0, -1).concat('_timestamp').join('.')] = 1
-    else if diff[2] != diff[1]
-      parent = deviceData.paths.get(diff[0].slice(0, -1))
-      if parent and (parent.length == 0 or deviceData.attributes.has(parent))
-        update['$set'][diff[0].slice(0, -1).concat('_timestamp').join('.')] = new Date(diff[2])
-
-  iter = deviceData.attributes.diff()
-  while diff = iter.next().value
-    continue if diff[1] == diff[2]
-    path = diff[0]
-    switch path[0]
-      when 'Events'
-        if diff[0].length == 2 and diff[2]?.value?[1][0] != diff[1]?.value?[1][0]
-          if not diff[2]
-            switch path[1]
-              when 'Inform'
-                update['$unset']['_lastInform'] = 1
-              when '1_BOOT'
-                update['$unset']['_lastBoot'] = 1
-              when '0_BOOTSTRAP'
-                update['$unset']['_lastBootstrap'] = 1
-              when 'Registered'
-                update['$unset']['_registered'] = 1
-          else
-            t = new Date(diff[2].value[1][0])
-            switch path[1]
-              when 'Inform'
-                update['$set']['_lastInform'] = t
-              when '1_BOOT'
-                update['$set']['_lastBoot'] = t
-              when '0_BOOTSTRAP'
-                update['$set']['_lastBootstrap'] = t
-              when 'Registered'
-                update['$set']['_registered'] = t
-      when 'DeviceID'
-        if diff[2].value?[1]?[0] != diff[1]?.value?[1]?[0]
-          v = diff[2].value[1][0]
-          switch path[1]
-            when 'ID'
-              update['$set']['_id'] = v
-            when 'Manufacturer'
-              update['$set']['_deviceId._Manufacturer'] = v
-            when 'OUI'
-              update['$set']['_deviceId._OUI'] = v
-            when 'ProductClass'
-              update['$set']['_deviceId._ProductClass'] = v
-            when 'SerialNumber'
-              update['$set']['_deviceId._SerialNumber'] = v
-      when 'Tags'
-        if diff[2]?.value?[1]?[0] != diff[1]?.value?[1]?[0]
-          v = diff[2]?.value?[1][0]
-          if v?
-            update['$addToSet']['_tags'] ?= {'$each' : []}
-            update['$addToSet']['_tags']['$each'].push(diff[0][1])
-          else
-            update['$pull']['_tags'] ?= {'$in' : []}
-            update['$pull']['_tags']['$in'].push(diff[0][1])
-      else
-        if not diff[2]
-          update['$unset'][diff[0].join('.')] = 1
-          continue
-
-        for attrName of diff[2]
-          # Param timestamps may be greater than session timestamp to track revisions
-          if diff[2][attrName][0] > sessionTimestamp
-            diff[2][attrName][0] = sessionTimestamp
-
-          if diff[2][attrName][1]?
-            switch attrName
-              when 'value'
-                if diff[2].value[1][0] != diff[1]?.value?[1]?[0]
-                  if diff[2].value[1][1] == 'xsd:dateTime' and Number.isInteger(diff[2].value[1][0])
-                    update['$set'][path.concat('_value').join('.')] = new Date(diff[2].value[1][0])
-                  else
-                    update['$set'][path.concat('_value').join('.')] = diff[2].value[1][0]
-                if diff[2].value[1][1] != diff[1]?.value?[1]?[1]
-                  update['$set'][path.concat('_type').join('.')] = diff[2].value[1][1]
-                if diff[2].value[0] != diff[1]?.value?[0]
-                  update['$set'][path.concat('_timestamp').join('.')] = new Date(diff[2].value[0])
-              when 'object'
-                if not diff[1]?.object or diff[2].object[1] != diff[1].object?[1]
-                  update['$set'][path.concat('_object').join('.')] = !!diff[2].object[1]
-              when 'writable'
-                if not diff[1]?.writable or diff[2].writable[1] != diff[1].writable?[1]
-                  update['$set'][path.concat('_writable').join('.')] = !!diff[2].writable[1]
-
-        if diff[1]
-          for attrName of diff[1]
-            if diff[1][attrName][1]? and not diff[2]?[attrName]?[1]?
-              update['$unset'][path.concat("_#{attrName}").join('.')] = 1
-              if attrName is 'value'
-                update['$unset'][path.concat('_type').join('.')] = 1
-                update['$unset'][path.concat('_timestamp').join('.')] = 1
-
-  update['$unset'] = optimizeProjection(update['$unset'])
-
-  # Remove overlap possibly caused by parameters changing from objects
-  # to regular parameters or vice versa. Reason being that _timestamp
-  # represents two different things depending on whether the parameter
-  # is an object or not.
-  for k of update['$unset']
-    delete update['$unset'][k] if update['$set'][k]?
-
-  # Remove empty keys
-  for k of update
-    if k == '$addToSet'
-      for kk of update[k]
-        delete update[k][kk] if update[k][kk]['$each'].length == 0
-    else if k == '$pull'
-      for kk of update[k]
-        delete update[k][kk] if update[k][kk]['$in'].length == 0
-    delete update[k] if Object.keys(update[k]).length == 0
-
-  return callback() if Object.keys(update).length == 0
-
-  if update['$addToSet'] and update['$pull']
-    # Mongo doesn't allow $addToSet and $pull at the same time
-    update2 = {'$pull': update['$pull']}
-    delete update['$pull']
-
-  devicesCollection.update({'_id' : deviceId}, update, {upsert: isNew}, (err, result) ->
-    if not err and result.result.n != 1
-      return callback(new Error("Device #{deviceId} not found in database"))
-
-    if update2
-      return devicesCollection.update({'_id': deviceId}, update2, callback)
-
-    return callback(err)
-  )
-
-
-getFaults = (deviceId, callback) ->
-  faultsCollection.find({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}).toArray((err, res) ->
-    return callback(err) if err
-
-    faults = {}
-    for r in res
-      channel = r._id.slice(deviceId.length + 1)
-      delete r._id
-      delete r.channel
-      delete r.device
-      r.timestamp = +r.timestamp
-      r.provisions = JSON.parse(r.provisions)
-      faults[channel] = r
-
-    callback(err, faults)
-  )
-
-
-saveFault = (deviceId, channel, fault, callback) ->
-  fault = Object.assign({}, fault)
-  fault._id = "#{deviceId}:#{channel}"
-  fault.device = deviceId
-  fault.channel = channel
-  fault.timestamp = new Date(fault.timestamp)
-  fault.provisions = JSON.stringify(fault.provisions)
-  faultsCollection.save(fault, callback)
-
-
-deleteFault = (deviceId, channel, callback) ->
-  faultsCollection.remove({_id: "#{deviceId}:#{channel}"}, callback)
-
-
-getDueTasks = (deviceId, timestamp, callback) ->
-  cur = tasksCollection.find({'device' : deviceId}).sort(['timestamp'])
-
-  tasks = []
-
-  cur.nextObject(f = (err, task) ->
-    return callback(err) if err
-
-    if not task?
-      return callback(null, tasks, null)
-
-    task.timestamp = +task.timestamp if task.timestamp?
-    task.expiry = +task.expiry if task.expiry?
-
-    if task.timestamp >= timestamp
-      return callback(null, tasks, +task.timestamp)
-
-    task._id = String(task._id)
-    tasks.push(task)
-
-    # For API compatibility
-    if task.name is 'download' and task.file?
-      if mongodb.ObjectID.isValid(task.file)
-        q = {_id: {'$in' : [task.file, new mongodb.ObjectID(task.file)]}}
-      else
-        q = {_id: task.file}
-
-      filesCollection.find(q).toArray((err, res) ->
-        return callback(err) if err
-        if res[0]?
-          task.fileType ?= res[0].metadata.fileType
-          task.fileName ?= res[0].filename or res[0]._id.toString()
-        cur.nextObject(f)
-      )
-    else
-      cur.nextObject(f)
-  )
-
-
-clearTasks = (deviceId, taskIds, callback) ->
-  tasksCollection.remove({'_id' : {'$in' : (new mongodb.ObjectID(id) for id in taskIds)}}, callback)
-
-
-getOperations = (deviceId, callback) ->
-  operationsCollection.find({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}).toArray((err, res) ->
-    return callback(err) if err
-
-    operations = {}
-    for r in res
-      commandKey = r._id.slice(deviceId.length + 1)
-      delete r._id
-      r.timestamp = +r.timestamp
-      r.args = JSON.parse(r.args) if r.args
-      r.provisions = JSON.parse(r.provisions)
-      r.retries = JSON.parse(r.retries)
-      operations[commandKey] = r
-
-    callback(err, operations)
-  )
-
-
-saveOperation = (deviceId, commandKey, operation, callback) ->
-  operation = Object.assign({}, operation)
-  operation._id = "#{deviceId}:#{commandKey}"
-  operation.timestamp = new Date(operation.timestamp)
-  operation.provisions = JSON.stringify(operation.provisions)
-  operation.retries = JSON.stringify(operation.retries)
-  operation.args = JSON.stringify(operation.args)
-  operationsCollection.save(operation, callback)
-
-
-deleteOperation = (deviceId, commandKey, callback) ->
-  operationsCollection.remove({_id: "#{deviceId}:#{commandKey}"}, callback)
-
-
-getPresets = (callback) ->
-  presetsCollection.find().toArray(callback)
-
-
-getObjects = (callback) ->
-  objectsCollection.find().toArray(callback)
-
-
-getProvisions = (callback) ->
-  provisionsCollection.find().toArray(callback)
-
-
-getVirtualParameters = (callback) ->
-  virtualParametersCollection.find().toArray(callback)
-
-
-getFiles = (callback) ->
-  filesCollection.find().toArray(callback)
-
-
-exports.connect = connect
-exports.disconnect = disconnect
-exports.fetchDevice = fetchDevice
-exports.saveDevice = saveDevice
-exports.getFaults = getFaults
-exports.saveFault = saveFault
-exports.deleteFault = deleteFault
-exports.clearTasks = clearTasks
-exports.saveOperation = saveOperation
-exports.deleteOperation = deleteOperation
-exports.getPresets = getPresets
-exports.getObjects = getObjects
-exports.getProvisions = getProvisions
-exports.getVirtualParameters = getVirtualParameters
-exports.getFiles = getFiles
-exports.getDueTasks = getDueTasks
-exports.getOperations = getOperations

lib/db/db.ts

@@ -0,0 +1,57 @@
+import { MongoClient, Collection, GridFSBucket } from "mongodb";
+import { get } from "../config.ts";
+import * as MongoTypes from "./types.ts";
+
+export let filesBucket: GridFSBucket;
+
+export const collections = {
+  devices: null as Collection<MongoTypes.Device>,
+  presets: null as Collection<MongoTypes.Preset>,
+  objects: null as Collection<MongoTypes.Object>,
+  provisions: null as Collection<MongoTypes.Provision>,
+  virtualParameters: null as Collection<MongoTypes.VirtualParameter>,
+  faults: null as Collection<MongoTypes.Fault>,
+  tasks: null as Collection<MongoTypes.Task>,
+  files: null as Collection<MongoTypes.File>,
+  operations: null as Collection<MongoTypes.Operation>,
+  permissions: null as Collection<MongoTypes.Permission>,
+  users: null as Collection<MongoTypes.User>,
+  config: null as Collection<MongoTypes.Config>,
+  cache: null as Collection<MongoTypes.Cache>,
+  locks: null as Collection<MongoTypes.Lock>,
+};
+
+let clientPromise: Promise<MongoClient>;
+
+export async function connect(): Promise<void> {
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"));
+
+  const client = await clientPromise;
+  const db = client.db();
+
+  collections.tasks = db.collection("tasks");
+  collections.devices = db.collection("devices");
+  collections.presets = db.collection("presets");
+  collections.objects = db.collection("objects");
+  collections.files = db.collection("fs.files");
+  collections.provisions = db.collection("provisions");
+  collections.virtualParameters = db.collection("virtualParameters");
+  collections.faults = db.collection("faults");
+  collections.operations = db.collection("operations");
+  collections.permissions = db.collection("permissions");
+  collections.users = db.collection("users");
+  collections.config = db.collection("config");
+  collections.cache = db.collection("cache");
+  collections.locks = db.collection("locks");
+  filesBucket = new GridFSBucket(db);
+
+  await Promise.all([
+    collections.tasks.createIndex({ device: 1, timestamp: 1 }),
+    collections.cache.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }),
+    collections.locks.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }),
+  ]);
+}
+
+export async function disconnect(): Promise<void> {
+  if (clientPromise != null) await (await clientPromise).close();
+}

lib/db/synth.ts

@@ -0,0 +1,654 @@
+import { Filter } from "mongodb";
+import { EJSON } from "bson";
+import { parseLikePattern } from "../common/expression/parser.ts";
+import { Expression } from "../types.ts";
+import { decodeTag } from "../util.ts";
+import {
+  SynthContextBase,
+  likeDisjoint,
+  likeImplies,
+  Clause,
+} from "../common/expression/synth.ts";
+import normalize from "../common/expression/normalize.ts";
+import * as BI from "../common/expression/bigint.ts";
+
+type Minterm = number[];
+
+function getParam(exp: Expression, collection: string): string {
+  if (!Array.isArray(exp))
+    throw new Error("Left-hand operand must be a parameter");
+
+  if (exp[0] !== "PARAM")
+    throw new Error("Left-hand operand must be a parameter");
+
+  if (typeof exp[1] !== "string")
+    throw new Error("Left-hand operand must be a parameter");
+
+  const p = exp[1];
+  if (collection === "devices") {
+    if (p === "DeviceID.ID") return "_id";
+    else if (p === "DeviceID") return "_deviceId";
+    else if (p.startsWith("DeviceID.")) return "_deviceId._" + p.slice(9);
+    else if (p === "Events.Inform") return "_lastInform";
+    else if (p === "Events.Registered") return "_registered";
+    else if (p === "Events.0_BOOTSTRAP") return "_lastBootstrap";
+    else if (p === "Events.1_BOOT") return "_lastBoot";
+    else if (!p.endsWith("._value") && !p.startsWith("Tags."))
+      return `${p}._value`;
+  }
+
+  return p;
+}
+
+function getTypes(parameter: string, collection: string): string[] {
+  if (collection === "devices") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "_lastInform") return ["date"];
+    if (parameter === "_registered") return ["date"];
+    if (parameter === "_lastBootstrap") return ["date"];
+    if (parameter === "_lastBoot") return ["date"];
+    if (parameter.startsWith("_deviceId.")) return ["string"];
+    if (parameter === "Reboot._value") return ["date"];
+    if (parameter === "FactoryReset._value") return ["date"];
+    if (parameter.endsWith("_timestamp")) return ["date"];
+    if (parameter.startsWith("Downloads.")) {
+      if (parameter.endsWith("Download._value")) return ["date"];
+      if (parameter.endsWith("Time._value")) return ["date"];
+      if (parameter.endsWith("Name._value")) return ["string"];
+      if (parameter.endsWith("Type._value")) return ["string"];
+    }
+    if (parameter.endsWith("_value"))
+      return ["bool", "number", "date", "string"];
+  } else if (collection === "tasks") {
+    if (parameter === "_id") return ["oid"];
+    if (parameter === "timestamp") return ["date"];
+    if (parameter === "expiry") return ["date"];
+    if (parameter === "name") return ["string"];
+    if (parameter === "device") return ["string"];
+  } else if (collection === "faults") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "timestamp") return ["date"];
+    if (parameter === "expiry") return ["date"];
+    if (parameter === "code") return ["string"];
+    if (parameter === "retries") return ["number"];
+    if (parameter === "channel") return ["string"];
+    if (parameter === "device") return ["string"];
+    if (parameter === "message") return ["string"];
+  } else if (collection === "users") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "roles") return ["string"];
+    else if (parameter === "password")
+      throw new Error("Cannot query restricted parameters");
+    else if (parameter === "salt")
+      throw new Error("Cannot query restricted parameters");
+  } else if (collection === "config") {
+    if (parameter === "_id") return ["string"];
+    else if (parameter === "value") return ["string"];
+  } else if (collection === "files") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "metadata.fileType") return ["string"];
+    if (parameter === "metadata.oui") return ["string"];
+    if (parameter === "metadata.productClass") return ["string"];
+    if (parameter === "metadata.version") return ["string"];
+  } else if (collection === "permissions") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "role") return ["string"];
+    if (parameter === "resource") return ["devices"];
+    if (parameter === "access") return ["number"];
+    if (parameter === "filter") return ["string"];
+    if (parameter === "validate") return ["string"];
+  } else if (collection === "presets") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "weight") return ["number"];
+    if (parameter === "channel") return ["string"];
+    if (parameter === "precondition") return ["string"];
+    if (parameter.startsWith("events.")) return ["bool"];
+  } else if (collection === "provisions") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "script") return ["string"];
+  } else if (collection === "virtualParameters") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "script") return ["string"];
+  }
+
+  if (parameter === "_id") return ["oid", "string"];
+  return ["bool", "number", "date", "string"];
+}
+
+function roundOid(oid: string, roundUp: boolean): string {
+  const match = (oid.match(/^[0-9a-f]*/)?.[0] ?? "").slice(0, 24);
+  let num = BI.BigInt("0x0" + match);
+  let lastChar = 0;
+  if (oid.length > match.length) lastChar += oid.charCodeAt(match.length);
+  if (match.length < 24) lastChar -= 48;
+  if (lastChar > 0 && roundUp) num++;
+  num <<= BI.BigInt(4 * (24 - match.length));
+  if (lastChar < 0 && !roundUp) --num;
+  const str = num.toString(16);
+  if (str.length > 24 || str.startsWith("-")) return null;
+  return str.padStart(24, "0");
+}
+
+function groupBy<T, K>(
+  input: T[],
+  callback: (item: T) => K,
+): Iterable<[K, T[]]> {
+  const groups = new Map<K, T[]>();
+  for (const item of input) {
+    const key = callback(item);
+    let arr = groups.get(key);
+    if (!arr) groups.set(key, (arr = []));
+    arr.push(item);
+  }
+
+  return groups.entries();
+}
+
+abstract class MongoClause {
+  abstract readonly parameter: string;
+  abstract toQuery(truthy: boolean): Filter<unknown>;
+  toString(): string {
+    return JSON.stringify(this.toQuery(true));
+  }
+}
+
+class MongoClauseArray extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly value: string,
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    if (truthy) return { [this.parameter]: { $eq: this.value } };
+    else return { [this.parameter]: { $ne: this.value } };
+  }
+}
+
+class MongoClauseCompare<T> extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly op: "$eq" | "$gt" | "$lt" | "$gte" | "$lte",
+    public readonly value: T,
+    public readonly type: "" | "bool" | "number" | "string" | "date" | "oid",
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    let v: any = this.value;
+    if (this.type === "date" && typeof v === "number")
+      v = { $date: new Date(v).toISOString() };
+    if (this.type === "oid" && typeof v === "string") v = { $oid: v };
+
+    if (truthy) return { [this.parameter]: { [this.op]: v } };
+    else if (this.op === "$eq") return { [this.parameter]: { $ne: v } };
+    else return { [this.parameter]: { $not: { [this.op]: v } } };
+  }
+}
+
+class MongoClauseType extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly type: string,
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    if (truthy) return { [this.parameter]: { $type: this.type } };
+    else return { [this.parameter]: { $not: { $type: this.type } } };
+  }
+}
+
+class MongoClauseLike extends MongoClause {
+  readonly pattern: string[];
+  constructor(
+    public readonly parameter: string,
+    pat: string,
+    esc: string,
+    public readonly caseSensitive: boolean,
+  ) {
+    super();
+    this.pattern = parseLikePattern(pat, esc);
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    const convChars = {
+      "-": "\\-",
+      "/": "\\/",
+      "\\": "\\/",
+      "^": "\\^",
+      $: "\\$",
+      "*": "\\*",
+      "+": "\\+",
+      "?": "\\?",
+      ".": "\\.",
+      "(": "\\(",
+      ")": "\\)",
+      "|": "\\|",
+      "[": "\\[",
+      "]": "\\]",
+      "{": "\\{",
+      "}": "\\}",
+      "\\%": ".*",
+      "\\_": ".",
+    };
+
+    const chars = this.pattern.map((c) => convChars[c] ?? c);
+    chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
+    const l = chars.length - 1;
+    chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
+    const pattern = chars.join("");
+    const options = this.caseSensitive ? "s" : "is";
+    if (truthy) {
+      return { [this.parameter]: { $regularExpression: { options, pattern } } };
+    } else {
+      return {
+        [this.parameter]: {
+          $not: { $regularExpression: { options, pattern } },
+        },
+      };
+    }
+  }
+}
+
+class MongoSynthContext extends SynthContextBase<Clause, MongoClause> {
+  constructor(private readonly collection: string) {
+    super();
+  }
+
+  getMinterms(clause: Clause, res: true | false | null): number[][] {
+    const exp = clause.expression();
+    if (!Array.isArray(exp)) throw new Error("Invalid query expression");
+
+    if (res === null) {
+      const minterms: number[][] = [];
+      for (const dep of clause.getNullables()) {
+        const e = dep.operand.expression();
+        const param = getParam(e, this.collection);
+        if (param.startsWith("Tags.") && this.collection === "devices") {
+          const t = decodeTag(param.slice(5));
+          const c = new MongoClauseArray("_tags", t);
+          minterms.push([this.getVar(c) << 1]);
+          continue;
+        }
+        const c = new MongoClauseCompare(param, "$eq", null, "");
+        minterms.push([(this.getVar(c) << 1) ^ 1]);
+      }
+      return minterms;
+    }
+
+    if ([">", "<", "="].includes(exp[0])) {
+      const param = getParam(exp[1], this.collection);
+
+      let rhs = exp[2];
+      if (typeof rhs === "boolean") rhs = +rhs;
+      else if (typeof rhs !== "string" && typeof rhs !== "number")
+        throw new Error(`Right-hand operand must be a literal value`);
+
+      if (
+        param.startsWith("Tags.") &&
+        this.collection === "devices" &&
+        exp[0] === "="
+      ) {
+        const t = decodeTag(param.slice(5));
+        const c = new MongoClauseArray("_tags", t);
+        if (typeof rhs === "string") rhs = 2;
+        if (exp[0] === "=") {
+          if ((rhs === 1) !== res) return [];
+        } else if (exp[0] === ">") {
+          if ((res && rhs >= 1) || (!res && rhs < 1)) return [];
+        } else if (exp[0] === "<") {
+          if ((res && rhs <= 1) || (!res && rhs > 1)) return [];
+        }
+        return [[(this.getVar(c) << 1) ^ 1]];
+      }
+
+      let op: "$gt" | "$lt" | "$eq" | "$gte" | "$lte";
+      if (exp[0] === "=") op = "$eq";
+      else if (exp[0] === ">") op = res ? "$gt" : "$lte";
+      else if (exp[0] === "<") op = res ? "$lt" : "$gte";
+
+      const possibleTypes = new Set(getTypes(param, this.collection));
+      const clauses: MongoClause[] = [];
+
+      if (typeof rhs === "number") {
+        if (possibleTypes.has("number"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "number"));
+
+        if (possibleTypes.has("date"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "date"));
+        if (possibleTypes.has("bool") && (rhs === 0 || rhs === 1))
+          clauses.push(new MongoClauseCompare(param, op, !!rhs, "bool"));
+      } else if (typeof rhs === "string") {
+        if (possibleTypes.has("string"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "string"));
+
+        if (possibleTypes.has("oid")) {
+          const oid = roundOid(rhs, op === "$lt" || op === "$lte");
+          if (oid && (op !== "$eq" || oid === rhs))
+            clauses.push(new MongoClauseCompare(param, op, oid, "oid"));
+        }
+      }
+
+      if (op === "$eq" && !res) {
+        clauses.push(new MongoClauseCompare(param, "$eq", null, ""));
+        return [clauses.map((c) => this.getVar(c) << 1)];
+      }
+
+      // In the following clauses we could use $type operator, but we want the
+      // final query to use comparison operators to check for type because the
+      // $type operator in MongoDB doesn't use indexes
+      if (typeof rhs === "number") {
+        if (possibleTypes.has("bool")) {
+          if (
+            (rhs > 1 && (op === "$lt" || op === "$lte")) ||
+            (rhs < 0 && (op === "$gt" || op === "$gte"))
+          )
+            clauses.push(new MongoClauseCompare(param, "$gte", false, "bool"));
+        }
+
+        if (op === "$gt" || op === "$gte") {
+          if (possibleTypes.has("string"))
+            clauses.push(new MongoClauseCompare(param, "$gte", "", "string"));
+
+          if (possibleTypes.has("oid")) {
+            clauses.push(
+              new MongoClauseCompare(
+                param,
+                "$gte",
+                "000000000000000000000000",
+                "oid",
+              ),
+            );
+          }
+        }
+      } else if (typeof rhs === "string") {
+        if (op === "$lt" || op === "$lte") {
+          if (possibleTypes.has("bool"))
+            clauses.push(new MongoClauseCompare(param, "$gte", false, "bool"));
+
+          if (possibleTypes.has("number")) {
+            clauses.push(new MongoClauseCompare(param, "$gte", 0, "number"));
+            clauses.push(new MongoClauseCompare(param, "$lt", 0, "number"));
+          }
+          if (possibleTypes.has("date")) {
+            clauses.push(new MongoClauseCompare(param, "$gte", 0, "date"));
+            clauses.push(new MongoClauseCompare(param, "$lt", 0, "date"));
+          }
+        }
+      }
+
+      return clauses.map((c) => [(this.getVar(c) << 1) ^ 1]);
+    } else if (exp[0] === "LIKE") {
+      const pat = exp[2];
+      if (typeof pat !== "string")
+        throw new Error("Right-hand operand of 'LIKE' must be a string");
+      let p = exp[1];
+      let caseSensitive = true;
+      if (
+        Array.isArray(p) &&
+        p[0] === "FUNC" &&
+        ["UPPER", "LOWER"].includes(p[1])
+      ) {
+        if (p[1] === "UPPER" && pat !== pat.toUpperCase())
+          return res ? [] : [[]];
+        if (p[1] === "LOWER" && pat !== pat.toLowerCase())
+          return res ? [] : [[]];
+        caseSensitive = false;
+        p = p[2];
+      }
+
+      const param = getParam(p, this.collection);
+      const c = new MongoClauseLike(param, pat, exp[3], caseSensitive);
+      if (res) return [[(this.getVar(c) << 1) ^ 1]];
+      const typeClase = new MongoClauseType(param, "string");
+      const r = [[this.getVar(c) << 1, (this.getVar(typeClase) << 1) ^ 1]];
+      return r;
+    } else {
+      throw new Error("Invalid query expression");
+    }
+  }
+
+  getDcSet(minterms: Minterm[]): number[][] {
+    const dcSet: number[][] = [];
+
+    const vars = new Set(minterms.flat().map((l) => l >> 1));
+    const clauses = Array.from(vars).map((v) => this.getClause(v));
+
+    for (const [parameter, clauses2] of groupBy(clauses, (c) => c.parameter)) {
+      const comparisons = clauses2.filter(
+        (c) => c instanceof MongoClauseCompare && c.value !== null,
+      ) as MongoClauseCompare<unknown>[];
+
+      for (const [type, clauses3] of groupBy(comparisons, (c) => c.type)) {
+        const isType = this.getVar(new MongoClauseType(parameter, type));
+        const values = new Set(clauses3.map((c) => c.value));
+        const valuesSorted = Array.from(values).sort((a, b) =>
+          a > b ? 1 : -1,
+        );
+        for (const [i, v] of valuesSorted.entries()) {
+          const eq = this.getVar(
+            new MongoClauseCompare(parameter, "$eq", v, type),
+          );
+          const gt = this.getVar(
+            new MongoClauseCompare(parameter, "$gt", v, type),
+          );
+          const lt = this.getVar(
+            new MongoClauseCompare(parameter, "$lt", v, type),
+          );
+          const gte = this.getVar(
+            new MongoClauseCompare(parameter, "$gte", v, type),
+          );
+          const lte = this.getVar(
+            new MongoClauseCompare(parameter, "$lte", v, type),
+          );
+
+          if (type === "bool") {
+            if (v === false) dcSet.push([(lt << 1) ^ 1]);
+            else if (v === true) dcSet.push([(gt << 1) ^ 1]);
+          } else if (type === "string") {
+            if (v === "") dcSet.push([(lt << 1) ^ 1]);
+          } else if (type === "oid") {
+            if (v < "000000000000000000000000") dcSet.push([(lte << 1) ^ 1]);
+            else if (v === "000000000000000000000000")
+              dcSet.push([(lt << 1) ^ 1]);
+            else if (v > "ffffffffffffffffffffffff")
+              dcSet.push([(gte << 1) ^ 1]);
+            else if (v === "ffffffffffffffffffffffff")
+              dcSet.push([(gt << 1) ^ 1]);
+          }
+          dcSet.push([(lt << 1) ^ 1, (gte << 1) ^ 1]);
+          dcSet.push([(gt << 1) ^ 1, (gte << 1) ^ 0]);
+          dcSet.push([(eq << 1) ^ 0, (lt << 1) ^ 0, (lte << 1) ^ 1]);
+          dcSet.push([(eq << 1) ^ 1, (gte << 1) ^ 0]);
+          dcSet.push([(eq << 1) ^ 1, (gt << 1) ^ 1]);
+
+          if (i === 0) {
+            dcSet.push([(isType << 1) ^ 0, (gte << 1) ^ 1]);
+            dcSet.push([(isType << 1) ^ 0, (lt << 1) ^ 1]);
+          } else {
+            const gt2 = this.getVar(
+              new MongoClauseCompare(
+                parameter,
+                "$gt",
+                valuesSorted[i - 1],
+                type,
+              ),
+            );
+            const lte2 = this.getVar(
+              new MongoClauseCompare(
+                parameter,
+                "$lte",
+                valuesSorted[i - 1],
+                type,
+              ),
+            );
+            dcSet.push([(gt2 << 1) ^ 0, (gte << 1) ^ 1]);
+            dcSet.push([(gt2 << 1) ^ 0, (lte2 << 1) ^ 0, (lt << 1) ^ 1]);
+          }
+
+          if (i === valuesSorted.length - 1)
+            dcSet.push([(isType << 1) ^ 1, (gt << 1) ^ 0, (lte << 1) ^ 0]);
+        }
+      }
+
+      const likes = clauses2.filter(
+        (c) => c instanceof MongoClauseLike,
+      ) as MongoClauseLike[];
+      if (likes.length) {
+        const isType = this.getVar(new MongoClauseType(parameter, "string"));
+        for (let i1 = 0; i1 < likes.length; ++i1) {
+          const l1 = likes[i1];
+          let p1 = l1.pattern;
+          dcSet.push([(this.getVar(l1) << 1) ^ 1, (isType << 1) ^ 0]);
+          for (let i2 = i1 + 1; i2 < likes.length; ++i2) {
+            const l2 = likes[i2];
+            let p2 = l2.pattern;
+            if (!l1.caseSensitive || !l2.caseSensitive) {
+              p1 = p1.map((c) => c.toLowerCase());
+              p2 = p2.map((c) => c.toLowerCase());
+            }
+            if (likeDisjoint(p1, p2)) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 1,
+                (this.getVar(l2) << 1) ^ 1,
+              ]);
+            } else if (
+              (!l1.caseSensitive || l2.caseSensitive) &&
+              likeImplies(p1, p2)
+            ) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 0,
+                (this.getVar(l2) << 1) ^ 1,
+              ]);
+            } else if (
+              (!l2.caseSensitive || l1.caseSensitive) &&
+              likeImplies(p2, p1)
+            ) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 1,
+                (this.getVar(l2) << 1) ^ 0,
+              ]);
+            }
+          }
+        }
+      }
+
+      const isNull = this.getVar(
+        new MongoClauseCompare(parameter, "$eq", null, ""),
+      );
+
+      const types = getTypes(parameter, this.collection).map((t) =>
+        this.getVar(new MongoClauseType(parameter, t)),
+      );
+
+      dcSet.push([(isNull << 1) ^ 0, ...types.map((t) => (t << 1) ^ 0)]);
+      for (let i = 0; i < types.length; ++i) {
+        const t1 = types[i];
+        dcSet.push([(t1 << 1) ^ 1, (isNull << 1) ^ 1]);
+        for (let j = i + 1; j < types.length; ++j) {
+          const t2 = types[j];
+          dcSet.push([(t1 << 1) ^ 1, (t2 << 1) ^ 1]);
+        }
+      }
+
+      if (parameter === "_id") dcSet.push([(isNull << 1) ^ 1]);
+    }
+    return dcSet;
+  }
+
+  canRaise(i: number, s: Set<number>): boolean {
+    if (!(i & 1)) return true;
+    const c = this.getClause(i >> 1);
+    if (c instanceof MongoClauseCompare) {
+      for (const j of s) {
+        if (j === i) continue;
+        const c2 = this.getClause(j >> 1);
+        if (c2.parameter !== c.parameter) continue;
+        if (!(j & 1)) return false;
+        if (c2 instanceof MongoClauseType) return false;
+      }
+    }
+    return true;
+  }
+
+  toQuery(minterms: Minterm[]): Filter<unknown> {
+    const or = [];
+
+    const ejsonOps = ["$oid", "$date", "$regularExpression"];
+
+    for (const minterm of minterms) {
+      const query = {};
+      loop: for (const clause of minterm) {
+        if (!minterm.length) return {};
+        const negate = !!(clause & 1);
+        const c = this.getClause(clause >> 1);
+        const q = c.toQuery(negate);
+        if (Object.keys(q).length !== 1)
+          throw new Error("Invalid query expression");
+        const [param, value] = Object.entries(q)[0];
+        const dests = [query, ...(query["$and"] ?? [])];
+        for (const dest of dests) {
+          if (!(param in dest)) {
+            dest[param] = value;
+            continue loop;
+          }
+
+          let src = value;
+          let dst = dest[param];
+          if (Object.getPrototypeOf(src).constructor !== Object) continue;
+          if (Object.getPrototypeOf(dst).constructor !== Object) continue;
+          let srcKeys = Object.keys(src);
+          let dstKeys = Object.keys(dst);
+
+          if ([...srcKeys, ...dstKeys].every((k) => k === "$not")) {
+            src = src["$not"];
+            dst = dst["$not"];
+            if (Object.getPrototypeOf(src).constructor !== Object) continue;
+            if (Object.getPrototypeOf(dst).constructor !== Object) continue;
+            srcKeys = Object.keys(src);
+            dstKeys = Object.keys(dst);
+          }
+
+          if (srcKeys.some((k) => dstKeys.includes(k))) continue;
+
+          // Don't mix regular operators with EJSON special operators
+          if (srcKeys.some((k) => ejsonOps.includes(k))) continue;
+          if (dstKeys.some((k) => ejsonOps.includes(k))) continue;
+
+          Object.assign(dst, src);
+          continue loop;
+        }
+
+        query["$and"] ??= [];
+        query["$and"].push({ [param]: value });
+      }
+      or.push(query);
+    }
+
+    if (or.length === 1) return or[0];
+    return { $or: or };
+  }
+}
+
+export function toMongoQuery(
+  exp: Expression,
+  resource: string,
+): Filter<unknown> | false {
+  exp = normalize(exp);
+  const clause = Clause.fromExpression(normalize(exp));
+  const context = new MongoSynthContext(resource);
+  const minterms = clause.true(context);
+  const minimized = context.minimize(minterms);
+  if (!minimized.length) return false;
+  return EJSON.deserialize(context.toQuery(minimized));
+}
+
+export function validQuery(exp: Expression, resource: string): void {
+  const clause = Clause.fromExpression(normalize(exp));
+  const context = new MongoSynthContext(resource);
+  clause.true(context);
+}

lib/db/types.ts

@@ -0,0 +1,183 @@
+import { ObjectId } from "mongodb";
+import { Expression, FaultStruct } from "../types.ts";
+
+export interface Fault {
+  _id: string;
+  device: string;
+  channel: string;
+  timestamp: Date;
+  provisions: string;
+  retries: number;
+  code: string;
+  message: string;
+  detail?:
+    | FaultStruct
+    | {
+        name: string;
+        message: string;
+        stack?: string;
+      };
+  expiry?: Date;
+}
+
+interface TaskBase {
+  _id: ObjectId;
+  timestamp?: Date;
+  expiry?: Date;
+  name: string;
+  device: string;
+}
+
+interface TaskGetParameterValues extends TaskBase {
+  name: "getParameterValues";
+  parameterNames: string[];
+}
+
+interface TaskSetParameterValues extends TaskBase {
+  name: "setParameterValues";
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskRefreshObject extends TaskBase {
+  name: "refreshObject";
+  objectName: string;
+}
+
+interface TaskReboot extends TaskBase {
+  name: "reboot";
+}
+
+interface TaskFactoryReset extends TaskBase {
+  name: "factoryReset";
+}
+
+interface TaskDownload extends TaskBase {
+  name: "download";
+  fileType: string;
+  fileName: string;
+  targetFileName?: string;
+}
+
+interface TaskAddObject extends TaskBase {
+  name: "addObject";
+  objectName: string;
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskDeleteObject extends TaskBase {
+  name: "deleteObject";
+  objectName: string;
+}
+
+interface TaskProvisions extends TaskBase {
+  name: "provisions";
+  provisions?: [string, ...Expression[]][];
+}
+
+export type Task =
+  | TaskGetParameterValues
+  | TaskSetParameterValues
+  | TaskRefreshObject
+  | TaskReboot
+  | TaskFactoryReset
+  | TaskDownload
+  | TaskAddObject
+  | TaskDeleteObject
+  | TaskProvisions;
+
+export interface Operation {
+  _id: string;
+  name: string;
+  timestamp: Date;
+  channels: string;
+  retries: string;
+  provisions: string;
+  args: string;
+}
+
+export interface Config {
+  _id: string;
+  value: string;
+}
+
+export interface Cache {
+  _id: string;
+  value: string;
+  timestamp: Date;
+  expire: Date;
+}
+
+export interface Device {
+  _id: string;
+  _lastInform: Date;
+  _registered: Date;
+  _tags?: string[];
+  _timestamp?: Date;
+}
+
+type Configuration =
+  | { type: "age"; name: string; age: number }
+  | { type: "value"; name: string; value: boolean | number | string }
+  | { type: "add_tag"; tag: string }
+  | { type: "delete_tag"; tag: string }
+  | { type: "add_object"; name: string; object: string }
+  | { type: "delete_object"; name: string; object: string }
+  | { type: "provision"; name: string; args?: Expression[] };
+
+export interface Preset {
+  _id: string;
+  weight: number;
+  channel: string;
+  events: Record<string, boolean>;
+  configurations: Configuration[];
+}
+
+export interface Object {
+  _id: string;
+}
+
+export interface Provision {
+  _id: string;
+  script: string;
+}
+
+export interface VirtualParameter {
+  _id: string;
+  script: string;
+}
+
+export interface File {
+  _id: string;
+  length: number;
+  filename: string;
+  uploadDate: Date;
+  metadata?: {
+    fileType?: string;
+    oui?: string;
+    productClass?: string;
+    version?: string;
+  };
+}
+
+export interface Permission {
+  _id: string;
+  role: string;
+  resource: string;
+  access: 1 | 2 | 3;
+  filter?: string;
+  validate?: string;
+}
+
+export interface User {
+  _id: string;
+  password: string;
+  roles: string;
+  salt: string;
+}
+
+export interface Lock {
+  _id: string;
+  value: string;
+  timestamp: Date;
+  expire: Date;
+}

lib/db/util.ts

@@ -0,0 +1,123 @@
+import { Expression } from "../types.ts";
+import { encodeTag } from "../util.ts";
+
+// Optimize projection by removing overlaps
+// This can modify the object
+export function optimizeProjection(obj: { [path: string]: 1 }): {
+  [path: string]: 1;
+} {
+  if (obj[""]) return { "": obj[""] };
+
+  const keys = Object.keys(obj).sort();
+  if (keys.length <= 1) return obj;
+
+  for (let i = 1; i < keys.length; ++i) {
+    const a = keys[i - 1];
+    const b = keys[i];
+    if (b.startsWith(a)) {
+      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
+        delete obj[b];
+        keys.splice(i--, 1);
+      }
+    }
+  }
+  return obj;
+}
+
+export function convertOldPrecondition(q: Record<string, unknown>): Expression {
+  function recursive(_query): Expression {
+    const expressions: Expression[] = [];
+    for (const [k, v] of Object.entries(_query)) {
+      if (k[0] === "$") {
+        if (k === "$and") {
+          const and: Expression = ["AND"];
+          for (const vv of Object.values(v)) and.push(recursive(vv));
+          expressions.push(and);
+        } else if (k === "$or") {
+          const or: Expression = ["OR"];
+          for (const vv of Object.values(v)) or.push(recursive(vv));
+          expressions.push(or);
+        } else {
+          throw new Error(`Operator ${k} not supported`);
+        }
+      } else if (k === "_tags") {
+        if (typeof v === "object") {
+          if (Array.isArray(v)) throw new Error(`Invalid type`);
+          const conjs: Expression[] = [];
+          for (const [op, val] of Object.entries(v)) {
+            if (op === "$ne") {
+              if (typeof v["$ne"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else {
+              throw new Error(`Invalid tag query`);
+            }
+          }
+          if (conjs.length === 1) expressions.push(conjs[0]);
+          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
+        } else {
+          expressions.push([
+            "IS NOT NULL",
+            ["PARAM", `Tags.${encodeTag(v as string)}`],
+          ]);
+        }
+      } else if (k.startsWith("Tags.")) {
+        let exists: boolean;
+        if (typeof v === "boolean") exists = v;
+        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
+        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
+        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
+        else throw new Error(`Invalid tag query`);
+
+        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
+      } else if (typeof v === "object") {
+        if (Array.isArray(v)) throw new Error(`Invalid type`);
+
+        const exps: Expression[] = [];
+        for (const [kk, vv] of Object.entries(v)) {
+          if (kk === "$eq") {
+            exps.push(["=", ["PARAM", k], vv]);
+          } else if (kk === "$ne") {
+            exps.push([
+              "OR",
+              ["<>", ["PARAM", k], vv],
+              ["IS NULL", ["PARAM", k]],
+            ]);
+          } else if (kk === "$lt") {
+            exps.push(["<", ["PARAM", k], vv]);
+          } else if (kk === "$lte") {
+            exps.push(["<=", ["PARAM", k], vv]);
+          } else if (kk === "$gt") {
+            exps.push([">", ["PARAM", k], vv]);
+          } else if (kk === "$gte") {
+            exps.push([">=", ["PARAM", k], vv]);
+          } else {
+            throw new Error(`Operator ${kk} not supported`);
+          }
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
+        }
+        if (exps.length === 1) {
+          expressions.push(exps[0]);
+        } else if (exps.length > 1) {
+          const and: Expression = ["AND"];
+          expressions.push(and.concat(exps));
+        }
+      } else {
+        expressions.push(["=", ["PARAM", k], v]);
+      }
+    }
+    if (expressions.length === 1) return expressions[0];
+    if (expressions.length === 0) return true;
+    return ["AND", ...expressions];
+  }
+
+  // empty filter
+  if (!Object.keys(q).length) return true;
+
+  return recursive(q);
+}

lib/debug.ts

@@ -0,0 +1,236 @@
+import { IncomingMessage, ServerResponse, ClientRequest } from "node:http";
+import { Socket } from "node:net";
+import { appendFileSync } from "node:fs";
+import { stringify } from "./common/yaml.ts";
+import * as config from "./config.ts";
+import { getSocketEndpoints } from "./server.ts";
+
+const DEBUG_FILE = "" + config.get("DEBUG_FILE");
+const DEBUG_FORMAT = "" + config.get("DEBUG_FORMAT");
+
+const connectionTimestamps = new WeakMap<Socket, Date>();
+
+function getConnectionTimestamp(connection: Socket): Date {
+  let t = connectionTimestamps.get(connection);
+  if (!t) {
+    t = new Date();
+    connectionTimestamps.set(connection, t);
+  }
+  return t;
+}
+
+export function incomingHttpRequest(
+  httpRequest: IncomingMessage,
+  deviceId: string,
+  body: string,
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpRequest.socket;
+  const socketEndpoints = getSocketEndpoints(con);
+  const msg = {
+    event: "incoming HTTP request",
+    timestamp: now,
+    remoteAddress: socketEndpoints.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    localPort: socketEndpoints.localPort,
+    method: httpRequest.method,
+    url: httpRequest.url,
+    headers: httpRequest.headers,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpResponse(
+  httpResponse: ServerResponse,
+  deviceId: string,
+  body: string,
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpResponse.socket;
+  const socketEndpoints = getSocketEndpoints(con);
+  const msg = {
+    event: "outgoing HTTP response",
+    timestamp: now,
+    remoteAddress: socketEndpoints.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    statusCode: httpResponse.statusCode,
+    headers: httpResponse.getHeaders(),
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpRequest(
+  httpRequest: ClientRequest,
+  deviceId: string,
+  method: "GET" | "PUT" | "POST" | "DELETE",
+  url: URL,
+  body: string,
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpRequest.socket;
+  const msg = {
+    event: "outgoing HTTP request",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    remotePort: url.port,
+    method: method,
+    url: url.pathname + url.search,
+    headers: httpRequest.getHeaders(),
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpRequestError(
+  httpRequest: ClientRequest,
+  deviceId: string,
+  method: "GET" | "PUT" | "POST" | "DELETE",
+  url: URL,
+  err: Error,
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing HTTP request",
+    timestamp: now,
+    remoteAddress: url.hostname,
+    deviceId: deviceId,
+    connection: null,
+    remotePort: url.port,
+    method: method,
+    url: url.pathname + url.search,
+    headers: httpRequest.getHeaders(),
+    error: err.message,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function incomingHttpResponse(
+  httpResponse: IncomingMessage,
+  deviceId: string,
+  body: string,
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpResponse.socket;
+  const msg = {
+    event: "incoming HTTP response",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(httpResponse.socket),
+    statusCode: httpResponse.statusCode,
+    headers: httpResponse.headers,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingUdpMessage(
+  remoteAddress: string,
+  deviceId: string,
+  remotePort: number,
+  body: string,
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing UDP message",
+    timestamp: now,
+    remoteAddress: remoteAddress,
+    deviceId: deviceId,
+    remotePort: remotePort,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function clientError(remoteAddress: string, err: Error): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "client error",
+    timestamp: now,
+    remoteAddress: remoteAddress,
+    error: err.message,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function incomingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "incoming XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}

lib/default-provisions.js

@@ -1,117 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const config = require("./config");
-const common = require("./common");
-const device = require("./device");
-const scheduling = require("./scheduling");
-
-const MAX_DEPTH = config.get("MAX_DEPTH");
-
-
-function refresh(sessionContext, provision, declarations, startRevision, endRevision) {
-  let path = common.parsePath(provision[1]).slice();
-  let l = path.length;
-  path.length = MAX_DEPTH;
-  path.fill("*", l);
-  let every = 1000 * (provision[2] || 1);
-  let offset = scheduling.variance(sessionContext.deviceId, every);
-  let t = scheduling.interval(sessionContext.timestamp, every, offset);
-
-  for (let i = l; i < path.length; ++ i)
-    declarations.push([path.slice(0, i), t, {object: 1, writable: 1, value: t}]);
-
-  return true;
-}
-
-
-function value(sessionContext, provision, declarations, startRevision, endRevision) {
-  let path = common.parsePath(provision[1]);
-  declarations.push([path, 1, {value: 1}, null, {value: [provision[2]]}]);
-
-  return true;
-}
-
-
-function tag(sessionContext, provision, declarations, startRevision, endRevision) {
-  let path = ["Tags", provision[1]];
-  declarations.push([path, 1, {value: 1}, null, {value: [provision[2]]}]);
-
-  return true;
-}
-
-
-function reboot(sessionContext, provision, declarations, startRevision, endRevision) {
-  declarations.push([["Reboot"], 1, {value: 1}, null, {value: [sessionContext.timestamp]}]);
-
-  return true;
-}
-
-
-function reset(sessionContext, provision, declarations, startRevision, endRevision) {
-  declarations.push([["FactoryReset"], 1, {value: 1}, null, {value: [sessionContext.timestamp]}]);
-
-  return true;
-}
-
-
-function download(sessionContext, provision, declarations, startRevision, endRevision) {
-  let alias = [["FileType"], provision[1] || "", ["FileName"], provision[2] || "",
-    ["TargetFileName"], provision[3] || ""];
-
-  declarations.push([["Downloads", alias], 1, {}, 1]);
-  declarations.push([["Downloads", alias, "Download"], 1, {value: 1}, null,
-    {value: [sessionContext.timestamp]}]);
-
-  return true;
-}
-
-
-function instances(sessionContext, provision, declarations, startRevision, endRevision) {
-  let count = Number(provision[2]);
-
-  if (Number.isNaN(count))
-    return true;
-
-  let path = common.parsePath(provision[1]);
-
-  if (provision[2][0] === "+" || provision[2][0] === "-") {
-    declarations.push([path, 1]);
-
-    if (endRevision === startRevision)
-      return false;
-
-    let unpacked = device.unpack(sessionContext.deviceData, path, startRevision + 1);
-    count = Math.max(0, unpacked.length + count);
-  }
-
-  declarations.push([path, 1, null, count]);
-
-  return true;
-}
-
-
-exports.refresh = refresh;
-exports.value = value;
-exports.tag = tag;
-exports.reboot = reboot;
-exports.reset = reset;
-exports.download = download;
-exports.instances = instances;

lib/default-provisions.ts

@@ -0,0 +1,259 @@
+import Path from "./common/path.ts";
+import * as config from "./config.ts";
+import * as device from "./device.ts";
+import * as scheduling from "./scheduling.ts";
+import { SessionContext, Declaration } from "./types.ts";
+
+const MAX_DEPTH = +config.get("MAX_DEPTH");
+
+export function refresh(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (
+    (provision.length !== 2 || typeof provision[1] !== "string") &&
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number") &&
+    (provision.length < 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number" ||
+      typeof provision[3] !== "boolean")
+  )
+    throw new Error("Invalid arguments");
+
+  const every = 1000 * ((provision[2] as number) || 1);
+  const offset = scheduling.variance(sessionContext.deviceId, every);
+  const t = scheduling.interval(sessionContext.timestamp, every, offset);
+
+  let attrGet;
+  let refreshChildren;
+  if (provision[3] == null) {
+    refreshChildren = true;
+    attrGet = { object: 1, writable: 1, value: t };
+  } else {
+    attrGet = {};
+    refreshChildren = !!provision[3];
+    for (const a of provision.slice(4)) attrGet[a as string] = t;
+  }
+
+  let path = Path.parse(provision[1]);
+  let l = path.length;
+  if (refreshChildren) {
+    const segments = path.segments.slice();
+    l = segments.length;
+    segments.length = MAX_DEPTH;
+    segments.fill("*", l);
+    path = Path.parse(segments.join("."));
+  }
+
+  for (let i = l; i <= path.length; ++i) {
+    declarations.push({
+      path: path.slice(0, i),
+      pathGet: t,
+      pathSet: null,
+      attrGet: attrGet,
+      attrSet: null,
+      defer: true,
+    });
+  }
+
+  return true;
+}
+
+export function value(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (
+    provision.length < 3 ||
+    provision.length > 4 ||
+    typeof provision[1] !== "string"
+  )
+    throw new Error("Invalid arguments");
+
+  let attr: string, val: any;
+
+  if (provision.length === 3) {
+    attr = "value";
+    val = provision[2];
+  } else {
+    attr = (provision[2] as string) || "";
+    val = provision[3];
+  }
+
+  if (attr === "accessList") {
+    val = (val || "")
+      .split(",")
+      .map((s) => s.trim())
+      .filter((s) => !!s);
+  } else if (attr === "value") {
+    val = [val];
+  }
+
+  declarations.push({
+    path: Path.parse(provision[1]),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { [attr]: 1 },
+    attrSet: { [attr]: val },
+    defer: true,
+  });
+
+  return true;
+}
+
+export function tag(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (
+    provision.length !== 3 ||
+    typeof provision[1] !== "string" ||
+    typeof provision[2] !== "boolean"
+  )
+    throw new Error("Invalid arguments");
+
+  declarations.push({
+    path: Path.parse(`Tags.${provision[1]}`),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [provision[2]] },
+    defer: true,
+  });
+
+  return true;
+}
+
+export function reboot(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
+  declarations.push({
+    path: Path.parse("Reboot"),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true,
+  });
+
+  return true;
+}
+
+export function reset(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
+  declarations.push({
+    path: Path.parse("FactoryReset"),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true,
+  });
+
+  return true;
+}
+
+export function download(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string") &&
+    (provision.length !== 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string" ||
+      typeof provision[3] !== "string")
+  )
+    throw new Error("Invalid arguments");
+
+  const alias = [
+    `FileType:${JSON.stringify(provision[1] || "")}`,
+    `FileName:${JSON.stringify(provision[2] || "")}`,
+    `TargetFileName:${JSON.stringify(provision[3] || "")}`,
+  ].join(",");
+
+  declarations.push({
+    path: Path.parse(`Downloads.[${alias}]`),
+    pathGet: 1,
+    pathSet: 1,
+    attrGet: null,
+    attrSet: null,
+    defer: true,
+  });
+
+  declarations.push({
+    path: Path.parse(`Downloads.[${alias}].Download`),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true,
+  });
+
+  return true;
+}
+
+export function instances(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+  startRevision: number,
+  endRevision: number,
+): boolean {
+  if (provision.length !== 3 || typeof provision[1] !== "string")
+    throw new Error("Invalid arguments");
+
+  let count = Number(provision[2]);
+
+  if (Number.isNaN(count)) throw new Error("Invalid arguments");
+
+  const path = Path.parse(provision[1]);
+
+  if (provision[2][0] === "+" || provision[2][0] === "-") {
+    declarations.push({
+      path: path,
+      pathGet: 1,
+      pathSet: null,
+      attrGet: null,
+      attrSet: null,
+      defer: true,
+    });
+
+    if (endRevision === startRevision) return false;
+
+    const unpacked = device.unpack(
+      sessionContext.deviceData,
+      path,
+      startRevision + 1,
+    );
+    count = Math.max(0, unpacked.length + count);
+  }
+
+  declarations.push({
+    path: path,
+    pathGet: 1,
+    pathSet: count,
+    attrGet: null,
+    attrSet: null,
+    defer: true,
+  });
+
+  return true;
+}

lib/device.js

@@ -1,393 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const common = require('./common');
-
-
-const CHANGE_FLAGS = {
-  object: 2,
-  writable: 4,
-  value: 8
-};
-
-
-function parseBool(v) {
-  v = "" + v;
-  if (v === "true" || v === "TRUE" || v === "True" || v === "1")
-    return true;
-  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
-    return false;
-  else
-    return null;
-}
-
-
-function sanitizeParameterValue(parameterValue) {
-  if (parameterValue[0] != null) {
-    switch (parameterValue[1]) {
-      case 'xsd:boolean':
-        if (typeof parameterValue[0] !== 'boolean') {
-          let v = parseBool(parameterValue[0]);
-          if (v == null)
-            v = '' + parameterValue[0];
-          parameterValue[0] = v;
-        }
-        break;
-      case 'xsd:string':
-        if (typeof parameterValue[0] !== 'string') {
-          parameterValue[0] = '' + parameterValue[0];
-        }
-        break;
-      case 'xsd:int':
-      case 'xsd:unsignedInt':
-        if (typeof parameterValue[0] !== 'number') {
-          let v = parseInt(parameterValue[0]);
-          if (isNaN(v))
-            v = '' + parameterValue[0];
-          parameterValue[0] = v;
-        }
-        break;
-      case 'xsd:dateTime':
-        if (typeof parameterValue[0] !== 'number') {
-          // Don't use parseInt because it reads date string as a number
-          let v = +parameterValue[0];
-          if (isNaN(v)) {
-            v = Date.parse(parameterValue[0]);
-            if (isNaN(v))
-              v = '' + parameterValue[0];
-          }
-          parameterValue[0] = v;
-        }
-        break;
-      default:
-        if (parameterValue[1] != null)
-          throw new Error(`Parameter value type "${parameterValue[1]}" not recognized.`);
-
-        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
-        break;
-    }
-  }
-  return parameterValue;
-}
-
-
-function getAliasDeclarations(path, timestamp) {
-  var decs = [];
-
-  function recursive(pattern, prefix) {
-    var pattern = prefix.concat(pattern);
-
-    for (let i = prefix.length; i < pattern.length; ++ i) {
-      if (Array.isArray(pattern[i])) {
-        let pat = pattern[i];
-        pattern[i] = '*';
-        for (let j = 0; j < pat.length; j += 2)
-          recursive(pat[j], pattern.slice(0, i + 1));
-      }
-    }
-
-    if (prefix.length)
-      decs.push([pattern, timestamp, {value: timestamp}]);
-    else
-      decs.push([pattern, timestamp]);
-  }
-
-  recursive(path, []);
-
-  return decs;
-}
-
-
-function unpack(deviceData, path, revision) {
-  let allMatches = [];
-  if (!path.alias) {
-    for (let p of deviceData.paths.find(path, false, true))
-      if (deviceData.attributes.has(p, revision))
-        allMatches.push(p);
-  }
-  else {
-    let wildcardPath = path.slice();
-    wildcardPath.wildcard = path.wildcard;
-    for (let i = 0; i < wildcardPath.length; ++ i)
-      if (Array.isArray(wildcardPath[i])) {
-        wildcardPath[i] = '*';
-        wildcardPath.wildcard |= 1 << i;
-      }
-
-    for (let p of deviceData.paths.find(wildcardPath, false, true))
-      if (deviceData.attributes.has(p, revision))
-        allMatches.push(p);
-
-    for (let i = path.length - 1; i >= 0; -- i) {
-      if (Array.isArray(path[i])) {
-        for (let j = 0; j < path[i].length; j += 2) {
-          let p = wildcardPath.slice(0, i + 1).concat(path[i][j]);
-          p.alias = (wildcardPath.alias & ((1 << (i + 1)) - 1)) + path[i][j].alias << (i + 1);
-          p.wildcard = (wildcardPath.wildcard & ((1 << (i + 1)) - 1)) + path[i][j].wildcard << (i + 1);
-          let unpacked = unpack(deviceData, p, revision);
-          let filtered = [];
-          for (let up of unpacked) {
-            let attributes = deviceData.attributes.get(up, revision);
-            if (attributes && attributes.value && attributes.value[1] &&
-                sanitizeParameterValue([path[i][j + 1], attributes.value[1][1]])[0] === attributes.value[1][0]) {
-              for (let m = 0; m < allMatches.length; ++ m) {
-                let k;
-                for (k = i; k >= 0; -- k)
-                  if (allMatches[m][k] != up[k])
-                    break
-
-                if (k < 0) {
-                  filtered.push(allMatches[m]);
-                  allMatches[m] = [];
-                }
-              }
-            }
-          }
-          allMatches = filtered;
-        }
-      }
-    }
-  }
-
-  allMatches.sort(function(a, b) {
-    for (let i = 0; i < a.length; ++ i) {
-      if (a === b)
-        continue;
-
-      // Use numeric sorting for numbers
-      let ia = parseInt(a), ib = parseInt(b);
-      if (ia.toString() === a && ib.toString() === b) {
-        a = ia;
-        b = ib;
-      }
-
-      if (a < b)
-        return -1;
-      else
-        return 1;
-    }
-    return 0;
-  });
-
-  return allMatches;
-}
-
-
-function clear(deviceData, path, timestamp, attributes, changeFlags = 0) {
-  let changeTrackers = {};
-
-  timestamp = timestamp || 0;
-
-  let descendantsTimestamp = timestamp;
-  if (attributes && attributes.object) {
-    if (attributes.object > descendantsTimestamp)
-      descendantsTimestamp = attributes.object;
-    if (!(attributes.object <= attributes.value))
-      attributes.value = attributes.object;
-  }
-
-  for (let p of deviceData.paths.find(path, true, true, descendantsTimestamp ? 99 : path.length)) {
-    let tracker = deviceData.trackers.get(p);
-    for (let k in tracker)
-      changeTrackers[k] |= tracker[k];
-
-    let currentTimestamp = deviceData.timestamps.get(p);
-    if (currentTimestamp === undefined)
-      continue;
-
-    if (timestamp > currentTimestamp ||
-      (descendantsTimestamp > currentTimestamp && p.length > path.length)) {
-      deviceData.timestamps.delete(p);
-      deviceData.attributes.delete(p);
-      changeFlags |= 1;
-    }
-    else if (attributes && p.length == path.length) {
-      let currentAttributes = deviceData.attributes.get(p);
-      if (currentAttributes) {
-        let newAttrs;
-        for (let attrName in attributes) {
-          if (attrName in currentAttributes &&
-              attributes[attrName] > currentAttributes[attrName][0]) {
-            changeFlags |= CHANGE_FLAGS[attrName];
-            if (!newAttrs) {
-              newAttrs = Object.assign({}, currentAttributes);
-              deviceData.attributes.set(p, newAttrs);
-            }
-            delete newAttrs[attrName];
-          }
-        }
-      }
-    }
-  }
-
-  // Note: For performance, we're merging all changes together rather than
-  // mark changes based the exact parameters affected.
-  for (let k in changeTrackers) {
-    if (changeTrackers[k] & changeFlags)
-      deviceData.changes.add(k);
-  }
-}
-
-
-function compareEquality(a, b) {
-  var t = typeof a;
-  if (a === null ||
-      a === undefined ||
-      t === 'number' ||
-      t === 'boolean' ||
-      t === 'string' ||
-      t === 'symbol')
-    return a === b;
-
-  return JSON.stringify(a) === JSON.stringify(b);
-}
-
-
-function set(deviceData, path, timestamp, attributes, toClear) {
-  path = deviceData.paths.add(path);
-
-  let currentTimestamp = deviceData.timestamps.get(path);
-
-  let currentAttributes;
-
-  if (path.wildcard)
-    attributes = undefined;
-  else if (currentTimestamp)
-    currentAttributes = deviceData.attributes.get(path);
-
-  let changeFlags = 0;
-
-  if (attributes) {
-    if (attributes.value && attributes.value[1]
-        && attributes.value[0] >= (attributes.object ? attributes.object[0] : 0))
-      attributes.object = [attributes.value[0], 0];
-
-    if (attributes.object && attributes.object[1]
-      && attributes.object[0] >= (attributes.value ? attributes.value[0] : 0))
-      attributes.value = [attributes.object[0]];
-
-    let newAttributes = Object.assign({}, currentAttributes, attributes);
-
-    if (currentAttributes) {
-      for (let attrName in attributes) {
-        timestamp = Math.max(timestamp, attributes[attrName][0]);
-        if (!(attrName in currentAttributes))
-          changeFlags |= CHANGE_FLAGS[attrName];
-        else if (attributes[attrName][0] <= currentAttributes[attrName][0])
-          newAttributes[attrName] = currentAttributes[attrName];
-        else if (!compareEquality(attributes[attrName][1], currentAttributes[attrName][1]))
-          changeFlags |= CHANGE_FLAGS[attrName];
-      }
-    }
-    else
-      changeFlags |= 1;
-
-    deviceData.attributes.set(path, newAttributes);
-
-    if (!(timestamp <= currentTimestamp)) {
-      deviceData.timestamps.set(path, timestamp);
-      if (path.length > 1)
-        toClear = set(deviceData, path.slice(0, path.length - 1), timestamp,
-          {object: [timestamp, 1]}, toClear);
-    }
-  }
-  else if (!(timestamp <= currentTimestamp)) {
-    deviceData.timestamps.set(path, timestamp);
-
-    if (currentAttributes) {
-      deviceData.attributes.delete(path);
-      changeFlags |= 1;
-    }
-    else if (path.wildcard) {
-      for (let p of deviceData.paths.find(path, false, true, path.length))
-        if (timestamp > deviceData.timestamps.get(p)) {
-          toClear = toClear || [];
-          toClear.push([p, timestamp]);
-        }
-    }
-  }
-
-  if (changeFlags) {
-    if (changeFlags & 1) {
-      toClear = toClear || [];
-      toClear.push([path, timestamp, null, changeFlags]);
-    }
-    else if (changeFlags & CHANGE_FLAGS.object) {
-      toClear = toClear || [];
-      toClear.push([path, 0, {object: attributes.object[0]}, changeFlags]);
-    }
-    else {
-      for (let p of deviceData.paths.find(path, true, false, path.length)) {
-        let tracker = deviceData.trackers.get(p);
-        for (let k in tracker) {
-          if (tracker[k] & changeFlags)
-            deviceData.changes.add(k);
-        }
-      }
-    }
-  }
-
-  return toClear;
-}
-
-
-function track(deviceData, path, marker, attributes) {
-  path = deviceData.paths.add(path);
-  let f = 1;
-
-  if (attributes)
-    for (let attrName of attributes)
-      f |= CHANGE_FLAGS[attrName];
-
-  let cur = deviceData.trackers.get(path);
-  if (!cur) {
-    cur = {};
-    deviceData.trackers.set(path, cur);
-  }
-
-  cur[marker] |= f;
-}
-
-
-function clearTrackers(deviceData, tracker) {
-  if (Array.isArray(tracker)) {
-    deviceData.trackers.forEach(function(v, k) {
-      for (let t of tracker)
-        delete v[t];
-    });
-    for (let t of tracker)
-      deviceData.changes.delete(t);
-  }
-  else {
-    deviceData.trackers.forEach(function(v, k) {
-      delete v[tracker];
-    });
-    deviceData.changes.delete(tracker);
-  }
-}
-
-
-exports.sanitizeParameterValue = sanitizeParameterValue;
-exports.getAliasDeclarations = getAliasDeclarations;
-exports.unpack = unpack;
-exports.set = set;
-exports.clear = clear;
-exports.track = track;
-exports.clearTrackers = clearTrackers;

lib/device.ts

@@ -0,0 +1,390 @@
+import Path from "./common/path.ts";
+import {
+  DeviceData,
+  Declaration,
+  Clear,
+  AttributeTimestamps,
+  Attributes,
+} from "./types.ts";
+
+const CHANGE_FLAGS = {
+  object: 2,
+  writable: 4,
+  value: 8,
+  notification: 16,
+  accessList: 32,
+};
+
+function parseBool(v): boolean {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else return null;
+}
+
+export function sanitizeParameterValue(
+  parameterValue: [string | number | boolean, string],
+): [string | number | boolean, string] {
+  if (parameterValue[0] != null) {
+    switch (parameterValue[1]) {
+      case "xsd:boolean":
+        if (typeof parameterValue[0] !== "boolean") {
+          const b = parseBool(parameterValue[0]);
+          if (b == null) parameterValue[0] = "" + parameterValue[0];
+          else parameterValue[0] = b;
+        }
+        break;
+      case "xsd:int":
+      case "xsd:unsignedInt":
+        if (typeof parameterValue[0] !== "number") {
+          const i = parseInt(parameterValue[0] as string);
+          if (isNaN(i)) parameterValue[0] = "" + parameterValue[0];
+          else parameterValue[0] = i;
+        }
+        break;
+      case "xsd:dateTime":
+        if (typeof parameterValue[0] !== "number") {
+          // Don't use parseInt because it reads date string as a number
+          let i = +parameterValue[0];
+          if (isNaN(i)) {
+            i = Date.parse(parameterValue[0] as string);
+            if (isNaN(i)) parameterValue[0] = "" + parameterValue[0];
+            else parameterValue[0] = i;
+          } else {
+            parameterValue[0] = i;
+          }
+        }
+        break;
+      default:
+        parameterValue[0] = "" + parameterValue[0];
+        break;
+    }
+  }
+
+  return parameterValue;
+}
+
+export function getAliasDeclarations(
+  path: Path,
+  timestamp: number,
+  attrGet = null,
+): Declaration[] {
+  const stripped = path.stripAlias();
+  let decs: Declaration[] = [
+    {
+      path: stripped,
+      pathGet: timestamp,
+      pathSet: null,
+      attrGet: attrGet,
+      attrSet: null,
+      defer: true,
+    },
+  ];
+
+  if (path.alias) {
+    for (const [i, alias] of path.segments.entries()) {
+      if (Array.isArray(alias)) {
+        const parent = stripped.slice(0, i + 1);
+        for (const [p] of alias as [Path, string][]) {
+          decs = decs.concat(
+            getAliasDeclarations(parent.concat(p), timestamp, {
+              value: timestamp,
+            }),
+          );
+        }
+      }
+    }
+  }
+
+  return decs;
+}
+
+export function unpack(
+  deviceData: DeviceData,
+  path: Path,
+  revision?: number,
+): Path[] {
+  let allMatches = [] as Path[];
+  if (!path.alias) {
+    for (const p of deviceData.paths.find(path, false, true))
+      if (deviceData.attributes.has(p, revision)) allMatches.push(p);
+  } else {
+    const wildcardPath = path.stripAlias();
+
+    for (const p of deviceData.paths.find(wildcardPath, false, true))
+      if (deviceData.attributes.has(p, revision)) allMatches.push(p);
+
+    for (let i = path.length - 1; i >= 0; --i) {
+      if (path.alias & (1 << i)) {
+        for (const [param, val] of path.segments[i] as [Path, string][]) {
+          const p = wildcardPath.slice(0, i + 1).concat(param);
+          const unpacked = unpack(deviceData, p, revision);
+          const filtered: Path[] = [];
+          for (const up of unpacked) {
+            const attributes = deviceData.attributes.get(up, revision);
+            if (
+              attributes &&
+              attributes.value &&
+              attributes.value[1] &&
+              sanitizeParameterValue([val, attributes.value[1][1]])[0] ===
+                attributes.value[1][0]
+            ) {
+              for (let m = 0; m < allMatches.length; ++m) {
+                let k;
+                const match = allMatches[m];
+                if (!match) continue;
+                for (k = i; k >= 0; --k)
+                  if (match.segments[k] !== up.segments[k]) break;
+
+                if (k < 0) {
+                  filtered.push(match);
+                  allMatches[m] = null;
+                }
+              }
+            }
+          }
+          allMatches = filtered;
+        }
+      }
+    }
+  }
+
+  allMatches.sort((p1, p2) => {
+    for (let i = 0; i < p1.length; ++i) {
+      const a = p1.segments[i] as string;
+      const b = p2.segments[i] as string;
+      if (a !== b) {
+        // Use numeric sorting for numbers
+        const ia = parseInt(a);
+        const ib = parseInt(b);
+
+        if (ia === +a && ib === +b) return ia - ib;
+        else if (a < b) return -1;
+        else return 1;
+      }
+    }
+    return 0;
+  });
+
+  return allMatches;
+}
+
+export function clear(
+  deviceData: DeviceData,
+  path: Path,
+  timestamp: number,
+  attributes: AttributeTimestamps,
+  changeFlags = 0,
+): void {
+  const changeTrackers = {};
+
+  timestamp = timestamp || 0;
+
+  let descendantsTimestamp = timestamp;
+  if (attributes?.object) {
+    if (attributes.object > descendantsTimestamp)
+      descendantsTimestamp = attributes.object;
+    if (!(attributes.object <= attributes.value))
+      attributes.value = attributes.object;
+  }
+
+  for (const p of deviceData.paths.find(
+    path,
+    true,
+    true,
+    descendantsTimestamp ? 99 : path.length,
+  )) {
+    const tracker = deviceData.trackers.get(p);
+    for (const k in tracker) changeTrackers[k] |= tracker[k];
+
+    const currentTimestamp = deviceData.timestamps.get(p);
+    if (currentTimestamp === undefined) continue;
+
+    if (
+      timestamp > currentTimestamp ||
+      (descendantsTimestamp > currentTimestamp && p.length > path.length)
+    ) {
+      deviceData.timestamps.delete(p);
+      deviceData.attributes.delete(p);
+      changeFlags |= 1;
+    } else if (attributes && p.length === path.length) {
+      const currentAttributes = deviceData.attributes.get(p);
+      if (currentAttributes) {
+        let newAttrs;
+        for (const attrName in attributes) {
+          if (
+            attrName in currentAttributes &&
+            attributes[attrName] > currentAttributes[attrName][0]
+          ) {
+            changeFlags |= CHANGE_FLAGS[attrName];
+            if (!newAttrs) {
+              newAttrs = Object.assign({}, currentAttributes);
+              deviceData.attributes.set(p, newAttrs);
+            }
+            delete newAttrs[attrName];
+          }
+        }
+      }
+    }
+  }
+
+  // Note: For performance, we're merging all changes together rather than
+  // mark changes based the exact parameters affected.
+  for (const k in changeTrackers)
+    if (changeTrackers[k] & changeFlags) deviceData.changes.add(k);
+}
+
+function compareEquality(a, b): boolean {
+  const t = typeof a;
+  if (
+    a === null ||
+    a === undefined ||
+    t === "number" ||
+    t === "boolean" ||
+    t === "string" ||
+    t === "symbol"
+  )
+    return a === b;
+
+  return JSON.stringify(a) === JSON.stringify(b);
+}
+
+export function set(
+  deviceData: DeviceData,
+  path: Path,
+  timestamp: number,
+  attributes: Attributes,
+  toClear?: Clear[],
+): Clear[] {
+  path = deviceData.paths.add(path);
+
+  const currentTimestamp = deviceData.timestamps.get(path);
+
+  let currentAttributes;
+
+  if (path.wildcard) attributes = undefined;
+  else if (currentTimestamp)
+    currentAttributes = deviceData.attributes.get(path);
+
+  let changeFlags = 0;
+
+  if (attributes) {
+    if (
+      attributes.value &&
+      attributes.value[1] &&
+      attributes.value[0] >= (attributes.object ? attributes.object[0] : 0)
+    )
+      attributes.object = [attributes.value[0], 0];
+
+    if (
+      attributes.object &&
+      attributes.object[1] &&
+      attributes.object[0] >= (attributes.value ? attributes.value[0] : 0)
+    )
+      attributes.value = [attributes.object[0], null];
+
+    const newAttributes = Object.assign({}, currentAttributes, attributes);
+
+    if (currentAttributes) {
+      for (const attrName in attributes) {
+        timestamp = Math.max(timestamp, attributes[attrName][0]);
+        if (!(attrName in currentAttributes))
+          changeFlags |= CHANGE_FLAGS[attrName];
+        else if (attributes[attrName][0] <= currentAttributes[attrName][0])
+          newAttributes[attrName] = currentAttributes[attrName];
+        else if (
+          !compareEquality(
+            attributes[attrName][1],
+            currentAttributes[attrName][1],
+          )
+        )
+          changeFlags |= CHANGE_FLAGS[attrName];
+      }
+    } else {
+      changeFlags |= 1;
+    }
+
+    deviceData.attributes.set(path, newAttributes);
+
+    if (!(timestamp <= currentTimestamp)) {
+      deviceData.timestamps.set(path, timestamp);
+      if (path.length > 1) {
+        toClear = set(
+          deviceData,
+          path.slice(0, path.length - 1),
+          timestamp,
+          { object: [timestamp, 1] },
+          toClear,
+        );
+      }
+    }
+  } else if (!(timestamp <= currentTimestamp)) {
+    deviceData.timestamps.set(path, timestamp);
+
+    if (currentAttributes) {
+      deviceData.attributes.delete(path);
+      changeFlags |= 1;
+    } else if (path.wildcard) {
+      for (const p of deviceData.paths.find(path, false, true, path.length)) {
+        if (timestamp > deviceData.timestamps.get(p)) {
+          toClear = toClear || [];
+          toClear.push([p, timestamp]);
+        }
+      }
+    }
+  }
+
+  if (changeFlags) {
+    if (changeFlags & 1) {
+      toClear = toClear || [];
+      toClear.push([path, timestamp, null, changeFlags]);
+    } else if (changeFlags & CHANGE_FLAGS.object) {
+      toClear = toClear || [];
+      toClear.push([path, 0, { object: attributes.object[0] }, changeFlags]);
+    } else {
+      for (const p of deviceData.paths.find(path, true, false, path.length)) {
+        const tracker = deviceData.trackers.get(p);
+        for (const k in tracker)
+          if (tracker[k] & changeFlags) deviceData.changes.add(k);
+      }
+    }
+  }
+
+  return toClear;
+}
+
+export function track(
+  deviceData: DeviceData,
+  path: Path,
+  marker: string,
+  attributes?: string[],
+): void {
+  path = deviceData.paths.add(path);
+  let f = 1;
+
+  if (attributes)
+    for (const attrName of attributes) f |= CHANGE_FLAGS[attrName];
+
+  let cur = deviceData.trackers.get(path);
+  if (!cur) {
+    cur = {};
+    deviceData.trackers.set(path, cur);
+  }
+
+  cur[marker] |= f;
+}
+
+export function clearTrackers(
+  deviceData: DeviceData,
+  tracker: string | string[],
+): void {
+  if (Array.isArray(tracker)) {
+    for (const v of deviceData.trackers.values())
+      for (const t of tracker) delete v[t];
+    for (const t of tracker) deviceData.changes.delete(t);
+  } else {
+    for (const v of deviceData.trackers.values()) delete v[tracker];
+    deviceData.changes.delete(tracker);
+  }
+}

lib/extension-wrapper.js

@@ -1,90 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const jobs = new Set();
-const fileName = process.argv[2];
-let script;
-
-function errorToFault(err) {
-  if (!err)
-    return null;
-
-  if (!err.name)
-    return {code: "ext", message: `${err}`};
-
-  let fault = {
-    code: `ext.${err.name}`,
-    message: err.message,
-    detail: {
-      name: err.name,
-      message: err.message
-    }
-  };
-
-  if (err.stack) {
-    fault.detail.stack = err.stack;
-    // Trim the stack trace
-    let stackTrimIndex = fault.detail.stack.match(/\s+at\s[^\s]+\s\(.*extension-wrapper\.js:.+\)/);
-    if (stackTrimIndex)
-      fault.detail.stack = fault.detail.stack.slice(0, stackTrimIndex.index);
-  }
-
-  return fault;
-}
-
-process.on("uncaughtException", function(err) {
-  let fault = errorToFault(err);
-  jobs.forEach(function(jobId) {
-    process.send([jobId, fault]);
-  });
-  jobs.clear();
-  process.disconnect();
-});
-
-process.on("message", function(message) {
-  jobs.add(message[0]);
-
-  if (!script) {
-    let cwd = process.env["GENIEACS_CONFIG_DIR"] + "/ext";
-    process.chdir(cwd);
-    script = require(`${cwd}/${fileName}`);
-  }
-
-  const funcName = message[1][0];
-
-  if (!script[funcName]) {
-    let fault = {
-      code: "ext",
-      message: `No such function '${funcName}' in extension '${fileName}'`
-    };
-    process.send([message[0], fault]);
-    return;
-  }
-
-  script[funcName](message[1].slice(1), function(err, res) {
-    if (!jobs.delete(message[0]))
-      return;
-
-    process.send([message[0], errorToFault(err), res]);
-  });
-});
-
-// Ignore SIGINT
-process.on('SIGINT', function() {});

lib/extensions.js

@@ -1,117 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const childProcess = require("child_process");
-const crypto = require ("crypto");
-
-const config = require("./config");
-
-const TIMEOUT = config.get('EXT_TIMEOUT');
-
-const processes = {};
-const jobs = {};
-
-function messageHandler(message) {
-  let func = jobs[message[0]];
-  if (func) {
-    delete jobs[message[0]];
-    func(null, message[1], message[2]);
-  }
-}
-
-
-function run(args, callback) {
-  let scriptName = args[0];
-
-  if (!processes[scriptName]) {
-    let p = childProcess.fork('lib/extension-wrapper', [scriptName]);
-    processes[scriptName] = p;
-
-    p.on('error', function() {
-      if (processes[scriptName] === p) {
-        kill(processes[scriptName]);
-        delete processes[scriptName];
-      }
-    });
-
-    p.on('disconnect', function() {
-      if (processes[scriptName] === p) {
-        kill(processes[scriptName]);
-        delete processes[scriptName];
-      }
-    });
-
-    p.on('message', messageHandler);
-  }
-
-  let id = crypto.randomBytes(8).toString('hex');
-  jobs[id] = callback;
-  setTimeout(function() {
-    if (id in jobs) {
-      delete jobs[id];
-      return callback(null, {code: "timeout", message: "Extension timed out"});
-    }
-  }, TIMEOUT);
-
-  return processes[scriptName].send([id, args.slice(1)]);
-};
-
-
-function kill(process, callback) {
-  if (process.signalCode !== null || process.exitCode !== null)
-    return;
-
-  let timeToKill = Date.now() + 5000;
-
-  process.kill();
-
-  let t = setInterval(function() {
-    if (process.signalCode !== null || process.exitCode !== null) {
-      clearInterval(t);
-      if (callback)
-        callback();
-    }
-    else if (Date.now() > timeToKill) {
-      process.kill('SIGKILL');
-      clearInterval(t);
-      if (callback)
-        callback();
-    }
-  }, 100);
-}
-
-
-function killAll(callback) {
-  let counter = 1;
-  for (let n in processes) {
-    ++counter;
-    kill(processes[n], function() {
-      if (--counter == 0 && callback)
-        callback();
-    });
-    delete processes[n];
-  }
-  if (--counter == 0 && callback)
-    callback();
-}
-
-
-exports.run = run;
-exports.killAll = killAll;

lib/extensions.ts

@@ -0,0 +1,109 @@
+import { spawn, ChildProcess } from "node:child_process";
+import * as crypto from "node:crypto";
+import readline from "node:readline";
+import * as config from "./config.ts";
+import { Fault } from "./types.ts";
+import { ROOT_DIR } from "./config.ts";
+import * as logger from "./logger.ts";
+
+const TIMEOUT = +config.get("EXT_TIMEOUT");
+
+const processes: { [script: string]: ChildProcess } = {};
+const jobs = new Map();
+
+export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
+  return new Promise((resolve) => {
+    const scriptName = args[0];
+
+    const id = crypto.randomBytes(8).toString("hex");
+    jobs.set(id, resolve);
+
+    if (!processes[scriptName]) {
+      const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
+        stdio: ["ignore", "pipe", "pipe", "ipc"],
+      });
+      processes[scriptName] = p;
+
+      p.on("error", (err) => {
+        if (processes[scriptName] === p) {
+          if (jobs.delete(id)) {
+            resolve({
+              fault: { code: err.name, message: err.message },
+              value: null,
+            });
+          }
+
+          // eslint-disable-next-line @typescript-eslint/no-floating-promises
+          kill(processes[scriptName]);
+          delete processes[scriptName];
+        }
+      });
+
+      p.on("disconnect", () => {
+        if (processes[scriptName] === p) delete processes[scriptName];
+      });
+
+      p.on("message", (message) => {
+        const func = jobs.get(message[0]);
+        if (func) {
+          jobs.delete(message[0]);
+          // Wait for any disconnect even to fire
+          setTimeout(() => {
+            func({ fault: message[1], value: message[2] });
+          });
+        }
+      });
+
+      const rlstdout = readline.createInterface(p.stdout);
+      rlstdout.on("line", (line) => {
+        logger.info({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
+      });
+
+      const rlstderr = readline.createInterface(p.stderr);
+      rlstderr.on("line", (line) => {
+        logger.warn({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
+      });
+    }
+
+    setTimeout(() => {
+      if (jobs.delete(id)) {
+        resolve({
+          fault: { code: "timeout", message: "Extension timed out" },
+          value: null,
+        });
+      }
+    }, TIMEOUT);
+
+    if (!processes[scriptName].connected) return false;
+
+    return processes[scriptName].send([id, args.slice(1)]);
+  });
+}
+
+function kill(process: ChildProcess): Promise<void> {
+  return new Promise((resolve) => {
+    const timeToKill = Date.now() + 5000;
+
+    process.kill();
+
+    const t = setInterval(() => {
+      if (!process.connected) {
+        clearInterval(t);
+        resolve();
+      } else if (Date.now() > timeToKill) {
+        process.kill("SIGKILL");
+        clearInterval(t);
+        resolve();
+      }
+    }, 100);
+  });
+}
+
+export async function killAll(): Promise<void> {
+  await Promise.all(
+    Object.entries(processes).map(([k, p]) => {
+      delete processes[k];
+      return kill(p);
+    }),
+  );
+}

lib/forwarded.ts

@@ -0,0 +1,160 @@
+import { IncomingMessage } from "node:http";
+import { TLSSocket } from "node:tls";
+import { parseCIDR, parse, IPv6, IPv4 } from "ipaddr.js";
+import * as config from "./config.ts";
+import { getSocketEndpoints } from "./server.ts";
+
+interface RequestOrigin {
+  localAddress: string;
+  localPort: number;
+  remoteAddress: string;
+  remotePort: number;
+  host: string;
+  encrypted: boolean;
+}
+
+const FORWARDED_HEADER = "" + config.get("FORWARDED_HEADER");
+const cache = new WeakMap<IncomingMessage, RequestOrigin>();
+const cidrs: [IPv4 | IPv6, number][] = [];
+
+for (const str of FORWARDED_HEADER.split(",").map((s) => s.trim())) {
+  try {
+    cidrs.push(parseCIDR(str));
+  } catch (err) {
+    // Not a valid CIDR format, try parsing as IP
+    try {
+      const ip = parse(str);
+      cidrs.push([ip, ip.toByteArray().length * 8]);
+    } catch (err) {
+      // Not a valid IP either, ignore
+    }
+  }
+}
+
+function parseForwardedHeader(str: string): { [name: string]: string } {
+  str = str.toLowerCase();
+  const res: { [name: string]: string } = {};
+  let keyIdx = 0;
+  let valueIdx = -1;
+  let key: string;
+  for (let i = 0; i < str.length; ++i) {
+    const char = str.charCodeAt(i);
+    if (char === 61 /* = */) {
+      if (keyIdx >= 0) {
+        key = str.slice(keyIdx, i).trim();
+        keyIdx = -1;
+        valueIdx = i + 1;
+      }
+    } else if (char === 59 /* ; */) {
+      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
+      valueIdx = -1;
+      keyIdx = i + 1;
+    } else if (char === 44 /* , */) {
+      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
+      return res;
+    } else if (char === 34 /* " */) {
+      if (valueIdx >= 0) {
+        const quoteIdx = i;
+        if (!str.slice(valueIdx, quoteIdx).trim()) {
+          for (i = i + 1; i < str.length; ++i) {
+            const c = str.charCodeAt(i);
+            if (c === 92 /* \ */) ++i;
+            if (c === 34 /* " */) {
+              res[key] = JSON.parse(str.slice(quoteIdx, i + 1).trim());
+              valueIdx = -1;
+              keyIdx = i + 1;
+              break;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  if (valueIdx >= 0) res[key] = str.slice(valueIdx).trim();
+
+  return res;
+}
+
+export function getRequestOrigin(request: IncomingMessage): RequestOrigin {
+  let origin = cache.get(request);
+  if (!origin) {
+    const soc = request.socket;
+    const socketEndpoints = getSocketEndpoints(soc);
+    origin = {
+      localAddress: socketEndpoints.localAddress,
+      localPort: socketEndpoints.localPort,
+      remoteAddress: socketEndpoints.remoteAddress,
+      remotePort: socketEndpoints.remotePort,
+      host: request.headers["host"],
+      encrypted: !!(request.socket as TLSSocket).encrypted,
+    };
+
+    const header = request.headers["forwarded"];
+    if (header) {
+      const ip = parse(socketEndpoints.remoteAddress);
+      if (
+        cidrs.some((cidr) => ip.kind() === cidr[0].kind() && ip.match(cidr))
+      ) {
+        const parsed = parseForwardedHeader(header);
+
+        if (parsed["proto"] === "https") {
+          origin.encrypted = true;
+          origin.localPort = 443;
+        } else if (parsed["proto"] === "http") {
+          origin.encrypted = false;
+          origin.localPort = 80;
+        }
+
+        if (parsed["host"]) {
+          origin.host = parsed["host"];
+          const [, port] = parsed["host"].split(":", 2);
+          origin.localPort = +port || origin.localPort;
+        }
+
+        if (parsed["for"]) {
+          if (parsed["for"].startsWith("[")) {
+            const i = parsed["for"].lastIndexOf("]");
+            if (i >= 0) {
+              origin.remoteAddress = parsed["for"].slice(1, i);
+              origin.remotePort =
+                parseInt(parsed["for"].slice(i + 2)) || origin.remotePort;
+            }
+          } else {
+            const i = parsed["for"].lastIndexOf(":");
+            if (i >= 0) {
+              origin.remoteAddress = parsed["for"].slice(0, i);
+              origin.remotePort =
+                parseInt(parsed["for"].slice(i + 1)) || origin.remotePort;
+            } else {
+              origin.remoteAddress = parsed["for"];
+            }
+          }
+        }
+
+        if (parsed["by"]) {
+          if (parsed["by"].startsWith("[")) {
+            const i = parsed["by"].lastIndexOf("]");
+            if (i >= 0) {
+              origin.localAddress = parsed["by"].slice(1, i);
+              origin.localPort =
+                parseInt(parsed["by"].slice(i + 2)) || origin.localPort;
+            }
+          } else {
+            const i = parsed["by"].lastIndexOf(":");
+            if (i >= 0) {
+              origin.localAddress = parsed["by"].slice(0, i);
+              origin.localPort =
+                parseInt(parsed["by"].slice(i + 1)) || origin.localPort;
+            } else {
+              origin.localAddress = parsed["by"];
+            }
+          }
+        }
+      }
+    }
+
+    cache.set(request, origin);
+  }
+  return origin;
+}

lib/fs.coffee

@@ -1,49 +0,0 @@
-###
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-config = require './config'
-url = require 'url'
-mongodb = require 'mongodb'
-querystring = require 'querystring'
-db = require './db'
-
-
-listener = (request, response) ->
-  urlParts = url.parse(request.url, true)
-  if request.method == 'GET'
-    filename = querystring.unescape(urlParts.pathname.substring(1))
-    gs = new mongodb.GridStore(db.mongoDb, filename, 'r', {})
-    gs.open((err, gs) ->
-      if err
-        response.writeHead(404)
-        response.end()
-        return
-      stream = gs.stream(true)
-      response.writeHead(200, {'Content-Type' : 'application/octet-stream', 'Content-Length' : gs.length})
-      stream.pipe(response)
-    )
-  else
-    response.writeHead(405, {'Allow': 'GET'})
-    response.end('405 Method Not Allowed')
-
-
-exports.listener = listener

lib/fs.ts

@@ -0,0 +1,203 @@
+import * as url from "node:url";
+import { IncomingMessage, ServerResponse } from "node:http";
+import { PassThrough, pipeline, Readable } from "node:stream";
+import { createHash } from "node:crypto";
+import { filesBucket, collections } from "./db/db.ts";
+import * as logger from "./logger.ts";
+import { getRequestOrigin } from "./forwarded.ts";
+import memoize from "./common/memoize.ts";
+
+const getFile = memoize(
+  async (
+    etag: string,
+    size: number,
+    filename: string,
+  ): Promise<Iterable<Buffer>> => {
+    const chunks: Buffer[] = [];
+    // Using for-await over the download stream can throw ERR_STREAM_PREMATURE_CLOSE
+    // for very small files. Possibly a bug in MongoDB driver or Nodejs itself.
+    // Using a PassThrough stream to avoid this.
+    const downloadStream = pipeline(
+      filesBucket.openDownloadStreamByName(filename),
+      new PassThrough(),
+      (err) => {
+        if (err) throw err;
+      },
+    );
+    for await (const chunk of downloadStream) chunks.push(chunk);
+    // Node 12-14 don't throw error when stream is closed prematurely.
+    // However, we don't need to check for that since we're checking file size.
+    if (size !== chunks.reduce((a, b) => a + b.length, 0))
+      throw new Error("File size mismatch");
+    return chunks;
+  },
+);
+
+async function* partialContent(
+  chunks: Iterable<Buffer>,
+  start: number,
+  end: number,
+): AsyncIterable<Buffer> {
+  let bytesToSkip = start;
+  let bytesToRead = end - start;
+
+  for (let chunk of chunks) {
+    if (bytesToRead <= 0) return;
+    if (bytesToSkip >= chunk.length) {
+      bytesToSkip -= chunk.length;
+      continue;
+    }
+    chunk = chunk.subarray(bytesToSkip, bytesToSkip + bytesToRead);
+    bytesToRead -= chunk.length;
+    bytesToSkip = 0;
+    yield chunk;
+  }
+}
+
+function generateETag(file: {
+  _id: string;
+  uploadDate: Date;
+  length: number;
+}): string {
+  const hash = createHash("md5");
+  hash.update(`${file._id}-${file.uploadDate.getTime()}-${file.length}`);
+  return hash.digest("hex");
+}
+
+function matchEtag(etag: string, header: string): boolean {
+  for (let t of header.split(",")) {
+    t = t.trim();
+    if (t.startsWith("W/")) t = t.substring(2);
+    try {
+      t = JSON.parse(t);
+    } catch (e) {
+      // Ignore
+    }
+    if (t === "*") return true;
+    if (etag === t) return true;
+  }
+  return false;
+}
+
+export async function listener(
+  request: IncomingMessage,
+  response: ServerResponse,
+): Promise<void> {
+  if (request.method !== "GET" && request.method !== "HEAD") {
+    response.writeHead(405, { Allow: "GET, HEAD" });
+    response.end("405 Method Not Allowed");
+    return;
+  }
+
+  const urlParts = url.parse(request.url, true);
+  const filename = decodeURIComponent(urlParts.pathname.substring(1));
+
+  const log = {
+    message: "Fetch file",
+    filename: filename,
+    remoteAddress: getRequestOrigin(request).remoteAddress,
+    method: request.method,
+  };
+
+  const file = await collections.files.findOne({ _id: filename });
+
+  if (!file) {
+    response.writeHead(404);
+    response.end();
+    log.message += " not found";
+    logger.accessError(log);
+    return;
+  }
+
+  logger.accessInfo(log);
+
+  const etag = generateETag(file);
+  const lastModified = file["uploadDate"];
+  lastModified.setMilliseconds(0);
+
+  let status = 200;
+  let start = 0;
+  let end = file.length;
+
+  if (request.headers["if-match"])
+    if (!matchEtag(etag, request.headers["if-match"])) status = 412;
+
+  if (request.headers["if-unmodified-since"]) {
+    const d = new Date(request.headers["if-unmodified-since"]);
+    if (lastModified > d) status = 412;
+  }
+
+  if (request.headers["if-none-match"]) {
+    if (matchEtag(etag, request.headers["if-none-match"])) status = 304;
+  } else if (request.headers["if-modified-since"]) {
+    const d = new Date(request.headers["if-modified-since"]);
+    if (lastModified <= d) status = 304;
+  }
+
+  if (request.headers.range && status === 200) {
+    const match = request.headers.range.match(/^bytes=(\d*)-(\d*)$/);
+    status = 416;
+    if (match && (match[1] || match[2])) {
+      if (match[2]) end = parseInt(match[2]) + 1;
+      if (match[1]) start = parseInt(match[1]);
+      else start = file.length - parseInt(match[2]);
+      if (start < end && end <= file.length) status = 206;
+    }
+
+    if (request.headers["if-range"]) {
+      const h = request.headers["if-range"] as string;
+      const d = new Date(h);
+      if (!matchEtag(etag, h) && !(lastModified <= d)) {
+        status = 200;
+        start = 0;
+        end = file.length;
+      }
+    }
+  }
+
+  if (status === 412) {
+    response.writeHead(412);
+    response.end();
+    return;
+  }
+
+  if (status === 304) {
+    response.writeHead(304, {
+      ETag: etag,
+      "Last-Modified": lastModified.toUTCString(),
+    });
+    response.end();
+    return;
+  }
+
+  if (status === 416) {
+    response.writeHead(416, {
+      "Content-Range": `bytes */${file.length}`,
+      "Content-Length": "0",
+    });
+    response.end();
+    return;
+  }
+
+  response.writeHead(status, {
+    "Content-Type": "application/octet-stream",
+    "Content-Length": end - start,
+    "Accept-Ranges": "bytes",
+    ETag: etag,
+    "Last-Modified": lastModified.toUTCString(),
+    ...(status === 206 && {
+      "Content-Range": `bytes ${start}-${end - 1}/${file.length}`,
+    }),
+  });
+
+  if (request.method === "HEAD") {
+    response.end();
+    return;
+  }
+
+  const chunks = await getFile(etag, file.length, filename);
+
+  pipeline(Readable.from(partialContent(chunks, start, end)), response, () => {
+    // Ignore errors resulting from client disconnecting
+  });
+}

lib/gpn-heuristic.js

@@ -1,82 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const WILDCARD_MULTIPLIER = 2;
-const UNDISCOVERED_DEPTH = 7;
-
-// Simple heuristic to estimate GPN count given a set of patterns to be
-// discovered. Used to decide whether to use nextLevel = false in GPN.
-// gpnPatterns is [pattern, flags]
-// pattern is a path (array)
-// flags is an int where its bits mark the segments in the pattern that
-// need refreshing. Leading 0s indicate that the pattern up to that
-// point has been discovered.
-function estimateGpnCount(gpnPatterns, depth = 0) {
-  const children = {};
-  const wildcardChildren = [];
-  let wildcardDiscovered = false;
-  let gpnCount = 0;
-
-  for (let pattern of gpnPatterns) {
-    const path = pattern[0];
-    const flags = pattern[1] >> depth;
-
-    let k = path[depth];
-
-    if (!k) {
-      if (flags & 1)
-        gpnCount = 1;
-      continue;
-    }
-
-    if (flags & 1) {
-      gpnCount = 1;
-
-      if (depth > UNDISCOVERED_DEPTH)
-        continue;
-    }
-    else if (k === "*")
-      wildcardDiscovered = true;
-
-    if (k === "*") {
-      wildcardChildren.push(pattern);
-    }
-    else {
-      children[k] = children[k] || [];
-      children[k].push(pattern);
-    }
-  }
-
-  let wildcardGpnCount = 0;
-  if (!wildcardDiscovered && wildcardChildren.length)
-      wildcardGpnCount += estimateGpnCount(wildcardChildren, depth + 1) * WILDCARD_MULTIPLIER;
-
-  for (let k of Object.keys(children)) {
-    let c = estimateGpnCount(children[k].concat(wildcardChildren), depth + 1);
-    wildcardGpnCount -= c;
-    gpnCount += c;
-  }
-
-  gpnCount += Math.max(0, wildcardGpnCount);
-
-  return gpnCount;
-}
-
-exports.estimateGpnCount = estimateGpnCount;

lib/gpn-heuristic.ts

@@ -0,0 +1,64 @@
+import Path from "./common/path.ts";
+
+const WILDCARD_MULTIPLIER = 2;
+const UNDISCOVERED_DEPTH = 7;
+
+// Simple heuristic to estimate GPN count given a set of patterns to be
+// discovered. Used to decide whether to use nextLevel = false in GPN.
+// gpnPatterns is [pattern, flags]
+// pattern is a path (array)
+// flags is an int where its bits mark the segments in the pattern that
+// need refreshing. Leading 0s indicate that the pattern up to that
+// point has been discovered.
+export function estimateGpnCount(
+  gpnPatterns: [Path, number][],
+  depth = 0,
+): number {
+  const children: { [segment: string]: [Path, number][] } = {};
+  const wildcardChildren: [Path, number][] = [];
+  let wildcardDiscovered = false;
+  let gpnCount = 0;
+
+  for (const pattern of gpnPatterns) {
+    const path = pattern[0];
+    const flags = pattern[1] >> depth;
+
+    const k = path.segments[depth] as string;
+
+    if (!k) {
+      if (flags & 1) gpnCount = 1;
+      continue;
+    }
+
+    if (flags & 1) {
+      gpnCount = 1;
+
+      if (depth > UNDISCOVERED_DEPTH) continue;
+    } else if (k === "*") {
+      wildcardDiscovered = true;
+    }
+
+    if (k === "*") {
+      wildcardChildren.push(pattern);
+    } else {
+      children[k] = children[k] || [];
+      children[k].push(pattern);
+    }
+  }
+
+  let wildcardGpnCount = 0;
+  if (!wildcardDiscovered && wildcardChildren.length) {
+    wildcardGpnCount +=
+      estimateGpnCount(wildcardChildren, depth + 1) * WILDCARD_MULTIPLIER;
+  }
+
+  for (const k of Object.keys(children)) {
+    const c = estimateGpnCount(children[k].concat(wildcardChildren), depth + 1);
+    wildcardGpnCount -= c;
+    gpnCount += c;
+  }
+
+  gpnCount += Math.max(0, wildcardGpnCount);
+
+  return gpnCount;
+}

lib/init.ts

@@ -0,0 +1,416 @@
+import { getRevision, getUiConfig, getUsers } from "./ui/local-cache.ts";
+import { generateSalt, hashPassword } from "./auth.ts";
+import { collections } from "./db/db.ts";
+import {
+  putConfig,
+  putPermission,
+  putPreset,
+  putProvision,
+  putUser,
+} from "./ui/db.ts";
+import { del } from "./cache.ts";
+
+interface Status {
+  users: boolean;
+  presets: boolean;
+  filters: boolean;
+  device: boolean;
+  index: boolean;
+  overview: boolean;
+}
+
+const BOOTSTRAP_SCRIPT = `
+const now = Date.now();
+
+// Clear cached data model to force a refresh
+clear("Device", now);
+clear("InternetGatewayDevice", now);
+`.trim();
+
+const DEFAULT_SCRIPT = `
+const hourly = Date.now(3600000);
+
+// Refresh basic parameters hourly
+declare("InternetGatewayDevice.DeviceInfo.HardwareVersion", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.DeviceInfo.SoftwareVersion", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID", {path: hourly, value: hourly});
+// Don't refresh password field periodically because CPEs always report blank passowrds for security reasons
+declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase", {path: hourly, value: 1});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress", {path: hourly, value: hourly});
+`.trim();
+
+const INFORM_SCRIPT = `
+// Device ID as user name
+const username = declare("DeviceID.ID", {value: 1}).value[0]
+
+// Password will be fixed for a given device because Math.random() is seeded with device ID by default.
+const password = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER).toString(36);
+
+const informInterval = 300;
+
+// Refresh values daily
+const daily = Date.now(86400000);
+
+// Unique inform offset per device for better load distribution
+const informTime = daily % 86400000;
+
+declare("InternetGatewayDevice.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
+declare("InternetGatewayDevice.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
+
+declare("Device.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
+declare("Device.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
+declare("Device.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
+declare("Device.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
+declare("Device.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
+`.trim();
+
+export async function getStatus(): Promise<Status> {
+  const [configSnapshot, presetCount] = await Promise.all([
+    getRevision(),
+    collections.presets.countDocuments(),
+  ]);
+  const users = getUsers(configSnapshot);
+  const ui = getUiConfig(configSnapshot);
+
+  return {
+    users: !Object.keys(users).length,
+    presets: !presetCount,
+    filters: !Object.keys(ui["filters"]).length,
+    device: !Object.keys(ui["device"]).length,
+    index: !Object.keys(ui["index"]).length,
+    overview: !Object.keys(ui["overview"]).length,
+  };
+}
+
+export async function seed(options: Record<string, boolean>): Promise<void> {
+  const resources = {};
+  const proms = [];
+
+  if (options.users) {
+    resources["permissions"] = [
+      { role: "admin", resource: "devices", access: 3, validate: "true" },
+      { role: "admin", resource: "faults", access: 3, validate: "true" },
+      { role: "admin", resource: "files", access: 3, validate: "true" },
+      { role: "admin", resource: "presets", access: 3, validate: "true" },
+      { role: "admin", resource: "provisions", access: 3, validate: "true" },
+      { role: "admin", resource: "config", access: 3, validate: "true" },
+      { role: "admin", resource: "permissions", access: 3, validate: "true" },
+      { role: "admin", resource: "users", access: 3, validate: "true" },
+      {
+        role: "admin",
+        resource: "virtualParameters",
+        access: 3,
+        validate: "true",
+      },
+    ];
+
+    resources["users"] = [
+      { username: "admin", password: "admin", roles: ["admin"] },
+    ];
+  }
+
+  if (options.filters) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.filters.0.label", value: "'Serial number'" },
+      { _id: "ui.filters.0.parameter", value: "DeviceID.SerialNumber" },
+      { _id: "ui.filters.0.type", value: "'string'" },
+      { _id: "ui.filters.1.label", value: "'Product class'" },
+      { _id: "ui.filters.1.parameter", value: "DeviceID.ProductClass" },
+      { _id: "ui.filters.1.type", value: "'string'" },
+      { _id: "ui.filters.2.label", value: "'Tag'" },
+      { _id: "ui.filters.2.type", value: "'tag'" },
+    ]);
+  }
+
+  if (options.device) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.device.0.type", value: "'tags'" },
+      { _id: "ui.device.1.type", value: "'ping'" },
+      { _id: "ui.device.2.type", value: "'parameter-list'" },
+      { _id: "ui.device.2.parameters.0.type", value: "'container'" },
+      { _id: "ui.device.2.parameters.0.element", value: "'span.inform'" },
+      { _id: "ui.device.2.parameters.0.label", value: "'Last inform'" },
+      {
+        _id: "ui.device.2.parameters.0.components.0.type",
+        value: "'parameter'",
+      },
+      { _id: "ui.device.2.parameters.0.components.1.chart", value: "'online'" },
+      {
+        _id: "ui.device.2.parameters.0.components.1.type",
+        value: "'overview-dot'",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.type",
+        value: "'summon-button'",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.0",
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.1",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.2",
+        value:
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.3",
+        value:
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.4",
+        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.5",
+        value:
+          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.6",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.7",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.8",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress",
+      },
+      {
+        _id: "ui.device.2.parameters.0.parameter",
+        value: "DATE_STRING(Events.Inform)",
+      },
+      { _id: "ui.device.2.parameters.1.label", value: "'Serial number'" },
+      {
+        _id: "ui.device.2.parameters.1.parameter",
+        value: "DeviceID.SerialNumber",
+      },
+      { _id: "ui.device.2.parameters.2.label", value: "'Product class'" },
+      {
+        _id: "ui.device.2.parameters.2.parameter",
+        value: "DeviceID.ProductClass",
+      },
+      { _id: "ui.device.2.parameters.3.label", value: "'OUI'" },
+      {
+        _id: "ui.device.2.parameters.3.parameter",
+        value: "DeviceID.OUI",
+      },
+      { _id: "ui.device.2.parameters.4.label", value: "'Manufacturer'" },
+      {
+        _id: "ui.device.2.parameters.4.parameter",
+        value: "DeviceID.Manufacturer",
+      },
+      { _id: "ui.device.2.parameters.5.label", value: "'Hardware version'" },
+      {
+        _id: "ui.device.2.parameters.5.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
+      },
+      { _id: "ui.device.2.parameters.6.label", value: "'Software version'" },
+      {
+        _id: "ui.device.2.parameters.6.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
+      },
+      { _id: "ui.device.2.parameters.7.label", value: "'MAC'" },
+      {
+        _id: "ui.device.2.parameters.7.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress",
+      },
+      { _id: "ui.device.2.parameters.8.label", value: "'IP'" },
+      {
+        _id: "ui.device.2.parameters.8.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
+      },
+      { _id: "ui.device.2.parameters.9.label", value: "'WLAN SSID'" },
+      {
+        _id: "ui.device.2.parameters.9.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
+      },
+      { _id: "ui.device.2.parameters.10.label", value: "'WLAN passphrase'" },
+      {
+        _id: "ui.device.2.parameters.10.parameter",
+        value:
+          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
+      },
+      { _id: "ui.device.3.type", value: "'parameter-table'" },
+      { _id: "ui.device.3.label", value: "'LAN hosts'" },
+      {
+        _id: "ui.device.3.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host",
+      },
+      { _id: "ui.device.3.childParameters.0.label", value: "'Host name'" },
+      { _id: "ui.device.3.childParameters.0.parameter", value: "HostName" },
+      { _id: "ui.device.3.childParameters.1.label", value: "'IP address'" },
+      { _id: "ui.device.3.childParameters.1.parameter", value: "IPAddress" },
+      { _id: "ui.device.3.childParameters.2.label", value: "'MAC address'" },
+      { _id: "ui.device.3.childParameters.2.parameter", value: "MACAddress" },
+      { _id: "ui.device.4.type", value: "'container'" },
+      { _id: "ui.device.4.element", value: "'div'" },
+      { _id: "ui.device.4.components.0.components.0", value: "'Faults'" },
+      { _id: "ui.device.4.components.0.element", value: "'h3'" },
+      { _id: "ui.device.4.components.0.type", value: "'container'" },
+      { _id: "ui.device.4.components.1.type", value: "'device-faults'" },
+      { _id: "ui.device.5.type", value: "'container'" },
+      { _id: "ui.device.5.element", value: "'div.container-full-width'" },
+      {
+        _id: "ui.device.5.components.0.components.0",
+        value: "'All parameters'",
+      },
+      { _id: "ui.device.5.components.0.element", value: "'h3'" },
+      { _id: "ui.device.5.components.0.type", value: "'container'" },
+      { _id: "ui.device.5.components.1.type", value: "'all-parameters'" },
+      { _id: "ui.device.6.type", value: "'device-actions'" },
+    ]);
+  }
+
+  if (options.index) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.index.0.type", value: "'device-link'" },
+      { _id: "ui.index.0.label", value: "'Serial number'" },
+      { _id: "ui.index.0.parameter", value: "DeviceID.SerialNumber" },
+      { _id: "ui.index.0.components.0.type", value: "'parameter'" },
+      { _id: "ui.index.1.label", value: "'Product class'" },
+      { _id: "ui.index.1.parameter", value: "DeviceID.ProductClass" },
+      { _id: "ui.index.2.label", value: "'Software version'" },
+      {
+        _id: "ui.index.2.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
+      },
+      { _id: "ui.index.3.label", value: "'IP'" },
+      {
+        _id: "ui.index.3.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
+      },
+      { _id: "ui.index.4.label", value: "'SSID'" },
+      {
+        _id: "ui.index.4.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
+      },
+      { _id: "ui.index.5.type", value: "'container'" },
+      { _id: "ui.index.5.label", value: "'Last inform'" },
+      { _id: "ui.index.5.element", value: "'span.inform'" },
+      { _id: "ui.index.5.parameter", value: "DATE_STRING(Events.Inform)" },
+      { _id: "ui.index.5.components.0.type", value: "'parameter'" },
+      { _id: "ui.index.5.components.1.chart", value: "'online'" },
+      { _id: "ui.index.5.components.1.type", value: "'overview-dot'" },
+      { _id: "ui.index.6.type", value: "'tags'" },
+      { _id: "ui.index.6.label", value: "'Tags'" },
+      { _id: "ui.index.6.parameter", value: "Tags" },
+      { _id: "ui.index.6.unsortable", value: "true" },
+      { _id: "ui.index.6.writable", value: "false" },
+    ]);
+  }
+
+  if (options.overview) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.overview.charts.online.label", value: "'Online status'" },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.color",
+        value: "'#31a354'",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.filter",
+        value: "Events.Inform > NOW() - 5 * 60 * 1000",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.label",
+        value: "'Online now'",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.color",
+        value: "'#a1d99b'",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.filter",
+        value:
+          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.label",
+        value: "'Past 24 hours'",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.color",
+        value: "'#e5f5e0'",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.filter",
+        value:
+          "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.label",
+        value: "'Others'",
+      },
+      { _id: "ui.overview.groups.online.label", value: "''" },
+      { _id: "ui.overview.groups.online.charts.0", value: "'online'" },
+    ]);
+  }
+
+  if (options.presets) {
+    resources["presets"] = [
+      {
+        _id: "bootstrap",
+        weight: 0,
+        channel: "bootstrap",
+        events: "0 BOOTSTRAP",
+        provision: "bootstrap",
+      },
+      { _id: "default", weight: 0, channel: "default", provision: "default" },
+      { _id: "inform", weight: 0, channel: "inform", provision: "inform" },
+    ];
+
+    resources["provisions"] = [
+      { _id: "bootstrap", script: BOOTSTRAP_SCRIPT },
+      { _id: "default", script: DEFAULT_SCRIPT },
+      { _id: "inform", script: INFORM_SCRIPT },
+    ];
+  }
+
+  if (resources["permissions"]) {
+    for (const p of resources["permissions"]) {
+      p["_id"] = `${p["role"]}:${p["resource"]}:${p["access"]}`;
+      proms.push(putPermission(p["_id"], p));
+    }
+  }
+
+  if (resources["users"]) {
+    for (const u of resources["users"]) {
+      u["salt"] = await generateSalt(64);
+      u["password"] = await hashPassword(u["password"], u["salt"]);
+      u["roles"] = (u["roles"] || []).join(",");
+      u["_id"] = u["username"];
+      delete u["username"];
+      proms.push(putUser(u["_id"], u));
+    }
+  }
+
+  if (resources["provisions"]) {
+    for (const p of resources["provisions"])
+      proms.push(putProvision(p["_id"], p));
+  }
+
+  if (resources["presets"])
+    for (const p of resources["presets"]) proms.push(putPreset(p["_id"], p));
+
+  if (resources["config"])
+    for (const c of resources["config"]) proms.push(putConfig(c["_id"], c));
+
+  await proms;
+  await Promise.all([del("ui-local-cache-hash"), del("cwmp-local-cache-hash")]);
+}

lib/instance-set.js

@@ -1,138 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-
-class InstanceSet {
-
-  constructor() {
-    this.set = new Set();
-  }
-
-  add(instance) {
-    this.set.add(instance);
-  }
-
-  delete(instance) {
-    this.set.delete(instance);
-  }
-
-  superset(instance) {
-    let res = [];
-    for (let inst of this.set) {
-      let match = true;
-      for (let k in instance) {
-        if (inst[k] !== instance[k]) {
-          match = false;
-          break;
-        }
-      }
-
-      if (match)
-        res.push(inst);
-    }
-
-    res.sort(function(a, b) {
-      let keysA = Object.keys(a);
-      let keysB = Object.keys(b);
-
-      if (keysA.length != keysB.length)
-        return keysB.length - keysA.length;
-
-      keysA.sort();
-      keysB.sort();
-
-      for (let i = 0; i < keysA.length; ++ i) {
-        if (keysA[i] > keysB[i])
-          return 1;
-        else if (keysA[i] < keysB[i])
-          return -1;
-        else if (a[keysA[i]] > b[keysB[i]])
-          return 1;
-        else if (a[keysA[i]] < b[keysB[i]])
-          return -1;
-      }
-
-      return 0;
-    });
-
-    return res;
-  }
-
-  subset(instance) {
-    let res = [];
-
-    for (let inst of this.set) {
-      let match = true;
-      for (let k in inst) {
-        if (inst[k] !== instance[k]) {
-          match = false;
-          break;
-        }
-      }
-
-      if (match)
-        res.push(inst);
-    }
-
-    res.sort(function(a, b) {
-      let keysA = Object.keys(a);
-      let keysB = Object.keys(b);
-
-      if (keysA.length != keysB.length)
-        return keysA.length - keysB.length;
-
-      keysA.sort();
-      keysB.sort();
-
-      for (let i = 0; i < keysA.length; ++ i) {
-        if (keysA[i] > keysB[i])
-          return 1;
-        else if (keysA[i] < keysB[i])
-          return -1;
-        else if (a[keysA[i]] > b[keysB[i]])
-          return 1;
-        else if (a[keysA[i]] < b[keysB[i]])
-          return -1;
-      }
-
-      return 0;
-    });
-
-    return res;
-  }
-
-  forEach(callback) {
-    return this.set.forEach(callback);
-  }
-
-  values() {
-    return this.set.values();
-  }
-
-  clear() {
-    this.set.clear();
-  }
-
-  get size() {
-    return this.set.size;
-  }
-}
-
-module.exports = InstanceSet;

lib/instance-set.ts

@@ -0,0 +1,112 @@
+interface Instance {
+  [name: string]: string;
+}
+
+export default class InstanceSet {
+  declare private set: Set<Instance>;
+
+  public constructor() {
+    this.set = new Set();
+  }
+
+  public add(instance: Instance): void {
+    this.set.add(instance);
+  }
+
+  public delete(instance: Instance): void {
+    this.set.delete(instance);
+  }
+
+  public superset(instance: Instance): Instance[] {
+    const res = [];
+    for (const inst of this.set) {
+      let match = true;
+      for (const k in instance) {
+        if (inst[k] !== instance[k]) {
+          match = false;
+          break;
+        }
+      }
+
+      if (match) res.push(inst);
+    }
+
+    res.sort((a, b) => {
+      const keysA = Object.keys(a);
+      const keysB = Object.keys(b);
+
+      if (keysA.length !== keysB.length) return keysB.length - keysA.length;
+
+      keysA.sort();
+      keysB.sort();
+
+      for (let i = 0; i < keysA.length; ++i) {
+        if (keysA[i] > keysB[i]) return 1;
+        else if (keysA[i] < keysB[i]) return -1;
+        else if (a[keysA[i]] > b[keysB[i]]) return 1;
+        else if (a[keysA[i]] < b[keysB[i]]) return -1;
+      }
+
+      return 0;
+    });
+
+    return res;
+  }
+
+  public subset(instance: Instance): Instance[] {
+    const res = [];
+
+    for (const inst of this.set) {
+      let match = true;
+      for (const k in inst) {
+        if (inst[k] !== instance[k]) {
+          match = false;
+          break;
+        }
+      }
+
+      if (match) res.push(inst);
+    }
+
+    res.sort((a, b) => {
+      const keysA = Object.keys(a);
+      const keysB = Object.keys(b);
+
+      if (keysA.length !== keysB.length) return keysA.length - keysB.length;
+
+      keysA.sort();
+      keysB.sort();
+
+      for (let i = 0; i < keysA.length; ++i) {
+        if (keysA[i] > keysB[i]) return 1;
+        else if (keysA[i] < keysB[i]) return -1;
+        else if (a[keysA[i]] > b[keysB[i]]) return 1;
+        else if (a[keysA[i]] < b[keysB[i]]) return -1;
+      }
+
+      return 0;
+    });
+
+    return res;
+  }
+
+  public [Symbol.iterator](): IterableIterator<Instance> {
+    return this.set.values();
+  }
+
+  public forEach(callback: (instance: Instance) => void): void {
+    this.set.forEach(callback);
+  }
+
+  public values(): IterableIterator<Instance> {
+    return this.set.values();
+  }
+
+  public clear(): void {
+    this.set.clear();
+  }
+
+  public get size(): number {
+    return this.set.size;
+  }
+}

lib/local-cache.js

@@ -1,341 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const vm = require("vm");
-const crypto = require("crypto");
-
-const db = require("./db");
-const cache = require("./cache");
-const query = require("./query");
-const logger = require("./logger");
-const scheduling = require("./scheduling");
-
-const REFRESH = 3000;
-
-let nextRefresh = 1;
-let hash = null;
-let presets, provisions, virtualParameters, files;
-
-
-function computeHash() {
-  // MD5 hash for presets, provisions, virtual parameters for detecting changes
-  let h = crypto.createHash("md5");
-  for (let p of presets) {
-    h.update(JSON.stringify(p.name));
-    h.update(JSON.stringify(p.channel));
-    h.update(JSON.stringify(p.schedule));
-    h.update(JSON.stringify(p.events));
-    h.update(JSON.stringify(p.precondition));
-    h.update(JSON.stringify(p.provisions));
-  }
-
-  let keys;
-
-  keys = Object.keys(provisions).sort();
-  h.update(JSON.stringify(keys));
-  for (let k of keys)
-    h.update(provisions[k].md5);
-
-  keys = Object.keys(virtualParameters).sort();
-  h.update(JSON.stringify(keys));
-  for (let k of keys)
-    h.update(virtualParameters[k].md5);
-
-  hash = h.digest("hex");
-}
-
-
-function refresh(callback) {
-  if (!nextRefresh)
-    return setTimeout(function() {refresh(callback);}, 20);
-
-  nextRefresh = 0;
-  const now = Date.now();
-
-  cache.get("presets_hash", function(err, res) {
-    if (err)
-      return callback(err);
-
-    if (hash && res === hash) {
-      nextRefresh = now + (REFRESH - (now % REFRESH));
-      return callback();
-    }
-
-    cache.lock("presets_hash_lock", 3, function(err, unlockOrExtend) {
-      if (err)
-        return callback(err);
-
-      let counter = 3;
-
-      counter += 2;
-      db.getPresets(function(err, res) {
-        if (err) {
-          if (counter & 1)
-            callback(err);
-          return counter = 0;
-        }
-
-        db.getObjects(function(err, objects) {
-          if (err) {
-            if (counter & 1)
-              callback(err);
-            return counter = 0;
-          }
-
-          // If no keys are defined, consider all parameters as keys to keep the
-          // same behavior from v1.0
-          for (let obj of objects) {
-            if (!obj["_keys"] || !obj["_keys"].length)
-              obj["_keys"] = Object.keys(obj).filter(k => !k.startsWith("_"));
-          }
-
-          res.sort((a, b) => {
-            if (a.weight === b.weight)
-              return a._id > b._id;
-            else
-              return a.weight - b.weight;
-          });
-
-          presets = [];
-          for (let preset of res) {
-            let schedule = null;
-            if (preset.schedule) {
-              let parts = preset.schedule.trim().split(/\s+/);
-              schedule = {
-                md5: crypto.createHash("md5").update(preset.schedule).digest("hex")
-              };
-
-              try {
-                schedule.duration = +(parts.shift()) * 1000;
-                schedule.schedule = scheduling.parseCron(parts.join(" "));
-              }
-              catch (err) {
-                logger.warn({
-                  message: "Invalid preset schedule",
-                  preset: preset._id,
-                  schedule: preset.schedule
-                });
-                schedule.schedule = false;
-              }
-            }
-
-            let events = preset.events || {};
-            let precondition = query.convertMongoQueryToFilters(JSON.parse(preset.precondition));
-            let _provisions = preset.provisions || [];
-
-            // Generate provisions from the old configuration format
-            for (let c of preset.configurations) {
-              switch (c.type) {
-                case "age":
-                  _provisions.push(["refresh", c.name, c.age]);
-                  break;
-
-                case "value":
-                  _provisions.push(["value", c.name, c.value]);
-                  break;
-
-                case "add_tag":
-                  _provisions.push(["tag", c.tag, true]);
-                  break;
-
-                case "delete_tag":
-                  _provisions.push(["tag", c.tag, false]);
-                  break;
-
-                case "provision":
-                  _provisions.push([c.name].concat(c.args || []));
-                  break;
-
-                case "add_object":
-                  for (let obj of objects) {
-                    if (obj["_id"] === c.object) {
-                      let alias = obj["_keys"].map((k) => `${k}:${JSON.stringify(obj[k])}`).join(",");
-                      let p = `${c.name}.[${alias}]`;
-                      _provisions.push(["instances", p, 1]);
-
-                      for (let k in obj)
-                        if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
-                          _provisions.push(["value", `${p}.${k}`, obj[k]]);
-                    }
-                  }
-                  break;
-
-                case "delete_object":
-                  for (let obj of objects) {
-                    if (obj["_id"] === c.object) {
-                      let alias = obj["_keys"].map((k) => `${k}:${JSON.stringify(obj[k])}`).join(",");
-                      let p = `${c.name}.[${alias}]`;
-                      _provisions.push(["instances", p, 0]);
-                    }
-                  }
-                  break;
-
-                default:
-                  if (counter & 1)
-                    callback(new Error(`Unknown configuration type ${c.type}`));
-                  return counter = 0;
-                  break;
-              }
-            }
-
-            presets.push({
-              name: preset._id,
-              channel: preset.channel || "default",
-              schedule: schedule,
-              events: events,
-              precondition: precondition,
-              provisions: _provisions
-            });
-          }
-
-          if ((counter -= 2) === 1) {
-            computeHash();
-            cache.set("presets_hash", hash, 300, function(err) {
-              unlockOrExtend(0);
-              nextRefresh = now + (REFRESH - (now % REFRESH));
-              callback(err);
-            });
-          }
-        });
-      });
-
-      counter += 2;
-      db.getProvisions(function(err, res) {
-        if (err) {
-          if (counter & 1)
-            callback(err);
-          return counter = 0;
-        }
-
-        provisions = {};
-        for (let r of res) {
-          provisions[r._id] = {};
-          provisions[r._id].md5 = crypto.createHash("md5").update(r.script).digest("hex");
-          provisions[r._id].script = new vm.Script(`"use strict";(function(){\n${r.script}\n})();`, {filename: r._id, lineOffset: -1, timeout: 50});
-        }
-
-        if ((counter -= 2) === 1) {
-          computeHash();
-          cache.set("presets_hash", hash, 300, function(err) {
-            unlockOrExtend(0);
-            nextRefresh = now + (REFRESH - (now % REFRESH));
-            callback(err);
-          });
-        }
-      });
-
-      counter += 2;
-      db.getVirtualParameters(function(err, res) {
-        if (err) {
-          if (counter & 1)
-            callback(err);
-          return counter = 0;
-        }
-
-        virtualParameters = {};
-        for (let r of res) {
-          virtualParameters[r._id] = {}
-          virtualParameters[r._id].md5 = crypto.createHash("md5").update(r.script).digest("hex");
-          virtualParameters[r._id].script = new vm.Script(`"use strict";(function(){\n${r.script}\n})();`, {filename: r._id, lineOffset: -1, timeout: 50});
-        }
-
-        if ((counter -= 2) === 1) {
-          computeHash();
-          cache.set("presets_hash", hash, 300, function(err) {
-            unlockOrExtend(0);
-            nextRefresh = now + (REFRESH - (now % REFRESH));
-            callback(err);
-          });
-        }
-      });
-
-      counter += 2;
-      db.getFiles(function(err, res) {
-        if (err) {
-          if (counter & 1)
-            callback(err);
-          return counter = 0;
-        }
-
-        files = {};
-        for (let r of res) {
-          const id = r.filename || r._id.toString();
-          files[id] = {};
-          files[id].length = r.length;
-          files[id].md5 = r.md5;
-          files[id].contentType = r.contentType;
-        }
-
-        if ((counter -= 2) === 1) {
-          computeHash();
-          cache.set("presets_hash", hash, 300, function(err) {
-            unlockOrExtend(0);
-            nextRefresh = now + (REFRESH - (now % REFRESH));
-            callback(err);
-          });
-        }
-      });
-
-      if ((counter -= 2) === 1) {
-        computeHash();
-        cache.set("presets_hash", hash, 300, function(err) {
-          unlockOrExtend(0);
-          nextRefresh = now + (REFRESH - (now % REFRESH));
-          callback(err);
-        });
-      }
-    });
-  });
-}
-
-
-function getPresets(callback) {
-  if (Date.now() < nextRefresh)
-    return callback(null, hash, presets);
-
-  refresh(function(err) {
-    callback(err, hash, presets);
-  });
-}
-
-
-function getFiles(callback) {
-  if (Date.now() < nextRefresh)
-    return callback(null, hash, files);
-
-  refresh(function(err) {
-    callback(err, hash, files);
-  })
-}
-
-
-function getProvisionsAndVirtualParameters(callback) {
-  if (Date.now() < nextRefresh)
-    return callback(null, hash, provisions, virtualParameters);
-
-  refresh(function(err) {
-    callback(err, hash, provisions, virtualParameters);
-  });
-}
-
-
-exports.getPresets = getPresets;
-exports.getFiles = getFiles;
-exports.getProvisionsAndVirtualParameters = getProvisionsAndVirtualParameters;

lib/local-cache.ts

@@ -0,0 +1,73 @@
+import { setTimeoutPromise } from "./util.ts";
+import { get, set } from "./cache.ts";
+import { acquireLock, releaseLock } from "./lock.ts";
+
+const REFRESH = 5000;
+const EVICT_TIMEOUT = 120000;
+
+export class LocalCache<T> {
+  private nextRefresh = 1;
+  private currentRevision: string = null;
+  private snapshots: Map<string, T> = new Map();
+
+  constructor(
+    private cacheKey: string,
+    private callback: () => Promise<[string, T]>,
+  ) {}
+
+  async getRevision(): Promise<string> {
+    if (Date.now() > this.nextRefresh) await this.refresh();
+    return this.currentRevision;
+  }
+
+  hasRevision(revision: string): boolean {
+    return this.snapshots.has(revision);
+  }
+
+  get(revision: string): T {
+    const snapshot = this.snapshots.get(revision);
+    if (!snapshot) throw new Error("Cache snapshot does not exist");
+    return snapshot;
+  }
+
+  async refresh(): Promise<void> {
+    if (!this.nextRefresh) {
+      await setTimeoutPromise(20);
+      await this.refresh();
+      return;
+    }
+
+    const now = Date.now();
+    if (now < this.nextRefresh) return;
+    this.nextRefresh = 0;
+
+    const dbHash = await get(this.cacheKey);
+
+    if (this.currentRevision && dbHash === this.currentRevision) {
+      this.nextRefresh = now + (REFRESH - (now % REFRESH));
+      return;
+    }
+
+    const lockToken = await acquireLock(this.cacheKey, 5000);
+
+    const [hash, snapshot] = await this.callback();
+
+    if (this.currentRevision) {
+      const r = this.currentRevision;
+      const s = this.snapshots.get(r);
+      setTimeout(() => {
+        if (this.snapshots.get(r) === s) this.snapshots.delete(r);
+      }, EVICT_TIMEOUT).unref();
+    }
+
+    this.currentRevision = hash;
+    this.snapshots.set(hash, snapshot);
+
+    if (lockToken) {
+      if (hash !== dbHash) await set(this.cacheKey, hash, 300);
+      await releaseLock(this.cacheKey, lockToken);
+    }
+
+    this.nextRefresh = now + (REFRESH - (now % REFRESH));
+  }
+}

lib/lock.ts

@@ -0,0 +1,50 @@
+import { collections } from "./db/db.ts";
+
+const CLOCK_SKEW_TOLERANCE = 30000;
+
+export async function acquireLock(
+  lockName: string,
+  ttl: number,
+  timeout = 0,
+  token = Math.random().toString(36).slice(2),
+): Promise<string> {
+  try {
+    const now = Date.now();
+    const r = await collections.locks.findOneAndUpdate(
+      { _id: lockName, value: token },
+      {
+        $set: {
+          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
+        },
+        $currentDate: { timestamp: true },
+      },
+      { upsert: true, returnDocument: "after" },
+    );
+    if (Math.abs(r.value.timestamp.getTime() - now) > CLOCK_SKEW_TOLERANCE)
+      throw new Error("Database clock skew too great");
+  } catch (err) {
+    if (err.code !== 11000) throw err;
+    if (!(timeout > 0)) return null;
+    const w = 50 + Math.random() * 50;
+    await new Promise((resolve) => setTimeout(resolve, w));
+    return acquireLock(lockName, ttl, timeout - w, token);
+  }
+
+  return token;
+}
+
+export async function releaseLock(
+  lockName: string,
+  token: string,
+): Promise<void> {
+  const res = await collections.locks.deleteOne({
+    _id: lockName,
+    value: token,
+  });
+  if (res.deletedCount !== 1) throw new Error("Lock expired");
+}
+
+export async function getToken(lockName: string): Promise<string> {
+  const res = await collections.locks.findOne({ _id: lockName });
+  return res?.value;
+}

lib/logger.js

@@ -1,320 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const fs = require("fs");
-const os = require("os");
-
-const config = require("./config");
-
-const REOPEN_EVERY = 60000;
-
-const LOG_FORMAT = config.get("LOG_FORMAT");
-const ACCESS_LOG_FORMAT = config.get("ACCESS_LOG_FORMAT") || LOG_FORMAT;
-
-const defaultMeta = {};
-
-let LOG_SYSTEMD = false;
-let ACCESS_LOG_SYSTEMD = false;
-
-let LOG_FILE, ACCESS_LOG_FILE;
-
-let logStream = fs.createWriteStream(null, {fd: process.stderr.fd});
-let logStat = fs.fstatSync(logStream.fd);
-let accessLogStream = fs.createWriteStream(null, {fd: process.stdout.fd});
-let accessLogStat = fs.fstatSync(accessLogStream.fd);
-
-
-// Reopen if original files have been moved (e.g. logrotate)
-function reopen() {
-  let counter = 1;
-
-  if (LOG_FILE) {
-    ++ counter;
-    fs.stat(LOG_FILE, function(err, stat) {
-      if (err && !err.message.startsWith("ENOENT:"))
-        throw err;
-
-      if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
-        logStream.end();
-        logStream = fs.createWriteStream(null,
-          {fd: fs.openSync(LOG_FILE, "a")});
-        logStat = fs.fstatSync(logStream.fd);
-      }
-
-      if (-- counter === 0)
-        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-    });
-  }
-
-  if (ACCESS_LOG_FILE) {
-    ++ counter;
-    fs.stat(ACCESS_LOG_FILE, function(err, stat) {
-      if (err && !err.message.startsWith("ENOENT:"))
-        throw err;
-
-      if (!(stat && stat.dev === accessLogStat.dev && stat.ino === accessLogStat.ino)) {
-        accessLogStream.end();
-        accessLogStream = fs.createWriteStream(null,
-          {fd: fs.openSync(ACCESS_LOG_FILE, "a")});
-        accessLogStat = fs.fstatSync(accessLogStream.fd);
-      }
-
-      if (-- counter === 0)
-        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-    });
-  }
-
-  if (-- counter === 0)
-    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-}
-
-
-function init(service, version) {
-  defaultMeta.hostname = os.hostname();
-  defaultMeta.pid = process.pid;
-  defaultMeta.name = `genieacs-${service}`;
-  defaultMeta.version = version;
-
-  LOG_FILE = config.get(`${service.toUpperCase()}_LOG_FILE`);
-  ACCESS_LOG_FILE = config.get(`${service.toUpperCase()}_ACCESS_LOG_FILE`);
-
-  if (LOG_FILE) {
-    logStream = fs.createWriteStream(null,
-      {fd: fs.openSync(LOG_FILE, "a")});
-    logStat = fs.fstatSync(logStream.fd);
-  }
-
-  if (ACCESS_LOG_FILE) {
-    accessLogStream = fs.createWriteStream(null,
-      {fd: fs.openSync(ACCESS_LOG_FILE, "a")});
-    accessLogStat = fs.fstatSync(accessLogStream.fd);
-  }
-
-  // Determine if logs are going to journald
-  const JOURNAL_STREAM = process.env["JOURNAL_STREAM"];
-
-  if (JOURNAL_STREAM) {
-    let [dev, inode] = JOURNAL_STREAM.split(":");
-    dev = parseInt(dev);
-    inode = parseInt(inode);
-
-    LOG_SYSTEMD = (logStat.dev === dev && logStat.ino === inode);
-    ACCESS_LOG_SYSTEMD = (accessLogStat.dev === dev && accessLogStat.ino === inode);
-  }
-
-  if (LOG_FILE || ACCESS_LOG_FILE) {
-    // Can't use setInterval as we need all workers to cehck at the same time
-    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-  }
-}
-
-
-function close() {
-  accessLogStream.end();
-  logStream.end();
-}
-
-
-function flatten(details) {
-  if (details.sessionContext) {
-    details.deviceId = details.sessionContext.deviceId;
-    details.remoteAddress = details.sessionContext.httpRequest.connection.remoteAddress;
-    delete details.sessionContext;
-  }
-
-  if (details.exception) {
-    details.exceptionName = details.exception.name;
-    details.exceptionMessage = details.exception.message;
-    details.exceptionStack = details.exception.stack;
-    delete details.exception;
-  }
-
-  if (details.task) {
-    details.taskId = details.task._id;
-    delete details.task;
-  }
-
-  if (details.rpc) {
-    if (details.rpc.acsRequest) {
-      details.acsRequestId = details.rpc.id;
-      details.acsRequestName = details.rpc.acsRequest.name;
-      if (details.rpc.acsRequest.commandKey)
-        details.acsRequestCommandKey = details.rpc.acsRequest.commandKey;
-    }
-    else if (details.rpc.cpeRequest) {
-      details.cpeRequestId = details.rpc.id;
-      if (details.rpc.cpeRequest.name === "Inform") {
-        details.informEvent = details.rpc.cpeRequest.event.join(",");
-        details.informRetryCount = details.rpc.cpeRequest.retryCount;
-      }
-      else {
-        details.cpeRequestName = details.rpc.cpeRequest.name;
-        if (details.rpc.cpeRequest.commandKey)
-          details.cpeRequestCommandKey = details.rpc.cpeRequest.commandKey;
-      }
-    }
-    else if (details.rpc.cpeFault) {
-      details.acsRequestId = details.rpc.id;
-      details.cpeFaultCode = details.rpc.cpeFault.detail.faultCode;
-      details.cpeFaultString = details.rpc.cpeFault.detail.faultString;
-    }
-    delete details.rpc;
-  }
-
-  if (details.fault) {
-    details.faultCode = details.fault.code;
-    details.faultMessage = details.fault.message;
-    delete details.fault;
-  }
-
-  return details;
-}
-
-
-function formatJson(details, systemd) {
-  if (systemd) {
-    let severity = "";
-    if (details.severity === "info")
-      severity = "<6>";
-    else if (details.severity === "warn")
-      severity = "<4>";
-    else if (details.severity === "error")
-      severity = "<3>";
-
-    return `${severity}${JSON.stringify(flatten(details))}${os.EOL}`;
-  }
-
-  return `${JSON.stringify(flatten(details))}${os.EOL}`;
-}
-
-
-function formatSimple(details, systemd) {
-  flatten(details);
-
-  let remote = "";
-  if (details.remoteAddress) {
-    if (details.deviceId)
-      remote = `${details.remoteAddress} ${details.deviceId}: `;
-    else
-      remote = `${details.remoteAddress}: `;
-  }
-
-  const keys = Object.keys(details);
-  const skip = {
-    deviceId: 1,
-    remoteAddress: 1,
-    severity: 1,
-    timestamp: 1,
-    message: 1
-  };
-
-  let meta = "";
-
-  let kv = [];
-  for (let k of keys)
-    if (!skip[k])
-      kv.push(`${k}=${JSON.stringify(details[k])}`);
-
-  if (kv.length)
-    meta = `; ${kv.join(" ")}`;
-
-  if (systemd) {
-    let severity = "";
-    if (details.severity === "info")
-      severity = "<6>";
-    else if (details.severity === "warn")
-      severity = "<4>";
-    else if (details.severity === "error")
-      severity = "<3>";
-
-    return `${severity}${remote}${details.message}${meta}${os.EOL}`;
-  }
-
-  return `${details.timestamp} [${details.severity.toUpperCase()}] ${remote}${details.message}${meta}${os.EOL}`;
-}
-
-
-function log(details) {
-  details.timestamp = new Date().toISOString();
-  if (LOG_FORMAT === "json") {
-    details = Object.assign({}, defaultMeta, details);
-    logStream.write(formatJson(details, LOG_SYSTEMD));
-  } else {
-    logStream.write(formatSimple(details, LOG_SYSTEMD));
-  }
-}
-
-
-function info(details) {
-  details.severity = "info";
-  log(details);
-}
-
-
-function warn(details) {
-  details.severity = "warn";
-  log(details);
-}
-
-
-function error(details) {
-  details.severity = "error";
-  log(details);
-}
-
-
-function accessLog(details) {
-  details.timestamp = new Date().toISOString();
-  if (ACCESS_LOG_FORMAT === "json") {
-    Object.assign(details, defaultMeta);
-    accessLogStream.write(formatJson(details, ACCESS_LOG_SYSTEMD));
-  } else {
-    accessLogStream.write(formatSimple(details, ACCESS_LOG_SYSTEMD));
-  }
-}
-
-
-function accessInfo(details) {
-  details.severity = "info";
-  accessLog(details);
-}
-
-
-function accessWarn(details) {
-  details.severity = "warn";
-  accessLog(details);
-}
-
-
-function accessError(details) {
-  details.severity = "error";
-  accessLog(details);
-}
-
-
-exports.init = init;
-exports.close = close;
-exports.flatten = flatten;
-exports.info = info;
-exports.warn = warn;
-exports.error = error;
-exports.accessInfo = accessInfo;
-exports.accessWarn = accessWarn;
-exports.accessError = accessError;

lib/logger.ts

@@ -0,0 +1,328 @@
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as config from "./config.ts";
+import { getRequestOrigin } from "./forwarded.ts";
+import {
+  SessionContext,
+  AcsRequest,
+  CpeRequest,
+  CpeFault,
+  InformRequest,
+  Fault,
+} from "./types.ts";
+
+const REOPEN_EVERY = 60000;
+
+const LOG_FORMAT = config.get("LOG_FORMAT");
+const ACCESS_LOG_FORMAT = config.get("ACCESS_LOG_FORMAT") || LOG_FORMAT;
+
+const defaultMeta: { [name: string]: any } = {};
+
+let LOG_SYSTEMD = false;
+let ACCESS_LOG_SYSTEMD = false;
+
+let LOG_FILE, ACCESS_LOG_FILE;
+
+declare global {
+  /* eslint-disable-next-line @typescript-eslint/no-namespace */
+  namespace NodeJS {
+    export interface WritableStream {
+      fd?: number;
+    }
+  }
+}
+
+declare module "fs" {
+  interface WriteStream {
+    fd?: number;
+  }
+}
+
+let logStream = fs.createWriteStream(null, { fd: process.stderr.fd });
+let logStat = fs.fstatSync(logStream.fd);
+let accessLogStream = fs.createWriteStream(null, { fd: process.stdout.fd });
+let accessLogStat = fs.fstatSync(accessLogStream.fd);
+
+// Reopen if original files have been moved (e.g. logrotate)
+function reopen(): void {
+  let counter = 1;
+
+  if (LOG_FILE) {
+    ++counter;
+    fs.stat(LOG_FILE, (err, stat) => {
+      if (err && !err.message.startsWith("ENOENT:")) throw err;
+
+      if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
+        logStream.end();
+        logStream = fs.createWriteStream(null, {
+          fd: fs.openSync(LOG_FILE, "a"),
+        });
+        logStat = fs.fstatSync(logStream.fd);
+      }
+
+      if (--counter === 0)
+        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+    });
+  }
+
+  if (ACCESS_LOG_FILE) {
+    ++counter;
+    fs.stat(ACCESS_LOG_FILE, (err, stat) => {
+      if (err && !err.message.startsWith("ENOENT:")) throw err;
+
+      if (
+        !(
+          stat &&
+          stat.dev === accessLogStat.dev &&
+          stat.ino === accessLogStat.ino
+        )
+      ) {
+        accessLogStream.end();
+        accessLogStream = fs.createWriteStream(null, {
+          fd: fs.openSync(ACCESS_LOG_FILE, "a"),
+        });
+        accessLogStat = fs.fstatSync(accessLogStream.fd);
+      }
+
+      if (--counter === 0)
+        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+    });
+  }
+
+  if (--counter === 0)
+    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+}
+
+export function init(service: string, version: string): void {
+  defaultMeta.hostname = os.hostname();
+  defaultMeta.pid = process.pid;
+  defaultMeta.name = `genieacs-${service}`;
+  defaultMeta.version = version;
+
+  LOG_FILE = config.get(`${service.toUpperCase()}_LOG_FILE`);
+  ACCESS_LOG_FILE = config.get(`${service.toUpperCase()}_ACCESS_LOG_FILE`);
+
+  if (LOG_FILE) {
+    logStream = fs.createWriteStream(null, { fd: fs.openSync(LOG_FILE, "a") });
+    logStat = fs.fstatSync(logStream.fd);
+  }
+
+  if (ACCESS_LOG_FILE) {
+    accessLogStream = fs.createWriteStream(null, {
+      fd: fs.openSync(ACCESS_LOG_FILE, "a"),
+    });
+    accessLogStat = fs.fstatSync(accessLogStream.fd);
+  }
+
+  // Determine if logs are going to journald
+  const JOURNAL_STREAM = process.env["JOURNAL_STREAM"];
+
+  if (JOURNAL_STREAM) {
+    const [dev, inode] = JOURNAL_STREAM.split(":").map(parseInt);
+
+    LOG_SYSTEMD = logStat.dev === dev && logStat.ino === inode;
+    ACCESS_LOG_SYSTEMD =
+      accessLogStat.dev === dev && accessLogStat.ino === inode;
+  }
+
+  if (LOG_FILE || ACCESS_LOG_FILE)
+    // Can't use setInterval as we need all workers to cehck at the same time
+    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+}
+
+export function close(): void {
+  accessLogStream.end();
+  logStream.end();
+}
+
+export function flatten(
+  details: Record<string, unknown>,
+): Record<string, unknown> {
+  if (details.sessionContext) {
+    const sessionContext = details.sessionContext as SessionContext;
+    details.deviceId = sessionContext.deviceId;
+    details.remoteAddress = getRequestOrigin(
+      sessionContext.httpRequest,
+    ).remoteAddress;
+    delete details.sessionContext;
+  }
+
+  if (details.exception) {
+    const err = details.exception as Error;
+    details.exceptionName = err.name;
+    details.exceptionMessage = err.message;
+    details.exceptionStack = err.stack;
+    delete details.exception;
+  }
+
+  if (details.task) {
+    details.taskId = details.task["_id"];
+    delete details.task;
+  }
+
+  if (details.rpc) {
+    const rpc = details.rpc as {
+      id: string;
+      acsRequest?: AcsRequest;
+      cpeRequest?: CpeRequest;
+      cpeFault?: CpeFault;
+    };
+    if (rpc.acsRequest) {
+      details.acsRequestId = rpc.id;
+      details.acsRequestName = rpc.acsRequest.name;
+      if (rpc.acsRequest["commandKey"])
+        details.acsRequestCommandKey = rpc.acsRequest["commandKey"];
+    } else if (rpc.cpeRequest) {
+      details.cpeRequestId = rpc.id;
+      if (rpc.cpeRequest.name === "Inform") {
+        details.informEvent = (rpc.cpeRequest as InformRequest).event.join(",");
+        details.informRetryCount = (rpc.cpeRequest as InformRequest).retryCount;
+      } else {
+        details.cpeRequestName = rpc.cpeRequest.name;
+        if (rpc.cpeRequest["commandKey"])
+          details.cpeRequestCommandKey = rpc.cpeRequest["commandKey"];
+      }
+    } else if (rpc.cpeFault) {
+      details.acsRequestId = rpc.id;
+      details.cpeFaultCode = rpc.cpeFault.detail.faultCode;
+      details.cpeFaultString = rpc.cpeFault.detail.faultString;
+    }
+    delete details.rpc;
+  }
+
+  if (details.fault) {
+    const fault = details.fault as Fault;
+    details.faultCode = fault.code;
+    details.faultMessage = fault.message;
+    delete details.fault;
+  }
+
+  // For genieacs-ui
+  if (details.context) {
+    details.remoteAddress = getRequestOrigin(
+      details.context["req"],
+    ).remoteAddress;
+    if (details.context["state"].user)
+      details.user = details.context["state"].user.username;
+    delete details.context;
+  }
+
+  for (const [k, v] of Object.entries(details))
+    if (v == null) delete details[k];
+
+  return details;
+}
+
+function formatJson(
+  details: Record<string, unknown>,
+  systemd: boolean,
+): string {
+  if (systemd) {
+    let severity = "";
+    if (details.severity === "info") severity = "<6>";
+    else if (details.severity === "warn") severity = "<4>";
+    else if (details.severity === "error") severity = "<3>";
+
+    return `${severity}${JSON.stringify(flatten(details))}${os.EOL}`;
+  }
+
+  return `${JSON.stringify(flatten(details))}${os.EOL}`;
+}
+
+function formatSimple(
+  details: Record<string, unknown>,
+  systemd: boolean,
+): string {
+  const skip = {
+    user: true,
+    remoteAddress: true,
+    severity: true,
+    timestamp: true,
+    message: true,
+    deviceId: !!details.sessionContext,
+  };
+
+  flatten(details);
+
+  let remote = "";
+  if (details.remoteAddress) {
+    if (details.deviceId && skip["deviceId"])
+      remote = `${details.remoteAddress} ${details.deviceId}: `;
+    else if (details.user)
+      remote = `${details.user}@${details.remoteAddress}: `;
+    else remote = `${details.remoteAddress}: `;
+  }
+
+  const keys = Object.keys(details);
+
+  let meta = "";
+
+  const kv = [];
+  for (const k of keys)
+    if (!skip[k]) kv.push(`${k}=${JSON.stringify(details[k])}`);
+
+  if (kv.length) meta = `; ${kv.join(" ")}`;
+
+  if (systemd) {
+    let severity = "";
+    if (details.severity === "info") severity = "<6>";
+    else if (details.severity === "warn") severity = "<4>";
+    else if (details.severity === "error") severity = "<3>";
+
+    return `${severity}${remote}${details.message}${meta}${os.EOL}`;
+  }
+
+  return `${details.timestamp} [${(
+    details.severity as string
+  ).toUpperCase()}] ${remote}${details.message}${meta}${os.EOL}`;
+}
+
+function log(details: Record<string, unknown>): void {
+  details.timestamp = new Date().toISOString();
+  if (LOG_FORMAT === "json") {
+    details = Object.assign({}, defaultMeta, details);
+    logStream.write(formatJson(details, LOG_SYSTEMD));
+  } else {
+    logStream.write(formatSimple(details, LOG_SYSTEMD));
+  }
+}
+
+export function info(details: Record<string, unknown>): void {
+  details.severity = "info";
+  log(details);
+}
+
+export function warn(details: Record<string, unknown>): void {
+  details.severity = "warn";
+  log(details);
+}
+
+export function error(details: Record<string, unknown>): void {
+  details.severity = "error";
+  log(details);
+}
+
+export function accessLog(details: Record<string, unknown>): void {
+  details.timestamp = new Date().toISOString();
+  if (ACCESS_LOG_FORMAT === "json") {
+    Object.assign(details, defaultMeta);
+    accessLogStream.write(formatJson(details, ACCESS_LOG_SYSTEMD));
+  } else {
+    accessLogStream.write(formatSimple(details, ACCESS_LOG_SYSTEMD));
+  }
+}
+
+export function accessInfo(details: Record<string, unknown>): void {
+  details.severity = "info";
+  accessLog(details);
+}
+
+export function accessWarn(details: Record<string, unknown>): void {
+  details.severity = "warn";
+  accessLog(details);
+}
+
+export function accessError(details: Record<string, unknown>): void {
+  details.severity = "error";
+  accessLog(details);
+}

lib/nbi.coffee

@@ -1,516 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-url = require 'url'
-mongodb = require 'mongodb'
-querystring = require 'querystring'
-vm = require 'vm'
-
-config = require './config'
-common = require './common'
-db = require './db'
-query = require './query'
-apiFunctions = require './api-functions'
-cache = require './cache'
-
-VERSION = require('../package.json').version
-
-# regular expression objects
-DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/tasks\/?$/
-TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-\_\%]+)(\/[a-zA-Z_]*)?$/
-TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/tags\/([a-zA-Z0-9\-\_\%]+)\/?$/
-PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-\_\%]+)\/?$/
-OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-\_\%]+)\/?$/
-FILES_REGEX = /^\/files\/([a-zA-Z0-9\%\!\*\'\(\)\;\:\@\&\=\+\$\,\?\#\[\]\-\_\.\~]+)\/?$/
-PING_REGEX = /^\/ping\/([a-zA-Z0-9\-\_\.\:]+)\/?$/
-QUERY_REGEX = /^\/([a-zA-Z0-9_]+)\/?$/
-DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/?$/
-PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-\_\%]+)\/?$/
-VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-\_\%]+)\/?$/
-FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-\_\%\:]+)\/?$/
-
-
-
-throwError = (err, httpResponse) ->
-  if httpResponse
-    httpResponse.writeHead(500, {'Connection' : 'close'})
-    httpResponse.end("#{err.name}: #{err.message}")
-
-  throw err
-
-
-listener = (request, response) ->
-  chunks = []
-  bytes = 0
-
-  response.setHeader('GenieACS-Version', VERSION)
-
-  request.addListener 'data', (chunk) ->
-    chunks.push(chunk)
-    bytes += chunk.length
-
-  request.getBody = () ->
-    # Write all chunks into a Buffer
-    body = new Buffer(bytes)
-    offset = 0
-    chunks.forEach((chunk) ->
-      chunk.copy(body, offset, 0, chunk.length)
-      offset += chunk.length
-    )
-    return body
-
-  request.addListener 'end', () ->
-    body = request.getBody()
-    urlParts = url.parse(request.url, true)
-    if PRESETS_REGEX.test(urlParts.pathname)
-      presetName = querystring.unescape(PRESETS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        preset = JSON.parse(body)
-        preset._id = presetName
-
-        db.presetsCollection.save(preset, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.presetsCollection.remove({'_id' : presetName}, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if OBJECTS_REGEX.test(urlParts.pathname)
-      objectName = querystring.unescape(OBJECTS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        object = JSON.parse(body)
-        object._id = objectName
-
-        db.objectsCollection.save(object, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.objectsCollection.remove({'_id' : objectName}, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if PROVISIONS_REGEX.test(urlParts.pathname)
-      provisionName = querystring.unescape(PROVISIONS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        object = {
-          _id: provisionName
-          script: body.toString()
-        }
-        try
-          new vm.Script("\"use strict\";(function(){\n#{object.script}\n})();")
-        catch err
-          response.writeHead(400)
-          response.end("#{err.name}: #{err.message}")
-          return
-
-        db.provisionsCollection.save(object, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.provisionsCollection.remove({'_id' : provisionName}, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if VIRTUAL_PARAMETERS_REGEX.test(urlParts.pathname)
-      virtualParameterName = querystring.unescape(VIRTUAL_PARAMETERS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        object = {
-          _id: virtualParameterName
-          script: body.toString()
-        }
-        try
-          new vm.Script("\"use strict\";(function(){\n#{object.script}\n})();")
-        catch err
-          response.writeHead(400)
-          response.end("#{err.name}: #{err.message}")
-          return
-
-        db.virtualParametersCollection.save(object, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.virtualParametersCollection.remove({'_id' : virtualParameterName}, (err) ->
-          return throwError(err, response) if err
-          cache.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if TAGS_REGEX.test(urlParts.pathname)
-      r = TAGS_REGEX.exec(urlParts.pathname)
-      deviceId = querystring.unescape(r[1])
-      tag = querystring.unescape(r[2])
-      if request.method == 'POST'
-        db.devicesCollection.update({'_id' : deviceId}, {'$addToSet' : {'_tags' : tag}}, {safe: true}, (err) ->
-          return throwError(err, response) if err
-          response.writeHead(200)
-          response.end()
-        )
-      else if request.method == 'DELETE'
-        db.devicesCollection.update({'_id' : deviceId}, {'$pull' : {'_tags' : tag}}, {safe: true}, (err) ->
-          return throwError(err, response) if err
-          response.writeHead(200)
-          response.end()
-        )
-      else
-        response.writeHead 405, {'Allow': 'POST, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if FAULTS_REGEX.test(urlParts.pathname)
-      if request.method == 'DELETE'
-        faultId = querystring.unescape(FAULTS_REGEX.exec(urlParts.pathname)[1])
-        deviceId = faultId.split(':', 1)[0]
-        channel = faultId.slice(deviceId.length + 1)
-        db.faultsCollection.remove({_id : faultId}, (err) ->
-          return throwError(err, response) if err
-          if channel.startsWith('task_')
-            return db.tasksCollection.remove({_id : new mongodb.ObjectID(channel.slice(5))}, (err) ->
-              return throwError(err, response) if err
-              cache.del("#{deviceId}_tasks_faults_operations", (err) ->
-                return throwError(err, response) if err
-                response.writeHead(200)
-                response.end()
-              )
-            )
-
-          cache.del("#{deviceId}_tasks_faults_operations", (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'DELETE'}
-        response.end('405 Method Not Allowed')
-    else if DEVICE_TASKS_REGEX.test(urlParts.pathname)
-      if request.method == 'POST'
-        deviceId = querystring.unescape(DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1])
-        if body.length
-          task = JSON.parse(body)
-          task.device = deviceId
-          apiFunctions.insertTasks(task, (err) ->
-            return throwError(err, response) if err
-            cache.del("#{deviceId}_tasks_faults_operations", (err) ->
-              return throwError(err, response) if err
-
-              if urlParts.query.connection_request?
-                apiFunctions.connectionRequest(deviceId, (err) ->
-                  if err
-                    response.writeHead(202, err.message, {'Content-Type' : 'application/json'})
-                    response.end(JSON.stringify(task))
-                  else
-                    apiFunctions.watchTask(deviceId, task._id, config.get('DEVICE_ONLINE_THRESHOLD', deviceId), (err, status) ->
-                      return throwError(err, response) if err
-
-                      if status is 'timeout'
-                        response.writeHead(202, 'Task queued but not processed', {'Content-Type' : 'application/json'})
-                        response.end(JSON.stringify(task))
-                      else if status is 'fault'
-                        db.tasksCollection.findOne({_id : task._id}, (err, task) ->
-                          return throwError(err, response) if err
-
-                          response.writeHead(202, 'Task faulted', {'Content-Type' : 'application/json'})
-                          response.end(JSON.stringify(task))
-                        )
-                      else
-                        response.writeHead(200, {'Content-Type' : 'application/json'})
-                        response.end(JSON.stringify(task))
-                    )
-                )
-              else
-                response.writeHead(202, {'Content-Type' : 'application/json'})
-                response.end(JSON.stringify(task))
-            )
-          )
-        else if urlParts.query.connection_request?
-          # no task, send connection request only
-          apiFunctions.connectionRequest(deviceId, (err) ->
-            if err
-              response.writeHead 504
-              response.end("#{err.name}: #{err.message}")
-              return
-            response.writeHead 200
-            response.end()
-          )
-        else
-          response.writeHead(400)
-          response.end()
-      else
-        response.writeHead 405, {'Allow': 'POST'}
-        response.end('405 Method Not Allowed')
-    else if TASKS_REGEX.test(urlParts.pathname)
-      r = TASKS_REGEX.exec(urlParts.pathname)
-      taskId = new mongodb.ObjectID(querystring.unescape(r[1]))
-      action = r[2]
-      if not action? or action is '/'
-        if request.method == 'DELETE'
-          db.tasksCollection.findOne({'_id' : taskId}, {'device' : 1}, (err, task) ->
-            return throwError(err, response) if err
-            if not task?
-              response.writeHead(404)
-              response.end("Task not found")
-              return
-
-            deviceId = task.device
-            db.tasksCollection.remove({'_id' : taskId}, (err) ->
-              return throwError(err, response) if err
-              db.faultsCollection.remove({_id : "#{deviceId}:task_#{String(taskId)}"}, (err) ->
-                return throwError(err, response) if err
-                cache.del("#{deviceId}_tasks_faults_operations", (err) ->
-                  return throwError(err, response) if err
-                  response.writeHead(200)
-                  response.end()
-                )
-              )
-            )
-          )
-        else
-          response.writeHead 405, {'Allow': 'PUT DELETE'}
-          response.end('405 Method Not Allowed')
-      else if action is '/retry'
-        if request.method == 'POST'
-          db.tasksCollection.findOne({'_id' : taskId}, {'device' : 1}, (err, task) ->
-            return throwError(err, response) if err
-            deviceId = task.device
-            db.faultsCollection.remove({_id : "#{deviceId}:task_#{String(taskId)}"}, (err) ->
-              return throwError(err, response) if err
-              cache.del("#{deviceId}_tasks_faults_operations", (err) ->
-                return throwError(err, response) if err
-                response.writeHead(200)
-                response.end()
-              )
-            )
-          )
-        else
-          response.writeHead 405, {'Allow': 'POST'}
-          response.end('405 Method Not Allowed')
-      else
-        response.writeHead(404)
-        response.end()
-    else if FILES_REGEX.test(urlParts.pathname)
-      filename = querystring.unescape(FILES_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        metadata = {
-          fileType : request.headers.filetype,
-          oui : request.headers.oui,
-          productClass : request.headers.productclass,
-          version : request.headers.version,
-        }
-
-        gs = new mongodb.GridStore(db.mongoDb, filename, filename, 'w', {metadata : metadata})
-        gs.open((err, gs) ->
-          gs.write(body, (err, res) ->
-            return throwError(err, response) if err
-            gs.close((err) ->
-              return throwError(err, response) if err
-              response.writeHead(201)
-              response.end()
-            )
-          )
-        )
-      else if request.method == 'DELETE'
-        mongodb.GridStore.unlink(db.mongoDb, filename, (err) ->
-          response.writeHead(200)
-          response.end()
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if PING_REGEX.test(urlParts.pathname)
-      host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1])
-      require('child_process').exec("ping -w 1 -i 0.2 -c 3 #{host}", (err, stdout, stderr) ->
-        if err
-          response.writeHead(404, {'Cache-Control' : 'no-cache'})
-          response.end("#{err.name}: #{err.message}")
-          return
-        response.writeHead(200, {'Content-Type' : 'text/plain', 'Cache-Control' : 'no-cache'})
-        response.end(stdout)
-      )
-    else if DELETE_DEVICE_REGEX.test(urlParts.pathname)
-      if request.method isnt 'DELETE'
-        response.writeHead(405, {'Allow' : 'DELETE'})
-        response.end('405 Method Not Allowed')
-        return
-
-      deviceId = querystring.unescape(DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1])
-      apiFunctions.deleteDevice(deviceId, (err) ->
-        return throwError(err, response) if err
-        response.writeHead(200)
-        response.end()
-      )
-    else if QUERY_REGEX.test(urlParts.pathname)
-      collectionName = QUERY_REGEX.exec(urlParts.pathname)[1]
-
-      # Convert to camel case
-      i = collectionName.indexOf('_')
-      while i >= 0
-        ++ i
-        up = if i < collectionName.length then collectionName[i].toUpperCase() else ''
-        collectionName = collectionName.slice(0, i - 1) + up + collectionName.slice(i + 1)
-        i = collectionName.indexOf('_', i)
-
-      if request.method not in ['GET', 'HEAD']
-        response.writeHead 405, {'Allow' : 'GET, HEAD'}
-        response.end('405 Method Not Allowed')
-        return
-      collection = db["#{collectionName}Collection"]
-      if not collection?
-        response.writeHead 404
-        response.end('404 Not Found')
-        return
-
-      if urlParts.query.query?
-        try
-          q = JSON.parse(urlParts.query.query)
-        catch err
-          response.writeHead(400)
-          response.end("#{err.name}: #{err.message}")
-          return
-      else
-        q = {}
-
-      switch collectionName
-        when 'devices'
-          q = query.expand(q)
-        when 'tasks'
-          q = query.sanitizeQueryTypes(q, {
-            _id: ((v) -> return new mongodb.ObjectID(v))
-            timestamp: ((v) -> return new Date(v))
-            retries: Number
-          })
-        when 'faults'
-          q = query.sanitizeQueryTypes(q, {
-            timestamp: ((v) -> return new Date(v))
-            retries: Number
-          })
-
-      if urlParts.query.projection?
-        projection = {}
-        for p in urlParts.query.projection.split(',')
-          p = p.trim()
-          projection[p] = 1
-
-      cur = collection.find(q, projection, {batchSize : 50})
-      if urlParts.query.sort?
-        s = JSON.parse(urlParts.query.sort)
-        sort = {}
-        for k,v of s
-          if k[k.lastIndexOf('.') + 1] != '_' and collectionName is 'devices'
-            sort["#{k}._value"] = v
-          else
-            sort[k] = v
-        cur.sort(sort)
-
-      cur.skip(parseInt(urlParts.query.skip)) if urlParts.query.skip?
-      cur.limit(limit = parseInt(urlParts.query.limit)) if urlParts.query.limit?
-      cur.count(false, (err, total) ->
-        response.writeHead(200, {'Content-Type' : 'application/json', 'total' : total})
-        if request.method is 'HEAD'
-          response.end()
-          return
-        response.write("[\n")
-        i = 0
-        cur.each((err, item) ->
-          if err
-            throwError(err)
-            return false
-
-          if item?
-            response.write(",\n") if i++
-            response.write(JSON.stringify(item))
-
-          if not item? or (limit? and i >= limit)
-            response.end("\n]")
-        )
-      )
-
-    else
-      response.writeHead 404
-      response.end('404 Not Found')
-
-
-exports.listener = listener

lib/nbi.ts

@@ -0,0 +1,702 @@
+import * as vm from "node:vm";
+import { IncomingMessage, ServerResponse } from "node:http";
+import { Collection, ObjectId } from "mongodb";
+import { getRevision, getConfig } from "./ui/local-cache.ts";
+import { filesBucket, collections } from "./db/db.ts";
+import { optimizeProjection } from "./db/util.ts";
+import * as query from "./query.ts";
+import * as apiFunctions from "./api-functions.ts";
+import * as cache from "./cache.ts";
+import { version as VERSION } from "../package.json";
+import { ping } from "./ping.ts";
+import * as logger from "./logger.ts";
+import { flattenDevice } from "./ui/db.ts";
+import { getRequestOrigin } from "./forwarded.ts";
+import { acquireLock, releaseLock } from "./lock.ts";
+import { ResourceLockedError } from "./common/errors.ts";
+
+const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
+const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
+const TAGS_REGEX =
+  /^\/devices\/([a-zA-Z0-9\-_%]+)\/tags\/([a-zA-Z0-9\-_%]+)\/?$/;
+const PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-_%]+)\/?$/;
+const OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-_%]+)\/?$/;
+const FILES_REGEX = /^\/files\/([a-zA-Z0-9%!*'();:@&=+$,?#[\]\-_.~]+)\/?$/;
+const PING_REGEX = /^\/ping\/([a-zA-Z0-9\-_.:]+)\/?$/;
+const QUERY_REGEX = /^\/([a-zA-Z0-9_]+)\/?$/;
+const DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/?$/;
+const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
+const VIRTUAL_PARAMETERS_REGEX =
+  /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
+const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
+
+async function getBody(request: IncomingMessage): Promise<Buffer> {
+  const chunks: Buffer[] = [];
+  let readableEnded = false;
+  request.on("end", () => {
+    readableEnded = true;
+  });
+  for await (const chunk of request) chunks.push(chunk);
+  // In Node versions prior to 15, the stream will not emit an error if the
+  // connection is closed before the stream is finished.
+  // For Node 12.9+ we can just use stream.readableEnded
+  if (!readableEnded) throw new Error("Connection closed");
+  return Buffer.concat(chunks);
+}
+
+export async function listener(
+  request: IncomingMessage,
+  response: ServerResponse,
+): Promise<void> {
+  response.setHeader("GenieACS-Version", VERSION);
+
+  const origin = getRequestOrigin(request);
+  const url = new URL(
+    request.url,
+    (origin.encrypted ? "https://" : "http://") + origin.host,
+  );
+
+  const body = await getBody(request).catch(() => null);
+  // Ignore incomplete requests
+  if (body == null) return;
+
+  logger.accessInfo(
+    Object.assign({}, Object.fromEntries(url.searchParams), {
+      remoteAddress: origin.remoteAddress,
+      message: `${request.method} ${url.pathname}`,
+    }),
+  );
+  return handler(request, response, url, body);
+}
+
+async function handler(
+  request: IncomingMessage,
+  response: ServerResponse,
+  url: URL,
+  body: Buffer,
+): Promise<void> {
+  if (PRESETS_REGEX.test(url.pathname)) {
+    const presetName = decodeURIComponent(PRESETS_REGEX.exec(url.pathname)[1]);
+    if (request.method === "PUT") {
+      let preset;
+      try {
+        preset = JSON.parse(body.toString());
+      } catch (err) {
+        response.writeHead(400);
+        response.end(`${err.name}: ${err.message}`);
+        return;
+      }
+      preset._id = presetName;
+      await collections.presets.replaceOne({ _id: presetName }, preset, {
+        upsert: true,
+      });
+      await cache.del("cwmp-local-cache-hash");
+      response.writeHead(200);
+      response.end();
+    } else if (request.method === "DELETE") {
+      await collections.presets.deleteOne({ _id: presetName });
+      await cache.del("cwmp-local-cache-hash");
+      response.writeHead(200);
+      response.end();
+    } else {
+      response.writeHead(405, { Allow: "PUT, DELETE" });
+      response.end("405 Method Not Allowed");
+    }
+  } else if (OBJECTS_REGEX.test(url.pathname)) {
+    const objectName = decodeURIComponent(OBJECTS_REGEX.exec(url.pathname)[1]);
+    if (request.method === "PUT") {
+      let object;
+      try {
+        object = JSON.parse(body.toString());
+      } catch (err) {
+        response.writeHead(400);
+        response.end(`${err.name}: ${err.message}`);
+        return;
+      }
+      object._id = objectName;
+      await collections.objects.replaceOne({ _id: objectName }, object, {
+        upsert: true,
+      });
+      await cache.del("cwmp-local-cache-hash");
+      response.writeHead(200);
+      response.end();
+    } else if (request.method === "DELETE") {
+      await collections.objects.deleteOne({ _id: objectName });
+      await cache.del("cwmp-local-cache-hash");
+      response.writeHead(200);
+      response.end();
+    } else {
+      response.writeHead(405, { Allow: "PUT, DELETE" });
+      response.end("405 Method Not Allowed");
+    }
+  } else if (PROVISIONS_REGEX.test(url.pathname)) {
+    const provisionName = decodeURIComponent(
+      PROVISIONS_REGEX.exec(url.pathname)[1],
+    );
+    if (request.method === "PUT") {
+      const object = {
+        _id: provisionName,
+        script: body.toString(),
+      };
+
+      try {
+        new vm.Script(`"use strict";(function(){\n${object.script}\n})();`);
+      } catch (err) {
+        response.writeHead(400);
+        response.end(`${err.name}: ${err.message}`);
+        return;
+      }
+
+      await collections.provisions.replaceOne({ _id: provisionName }, object, {
+        upsert: true,
+      });
+      await cache.del("cwmp-local-cache-hash");
+      response.writeHead(200);
+      response.end();
+    } else if (request.method === "DELETE") {
+      await collections.provisions.deleteOne({ _id: provisionName });
+      await cache.del("cwmp-local-cache-hash");
+      response.writeHead(200);
+      response.end();
+    } else {
+      response.writeHead(405, { Allow: "PUT, DELETE" });
+      response.end("405 Method Not Allowed");
+    }
+  } else if (VIRTUAL_PARAMETERS_REGEX.test(url.pathname)) {
+    const virtualParameterName = decodeURIComponent(
+      VIRTUAL_PARAMETERS_REGEX.exec(url.pathname)[1],
+    );
+    if (request.method === "PUT") {
+      const object = {
+        _id: virtualParameterName,
+        script: body.toString(),
+      };
+
+      try {
+        new vm.Script(`"use strict";(function(){\n${object.script}\n})();`);
+      } catch (err) {
+        response.writeHead(400);
+        response.end(`${err.name}: ${err.message}`);
+        return;
+      }
+
+      await collections.virtualParameters.replaceOne(
+        { _id: virtualParameterName },
+        object,
+        { upsert: true },
+      );
+      await cache.del("cwmp-local-cache-hash");
+      response.writeHead(200);
+      response.end();
+    } else if (request.method === "DELETE") {
+      await collections.virtualParameters.deleteOne({
+        _id: virtualParameterName,
+      });
+      await cache.del("cwmp-local-cache-hash");
+      response.writeHead(200);
+      response.end();
+    } else {
+      response.writeHead(405, { Allow: "PUT, DELETE" });
+      response.end("405 Method Not Allowed");
+    }
+  } else if (TAGS_REGEX.test(url.pathname)) {
+    const r = TAGS_REGEX.exec(url.pathname);
+    const deviceId = decodeURIComponent(r[1]);
+    const tag = decodeURIComponent(r[2]);
+    if (request.method === "POST") {
+      const updateRes = await collections.devices.updateOne(
+        { _id: deviceId },
+        { $addToSet: { _tags: tag } },
+      );
+
+      if (!updateRes.matchedCount) {
+        response.writeHead(404);
+        response.end("No such device");
+        return;
+      }
+
+      response.writeHead(200);
+      response.end();
+    } else if (request.method === "DELETE") {
+      const updateRes = await collections.devices.updateOne(
+        { _id: deviceId },
+        { $pull: { _tags: tag } },
+      );
+
+      if (!updateRes.matchedCount) {
+        response.writeHead(404);
+        response.end("No such device");
+        return;
+      }
+
+      response.writeHead(200);
+      response.end();
+    } else {
+      response.writeHead(405, { Allow: "POST, DELETE" });
+      response.end("405 Method Not Allowed");
+    }
+  } else if (FAULTS_REGEX.test(url.pathname)) {
+    if (request.method === "DELETE") {
+      const faultId = decodeURIComponent(FAULTS_REGEX.exec(url.pathname)[1]);
+      try {
+        await apiFunctions.deleteFault(faultId);
+      } catch (err) {
+        if (err instanceof ResourceLockedError) {
+          response.writeHead(503);
+          response.end("Device is in session");
+          return;
+        }
+        throw err;
+      }
+
+      response.writeHead(200);
+      response.end();
+    } else {
+      response.writeHead(405, { Allow: "DELETE" });
+      response.end("405 Method Not Allowed");
+    }
+  } else if (DEVICE_TASKS_REGEX.test(url.pathname)) {
+    if (request.method === "POST") {
+      const deviceId = decodeURIComponent(
+        DEVICE_TASKS_REGEX.exec(url.pathname)[1],
+      );
+
+      const conReq = url.searchParams.has("connection_request");
+      let task;
+      if (body.length) {
+        try {
+          task = JSON.parse(body.toString());
+          task.device = deviceId;
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+      }
+
+      if (!task && !conReq) {
+        response.writeHead(400);
+        response.end();
+        return;
+      }
+
+      if (!task || !conReq) {
+        const dev = await collections.devices.findOne({ _id: deviceId });
+        if (!dev) {
+          response.writeHead(404);
+          response.end("No such device");
+          return;
+        }
+
+        if (task) {
+          await apiFunctions.insertTasks(task);
+          response.writeHead(202, { "Content-Type": "application/json" });
+          response.end(JSON.stringify(task));
+        } else {
+          const status = await apiFunctions.connectionRequest(
+            deviceId,
+            flattenDevice(dev),
+          );
+          if (status) {
+            response.writeHead(504, status);
+            response.end(status);
+          } else {
+            response.writeHead(200);
+            response.end();
+          }
+        }
+        return;
+      }
+
+      const socketTimeout: number = request.socket.timeout;
+
+      // Extend socket timeout while waiting for session
+      if (socketTimeout) request.socket.setTimeout(300000);
+
+      const token = await acquireLock(`cwmp_session_${deviceId}`, 5000, 30000);
+      if (!token) {
+        // Restore socket timeout
+        if (socketTimeout) request.socket.setTimeout(socketTimeout);
+        const dev = await collections.devices.findOne({ _id: deviceId });
+        if (!dev) {
+          response.writeHead(404);
+          response.end("No such device");
+          return;
+        }
+
+        await apiFunctions.insertTasks(task);
+        response.writeHead(202, "Task queued but not processed", {
+          "Content-Type": "application/json",
+        });
+        response.end(JSON.stringify(task));
+        return;
+      }
+
+      let dev;
+
+      try {
+        dev = await collections.devices.findOne({ _id: deviceId });
+        if (!dev) {
+          response.writeHead(404);
+          response.end("No such device");
+          return;
+        }
+        await apiFunctions.insertTasks(task);
+      } finally {
+        await releaseLock(`cwmp_session_${deviceId}`, token);
+      }
+
+      const lastInform = (dev["_lastInform"] as Date).getTime();
+      const device = flattenDevice(dev);
+
+      let onlineThreshold: number;
+      if (url.searchParams.has("timeout")) {
+        onlineThreshold = parseInt(url.searchParams.get("timeout"));
+      } else {
+        const revision = await getRevision();
+        onlineThreshold = getConfig(
+          revision,
+          "cwmp.deviceOnlineThreshold",
+          {},
+          Date.now(),
+          (exp) => {
+            if (!Array.isArray(exp)) return exp;
+            if (exp[0] === "PARAM") {
+              const p = device[exp[1]];
+              if (p?.value) return p.value[0];
+            } else if (exp[0] === "FUNC") {
+              if (exp[1] === "REMOTE_ADDRESS") {
+                for (const root of ["InternetGatewayDevice", "Device"]) {
+                  const p =
+                    device[`${root}.ManagementServer.ConnectionRequestURL`];
+                  if (p?.value) return new URL(p.value[0] as string).hostname;
+                }
+                return null;
+              }
+            }
+            return exp;
+          },
+        ) as number;
+      }
+
+      let status = await apiFunctions.connectionRequest(deviceId, device);
+      if (!status) {
+        const sessionStarted = await apiFunctions.awaitSessionStart(
+          deviceId,
+          lastInform,
+          onlineThreshold,
+        );
+        if (!sessionStarted) {
+          status = "Task queued but not processed";
+        } else {
+          const sessionEnded = await apiFunctions.awaitSessionEnd(
+            deviceId,
+            120000,
+          );
+          if (!sessionEnded) {
+            status = "Task queued but not processed";
+          } else {
+            const f = await collections.faults.count({
+              _id: `${deviceId}:task_${task._id}`,
+            });
+            if (f) status = "Task faulted";
+          }
+        }
+      }
+
+      // Restore socket timeout
+      if (socketTimeout) request.socket.setTimeout(socketTimeout);
+
+      if (status) {
+        response.writeHead(202, status, { "Content-Type": "application/json" });
+        response.end(JSON.stringify(task));
+      } else {
+        response.writeHead(200, { "Content-Type": "application/json" });
+        response.end(JSON.stringify(task));
+      }
+    } else {
+      response.writeHead(405, { Allow: "POST" });
+      response.end("405 Method Not Allowed");
+    }
+  } else if (TASKS_REGEX.test(url.pathname)) {
+    const r = TASKS_REGEX.exec(url.pathname);
+    const taskId = decodeURIComponent(r[1]);
+    const action = r[2];
+    if (!action || action === "/") {
+      if (request.method === "DELETE") {
+        const task = await collections.tasks.findOne(
+          { _id: new ObjectId(taskId) },
+          { projection: { device: 1 } },
+        );
+
+        if (!task) {
+          response.writeHead(404);
+          response.end("Task not found");
+          return;
+        }
+
+        const deviceId = task.device;
+        const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+        if (!token) {
+          response.writeHead(503);
+          response.end("Device is in session");
+          return;
+        }
+
+        try {
+          await Promise.all([
+            collections.tasks.deleteOne({ _id: new ObjectId(taskId) }),
+            collections.faults.deleteOne({ _id: `${deviceId}:task_${taskId}` }),
+          ]);
+        } finally {
+          await releaseLock(`cwmp_session_${deviceId}`, token);
+        }
+
+        response.writeHead(200);
+        response.end();
+      } else {
+        response.writeHead(405, { Allow: "PUT DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (action === "/retry") {
+      if (request.method === "POST") {
+        const task = await collections.tasks.findOne(
+          { _id: new ObjectId(taskId) },
+          { projection: { device: 1 } },
+        );
+
+        const deviceId = task.device;
+        const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+        if (!token) {
+          response.writeHead(503);
+          response.end("Device is in session");
+          return;
+        }
+        try {
+          await collections.faults.deleteOne({
+            _id: `${deviceId}:task_${taskId}`,
+          });
+        } finally {
+          await releaseLock(`cwmp_session_${deviceId}`, token);
+        }
+
+        response.writeHead(200);
+        response.end();
+      } else {
+        response.writeHead(405, { Allow: "POST" });
+        response.end("405 Method Not Allowed");
+      }
+    } else {
+      response.writeHead(404);
+      response.end();
+    }
+  } else if (FILES_REGEX.test(url.pathname)) {
+    const filename = decodeURIComponent(FILES_REGEX.exec(url.pathname)[1]);
+    if (request.method === "PUT") {
+      const metadata = {
+        fileType: request.headers.filetype,
+        oui: request.headers.oui,
+        productClass: request.headers.productclass,
+        version: request.headers.version,
+      };
+      try {
+        await filesBucket.delete(filename as unknown as ObjectId);
+      } catch (err) {
+        // Ignore error if file doesn't exist
+      }
+
+      return new Promise((resolve, reject) => {
+        const uploadStream = filesBucket.openUploadStreamWithId(
+          filename as unknown as ObjectId,
+          filename,
+          {
+            metadata: metadata,
+          },
+        );
+
+        uploadStream.on("error", reject);
+
+        uploadStream.end(body, () => {
+          response.writeHead(201);
+          response.end();
+          resolve();
+        });
+      });
+    } else if (request.method === "DELETE") {
+      try {
+        await filesBucket.delete(filename as unknown as ObjectId);
+      } catch (err) {
+        if (err.message.startsWith("FileNotFound")) {
+          response.writeHead(404);
+          response.end("404 Not Found");
+          return;
+        }
+        throw err;
+      }
+      response.writeHead(200);
+      response.end();
+    } else {
+      response.writeHead(405, { Allow: "PUT, DELETE" });
+      response.end("405 Method Not Allowed");
+    }
+  } else if (PING_REGEX.test(url.pathname)) {
+    const host = decodeURIComponent(PING_REGEX.exec(url.pathname)[1]);
+    return new Promise((resolve) => {
+      ping(host, (err, res, stdout) => {
+        if (err) {
+          if (!res) {
+            response.writeHead(500, { Connection: "close" });
+            response.end(`${err.name}: ${err.message}`);
+            return;
+          }
+          response.writeHead(404, { "Cache-Control": "no-cache" });
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+
+        response.writeHead(200, {
+          "Content-Type": "text/plain",
+          "Cache-Control": "no-cache",
+        });
+        response.end(stdout);
+        resolve();
+      });
+    });
+  } else if (DELETE_DEVICE_REGEX.test(url.pathname)) {
+    if (request.method !== "DELETE") {
+      response.writeHead(405, { Allow: "DELETE" });
+      response.end("405 Method Not Allowed");
+      return;
+    }
+
+    const deviceId = decodeURIComponent(
+      DELETE_DEVICE_REGEX.exec(url.pathname)[1],
+    );
+
+    try {
+      await apiFunctions.deleteDevice(deviceId);
+    } catch (err) {
+      if (err instanceof ResourceLockedError) {
+        response.writeHead(503);
+        response.end("Device is in session");
+        return;
+      }
+      throw err;
+    }
+
+    response.writeHead(200);
+    response.end();
+  } else if (QUERY_REGEX.test(url.pathname)) {
+    let collectionName = QUERY_REGEX.exec(url.pathname)[1];
+
+    // Convert to camel case
+    let i = collectionName.indexOf("_");
+    while (i++ >= 0) {
+      const up =
+        i < collectionName.length ? collectionName[i].toUpperCase() : "";
+      collectionName =
+        collectionName.slice(0, i - 1) + up + collectionName.slice(i + 1);
+      i = collectionName.indexOf("_", i);
+    }
+
+    if (request.method !== "GET" && request.method !== "HEAD") {
+      response.writeHead(405, { Allow: "GET, HEAD" });
+      response.end("405 Method Not Allowed");
+      return;
+    }
+
+    const collection = collections[collectionName] as Collection<unknown>;
+    if (!collection) {
+      response.writeHead(404);
+      response.end("404 Not Found");
+      return;
+    }
+
+    let q = {};
+    if (url.searchParams.has("query")) {
+      try {
+        q = JSON.parse(url.searchParams.get("query") as string);
+      } catch (err) {
+        response.writeHead(400);
+        response.end(`${err.name}: ${err.message}`);
+        return;
+      }
+    }
+
+    switch (collectionName) {
+      case "devices":
+        q = query.expand(q);
+        break;
+      case "tasks":
+        q = query.sanitizeQueryTypes(q, {
+          _id: (v) => new ObjectId(v as string),
+          timestamp: (v) => new Date(v as number),
+          retries: Number,
+        });
+        break;
+      case "faults":
+        q = query.sanitizeQueryTypes(q, {
+          timestamp: (v) => new Date(v as number),
+          retries: Number,
+        });
+    }
+
+    let projection = null;
+    if (url.searchParams.has("projection")) {
+      projection = {};
+      for (const p of (url.searchParams.get("projection") as string).split(","))
+        projection[p.trim()] = 1;
+      projection = optimizeProjection(projection);
+    }
+
+    const cur = collection.find(q, { projection: projection });
+
+    if (url.searchParams.has("sort")) {
+      let s;
+      try {
+        s = JSON.parse(url.searchParams.get("sort") as string);
+      } catch (err) {
+        response.writeHead(400);
+        response.end(`${err.name}: ${err.message}`);
+        return;
+      }
+      const sort = {};
+      for (const [k, v] of Object.entries(s)) {
+        if (k[k.lastIndexOf(".") + 1] !== "_" && collectionName === "devices")
+          sort[`${k}._value`] = v;
+        else sort[k] = v;
+      }
+
+      cur.sort(sort);
+    }
+
+    const total = await collection.countDocuments(q);
+
+    response.writeHead(200, {
+      "Content-Type": "application/json",
+      total: total,
+    });
+
+    if (request.method === "HEAD") {
+      response.end();
+      return;
+    }
+
+    if (url.searchParams.has("skip"))
+      cur.skip(parseInt(url.searchParams.get("skip") as string));
+
+    if (url.searchParams.has("limit"))
+      cur.limit(parseInt(url.searchParams.get("limit") as string));
+
+    response.write("[\n");
+    i = 0;
+    for await (const item of cur) {
+      if (i++) response.write(",\n");
+      response.write(JSON.stringify(item));
+    }
+    response.end("\n]");
+  } else {
+    response.writeHead(404);
+    response.end("404 Not Found");
+  }
+}

lib/path-set.js

@@ -1,179 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-
-class PathSet {
-
-  constructor() {
-    this.lengthIndex = [];
-    this.fragmentIndex = [];
-  }
-
-
-  get depth() {
-    return this.lengthIndex.length;
-  }
-
-
-  add(path) {
-    let p = this.get(path);
-
-    if (p)
-      return p;
-
-    if (!this.lengthIndex[path.length])
-      this.lengthIndex[path.length] = new Set();
-
-    this.lengthIndex[path.length].add(path);
-
-    let wildcard = 0;
-    for (let i = 0; i < path.length; ++ i) {
-      let fragment = path[i];
-      if (fragment === "*")
-        wildcard = wildcard | (1 << i);
-
-      if (!this.fragmentIndex[i])
-        this.fragmentIndex[i] = {};
-
-      if (!this.fragmentIndex[i][fragment])
-        this.fragmentIndex[i][fragment] = new Set();
-
-      this.fragmentIndex[i][fragment].add(path);
-    }
-
-    if (path.wildcard == null) {
-      path.alias = 0;
-      path.wildcard = wildcard;
-      Object.freeze(path);
-    }
-    return path;
-  }
-
-
-  get(pattern) {
-    if (!this.lengthIndex[pattern.length])
-      return null;
-
-    if (this.lengthIndex[pattern.length].has(pattern))
-      return pattern;
-
-    let sets = [];
-
-    sets.push(this.lengthIndex[pattern.length]);
-
-    for (let i = 0; i < pattern.length; ++ i) {
-      if (!this.fragmentIndex[i])
-        return null;
-
-      let fragment = pattern[i];
-
-      if (!this.fragmentIndex[i][fragment])
-        return null;
-
-      sets.push(this.fragmentIndex[i][fragment]);
-    }
-
-    sets.sort(function(a, b) {
-      return a.size - b.size;
-    });
-
-    let smallestSet = sets.shift();
-    for (let path of smallestSet) {
-      for (let s of sets) {
-        if (!s.has(path)) {
-          path = null;
-          break
-        }
-      }
-      if (path)
-        return path;
-    }
-
-    return null;
-  }
-
-
-  find(pattern, superset = false, subset = false, depth = pattern.length) {
-    let res = [];
-    let groups = [[]];
-
-    for (let i = pattern.length; i <= depth; ++ i)
-      if (this.lengthIndex[i])
-        groups[0].push(this.lengthIndex[i]);
-
-    if (!groups[0].length)
-      return res;
-
-    for (let i = 0; i < pattern.length; ++ i) {
-      if (!this.fragmentIndex[i])
-        return res;
-
-      let fragment = pattern[i];
-
-      if (fragment === "*" && subset)
-        continue;
-
-      let g = [];
-
-      if (this.fragmentIndex[i][fragment])
-        g.push(this.fragmentIndex[i][fragment]);
-
-      if (fragment !== "*" && superset && this.fragmentIndex[i]["*"])
-        g.push(this.fragmentIndex[i]["*"]);
-
-      if (!g.length)
-        return res;
-
-      groups.push(g);
-    }
-
-    groups.sort(function(a, b) {
-      return a.reduce(function(prev, cur) {return prev + cur.size;}, 0) -
-        b.reduce(function(prev, cur) {return prev + cur.size;}, 0);
-    });
-
-    let smallestGroup = groups.shift();
-    for (let set of smallestGroup) {
-      for (let path of set) {
-        for (let group of groups) {
-          let found = false;
-          for (let s of group) {
-            if (s.has(path)) {
-              found = true;
-              break;
-            }
-          }
-          if (!found) {
-            path = null;
-            break;
-          }
-        }
-        if (path)
-          res.push(path);
-      }
-    }
-
-    return res;
-  }
-
-}
-
-
-module.exports = PathSet;

lib/ping.ts

@@ -0,0 +1,107 @@
+import { platform } from "node:os";
+import { execFile } from "node:child_process";
+import { domainToASCII, domainToUnicode } from "node:url";
+
+export interface PingResult {
+  packetsTransmitted: number;
+  packetsReceived: number;
+  packetLoss: number;
+  min: number;
+  avg: number;
+  max: number;
+  mdev: number;
+}
+
+function isValidHost(host: string): boolean {
+  // Valid chars in IPv4, IPv6, domain names
+  if (/^[a-zA-Z0-9\-.:[\]-]+$/.test(host)) return true;
+
+  const lowercase = host.toLowerCase();
+  const ascii = domainToASCII(lowercase);
+
+  // Check if input is an IDN convert to Punycode
+  // Can't merge with above because domainToASCII doesn't accept IP addresses
+  if (!/^[a-zA-Z0-9\-.:[\]-]+$/.test(ascii)) return false;
+
+  // Round-trip to ensure the original host matches its normalized form.
+  // This prevents bypasses where domainToASCII strips paths (e.g. "example.com/;cmd").
+  return lowercase === domainToUnicode(ascii);
+}
+
+export function parsePing(osPlatform: string, stdout: string): PingResult {
+  let parseRegExp1: RegExp, parseRegExp2: RegExp, parsed: PingResult;
+  switch (osPlatform) {
+    case "linux":
+      parseRegExp1 =
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss[^]*= ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+      parseRegExp2 =
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss/;
+      break;
+
+    case "freebsd":
+      parseRegExp1 =
+        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
+      parseRegExp2 =
+        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
+      break;
+  }
+
+  const m1 = stdout.match(parseRegExp1);
+  if (m1) {
+    parsed = {
+      packetsTransmitted: +m1[1],
+      packetsReceived: +m1[2],
+      packetLoss: +m1[3],
+      min: +m1[4],
+      avg: +m1[5],
+      max: +m1[6],
+      mdev: +m1[7],
+    };
+  } else {
+    const m2 = stdout.match(parseRegExp2);
+    if (m2) {
+      parsed = {
+        packetsTransmitted: +m2[1],
+        packetsReceived: +m2[2],
+        packetLoss: +m2[3],
+        min: null,
+        avg: null,
+        max: null,
+        mdev: null,
+      };
+    }
+  }
+  return parsed;
+}
+
+export function ping(
+  host: string,
+  callback: (err: Error, res?: PingResult, stdout?: string) => void,
+): void {
+  // Validate input to prevent possible remote code execution
+  // Credit to Alex Hordijk for reporting this vulnerability
+  // Credit to JuHwiSang for reporting a follow-up vulnerability
+  if (!isValidHost(host)) return callback(new Error("Invalid host"));
+  host = host.replace("[", "").replace("]", "");
+  let args: string[];
+
+  switch (platform()) {
+    case "linux":
+      args = ["-w", "1", "-i", "0.2", "-c", "3", "--", host];
+      break;
+    case "freebsd":
+      // Send a single packet because on FreeBSD only superuser can send
+      // packets that are only 200 ms apart.
+      args = ["-t", "1", "-c", "3", "--", host];
+      break;
+    default:
+      return callback(new Error("Platform not supported"));
+  }
+
+  // Use execFile instead of exec to avoid shell interpretation of the host
+  execFile("ping", args, (err, stdout) => {
+    if (err) return callback(err);
+    const parsed: PingResult = parsePing(platform(), stdout);
+    return callback(err, parsed, stdout);
+  });
+}

lib/query.coffee

@@ -1,371 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-common = require './common'
-mongodb = require 'mongodb'
-
-
-stringToRegexp = (input, flags) ->
-  if (input.indexOf('*') == -1)
-    return false
-
-  output = input.replace(/[\[\]\\\^\$\.\|\?\+\(\)]/, "\\$&")
-  if output[0] == '*'
-    output = output.replace(/^\*+/g, '')
-  else
-    output = '^' + output
-
-  if output[output.length - 1] == '*'
-    output = output.replace(/\*+$/g, '')
-  else
-    output = output + '$'
-
-  output = output.replace(/[\*]/, '.*')
-
-  return new RegExp(output, flags)
-
-
-normalize = (input) ->
-  if common.typeOf(input) is common.STRING_TYPE
-    vals = [input]
-    if (m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input))
-      vals.push({'$regex' : new RegExp(m[1], m[2])})
-
-    f = parseFloat(input)
-    if not isNaN(f)
-      vals.push(f)
-
-    d = new Date(input)
-    if input.length >= 8 and d.getFullYear() > 1983
-      vals.push(d)
-
-    r = stringToRegexp(input)
-    if r isnt false
-      vals.push({'$regex' : r})
-
-    return vals
-  return input
-
-
-expandValue = (value) ->
-  if common.typeOf(value) is common.ARRAY_TYPE
-    a = []
-    for j in value
-      a = a.concat(expandValue(j))
-    return [a]
-  else if common.typeOf(value) isnt common.OBJECT_TYPE
-    n = normalize(value)
-    if common.typeOf(n) isnt common.ARRAY_TYPE
-      return [n]
-    else
-      return n
-
-  objs = []
-  indices = []
-  keys = []
-  values = []
-  for k,v of value
-    keys.push(k)
-    values.push(expandValue(v))
-    indices.push(0)
-
-  i = 0
-  while i < indices.length
-    obj = {}
-    for i in [0...keys.length]
-      obj[keys[i]] = values[i][indices[i]]
-    objs.push(obj)
-
-    for i in [0...indices.length]
-      indices[i] += 1
-      if indices[i] < values[i].length
-        break
-      indices[i] = 0
-  return objs
-
-
-permute = (param, val) ->
-  conditions = []
-
-  values = expandValue(val)
-  if param[param.lastIndexOf('.') + 1] != '_'
-    param += '._value'
-
-  for v in values
-    obj = {}
-    obj[param] = v
-    conditions.push(obj)
-
-  return conditions
-
-
-expand = (query) ->
-  new_query = {}
-  for k,v of query
-    if k[0] == '$' # operator
-      expressions = []
-      for e in v
-        expressions.push(expand(e))
-      new_query[k] = expressions
-    else
-      conditions = permute(k, v)
-      if conditions.length > 1
-        new_query['$and'] ?= []
-        if v?['$ne']?
-          for c in conditions
-            new_query['$and'].push(c)
-        else
-          new_query['$and'].push({'$or' : conditions})
-      else
-        Object.assign(new_query, conditions[0])
-
-  return new_query
-
-
-# Generate parameter projection from a query
-# If second arg is given, it's edited and returned
-queryProjection = (query, proj) ->
-  proj ?= {}
-  for k,v of query
-    if k.charAt(0) == '$' # this is a logical operator
-      for q in v
-        queryProjection(q, proj)
-    else
-      proj[k] = 1
-  return proj
-
-
-testExpressions = (params, expressions, lop) ->
-  for f in expressions
-    res = test(params, f)
-
-    switch lop
-      when '$and'
-        return false if not res
-      when '$or'
-        return true if res
-      when '$nor'
-        return false if res
-
-  return switch lop
-    when '$and' then true
-    when '$or' then false
-    when '$nor' then true
-    else throw new Error('Unknown logical operator')
-
-
-test = (params, query) ->
-  for k,v of query
-    if k.charAt(0) == '$' # this is a logical operator
-      res = testExpressions(params, v, k)
-    else
-      value = params[k]
-
-      if common.typeOf(v) isnt common.OBJECT_TYPE
-        # TODO comparing array to regex, array to array, and object to object
-        if common.typeOf(value) is common.ARRAY_TYPE
-          res = value.indexOf(v) != -1
-        else
-          if common.typeOf(v) is common.REGEXP_TYPE
-            res = v.test(value)
-          else
-            res = v == value
-      else
-        for k2,v2 of v
-          switch k2
-            when '$ne'
-              if common.typeOf(value) is common.ARRAY_TYPE
-                res = value.indexOf(v2) == -1
-              else
-                res = value != v2
-            when '$lt'
-              res = value < v2
-            when '$lte'
-              res = value <= v2
-            when '$gt'
-              res = value > v2
-            when '$gte'
-              res = value >= v2
-            when '$regex'
-              res = v2.test(value)
-            when '$in'
-              throw new Error('Operator not supported')
-            when '$nin'
-              throw new Error('Operator not supported')
-            when '$all'
-              throw new Error('Operator not supported')
-            when '$exists'
-              throw new Error('Operator not supported')
-            else
-              throw new Error('Operator not supported')
-
-    if not res
-      return false
-
-  return true
-
-
-matchType = (src, dst) ->
-  if typeof src is 'string'
-    return String(dst)
-  else if typeof src is 'number'
-    if +dst == parseFloat(dst)
-      return +dst
-    else if not isNaN(Date.parse(dst))
-      return Date.parse(dst)
-  else if typeof src is 'boolean'
-    v = String(dst).trim()
-    if v is 'true' or v is 'TRUE' or v is 'True' or v is '1'
-      return true
-    else if v is 'false' or v is 'FALSE' or v is 'False' or v is '0'
-      return false
-
-  return dst
-
-
-testFilter = (obj, filter) ->
-  for k, v of filter
-    [param, op] = k.split(/([^a-zA-Z0-9\-\_\.].*)/, 2)
-    val = matchType(obj[param], v)
-    switch op
-      when '=', undefined
-        return false if obj[param] != val
-      when '>'
-        return false if not (obj[param] > val)
-      when '>='
-        return false if not (obj[param] >= val)
-      when '<'
-        return false if not (obj[param] < val)
-      when '<='
-        return false if not (obj[param] <= val)
-      when '!'
-        return false if obj[param] == val
-      else
-        throw new Error("Unrecognized operator #{op}")
-
-  return true
-
-
-convertMongoQueryToFilters = (query, filters) ->
-  filters ?= {}
-  for k, v of query
-    if k[0] == '$'
-      if k == '$and'
-        for vv in v
-          convertMongoQueryToFilters(vv, filters)
-      else
-        throw new Error("Operator #{k} not supported")
-    else if k == '_tags'
-      if common.typeOf(v) is common.OBJECT_TYPE
-        for kk, vv of v
-          vv = vv.replace(/[^a-zA-Z0-9\-]+/g, '_')
-          switch kk
-            when '$ne'
-              filters["Tags.#{vv}!"] = true
-            else
-              throw new Error("Operator #{kk} not supported")
-      else
-        v = v.replace(/[^a-zA-Z0-9\-]+/g, '_')
-        filters["Tags.#{v}"] = true
-    else
-      switch k
-        when '_id' then k = 'DeviceID.ID'
-        when '_deviceId._Manufacturer' then k = 'DeviceID.Manufacturer'
-        when '_deviceId._OUI' then k = 'DeviceID.OUI'
-        when '_deviceId._ProductClass' then k = 'DeviceID.ProductClass'
-        when '_deviceId._SerialNumber' then k = 'DeviceID.SerialNumber'
-        when '_lastInform' then k = 'Events.Inform'
-        when '_lastBootstrap' then k = 'Events.0_BOOTSTRAP'
-        when '_lastBoot' then k = 'Events.1_BOOT'
-        when '_registered' then k = 'Events.Registered'
-
-      if common.typeOf(v) is common.OBJECT_TYPE
-        for kk, vv of v
-          switch kk
-            when '$eq'
-              filters[k] = vv
-            when '$ne'
-              filters["#{k}!"] = vv
-            when '$lt'
-              filters["#{k}<"] = vv
-            when '$lte'
-              filters["#{k}<="] = vv
-            when '$gt'
-              filters["#{k}>"] = vv
-            when '$gte'
-              filters["#{k}>="] = vv
-            else
-              throw new Error("Oprator #{kk} not supported")
-      else
-        filters[k] = v
-
-  return filters
-
-
-sanitizeQueryTypes = (query, types) ->
-  for k, v of query
-    if k[0] == '$' # logical operator
-      sanitizeQueryTypes(vv, types) for vv in v
-    else if k of types
-      if common.typeOf(v) is common.OBJECT_TYPE
-        for kk, vv of v
-          switch kk # operator
-            when '$in', '$nin'
-              vv[i] = types[k](vv[i]) for i in [0...vv.length]
-            when '$eq', '$gt', '$gte', '$lt', '$lte', '$ne'
-              v[kk] = types[k](vv)
-            when '$exists', '$type' then
-              # ignore
-            else
-              throw new Error('Operator not supported')
-      else
-        query[k] = types[k](query[k])
-
-  return query
-
-
-exports.expand = expand
-exports.queryProjection = queryProjection
-exports.test = test
-exports.sanitizeQueryTypes = sanitizeQueryTypes
-exports.convertMongoQueryToFilters = convertMongoQueryToFilters
-exports.testFilter = testFilter