implementação de mudança de interface

Signed-off-by: Rodrigo Bertoldo de Sousa Martins <rodrigo.sousa.577@gmail.com>

.dockerignore

@@ -0,0 +1,28 @@
+# Dependências e artefatos de build
+node_modules/
+dist/
+
+# Arquivos de ambiente (contêm segredos)
+.env
+
+# Logs e debug
+logs/
+*.log
+
+# Artefatos de teste
+test/*.js
+
+# Docs geradas
+docs/_build/
+
+# Dados de desenvolvimento
+.git/
+.github/
+
+# IDEs
+.vscode/
+.idea/
+
+# OS files
+.DS_Store
+Thumbs.db

.env.example

@@ -0,0 +1,69 @@
+# ============================================================
+# GenieACS — variáveis de ambiente
+# Copie este arquivo para .env e ajuste os valores
+# NUNCA faça commit do .env com dados reais em repositórios
+# públicos.
+# ============================================================
+
+# ── MongoDB ─────────────────────────────────────────────────
+# Quando usando docker-compose o host é o nome do serviço "mongo"
+GENIEACS_MONGODB_CONNECTION_URL=mongodb://mongo:27017/genieacs
+
+# ── Extensões ───────────────────────────────────────────────
+GENIEACS_EXT_DIR=/opt/genieacs/config/ext
+GENIEACS_EXT_TIMEOUT=3000
+
+# ── Logs ────────────────────────────────────────────────────
+# json | simple
+GENIEACS_LOG_FORMAT=json
+GENIEACS_ACCESS_LOG_FORMAT=json
+
+# Para gravar logs em arquivo (opcional, senão vai para stdout/stderr)
+# GENIEACS_CWMP_LOG_FILE=/opt/genieacs/logs/cwmp.log
+# GENIEACS_CWMP_ACCESS_LOG_FILE=/opt/genieacs/logs/cwmp-access.log
+# GENIEACS_NBI_LOG_FILE=/opt/genieacs/logs/nbi.log
+# GENIEACS_NBI_ACCESS_LOG_FILE=/opt/genieacs/logs/nbi-access.log
+# GENIEACS_FS_LOG_FILE=/opt/genieacs/logs/fs.log
+# GENIEACS_FS_ACCESS_LOG_FILE=/opt/genieacs/logs/fs-access.log
+# GENIEACS_UI_LOG_FILE=/opt/genieacs/logs/ui.log
+# GENIEACS_UI_ACCESS_LOG_FILE=/opt/genieacs/logs/ui-access.log
+
+# ── Debug (opcional) ────────────────────────────────────────
+# GENIEACS_DEBUG_FILE=/opt/genieacs/logs/debug.yaml
+# GENIEACS_DEBUG_FORMAT=yaml
+
+# ── Worker processes (0 = 1 por CPU core) ───────────────────
+GENIEACS_CWMP_WORKER_PROCESSES=1
+GENIEACS_NBI_WORKER_PROCESSES=1
+GENIEACS_FS_WORKER_PROCESSES=1
+GENIEACS_UI_WORKER_PROCESSES=1
+
+# ── Portas ──────────────────────────────────────────────────
+GENIEACS_CWMP_PORT=7547
+GENIEACS_NBI_PORT=7557
+GENIEACS_FS_PORT=7567
+GENIEACS_UI_PORT=3000
+
+# ── Interface de bind (:: = todas as interfaces) ────────────
+GENIEACS_CWMP_INTERFACE=::
+GENIEACS_NBI_INTERFACE=::
+GENIEACS_FS_INTERFACE=::
+GENIEACS_UI_INTERFACE=::
+
+# ── UI — JWT Secret (OBRIGATÓRIO trocar em produção!) ───────
+# Mínimo 32 chars, máximo 64 chars
+GENIEACS_UI_JWT_SECRET=mude-isto-em-producao-use-uma-string-aleatoria-longa
+
+# ── FS — URL prefix (configure se CWMP e FS estiverem em ────
+#         hosts/IPs diferentes ou atrás de proxy)            
+# GENIEACS_FS_URL_PREFIX=http://seu-ip-publico:7567/
+
+# ── SSL (opcional — passe o caminho para cert/key PEM) ──────
+# GENIEACS_CWMP_SSL_CERT=/opt/genieacs/certs/cwmp.crt
+# GENIEACS_CWMP_SSL_KEY=/opt/genieacs/certs/cwmp.key
+# GENIEACS_NBI_SSL_CERT=/opt/genieacs/certs/nbi.crt
+# GENIEACS_NBI_SSL_KEY=/opt/genieacs/certs/nbi.key
+# GENIEACS_FS_SSL_CERT=/opt/genieacs/certs/fs.crt
+# GENIEACS_FS_SSL_KEY=/opt/genieacs/certs/fs.key
+# GENIEACS_UI_SSL_CERT=/opt/genieacs/certs/ui.crt
+# GENIEACS_UI_SSL_KEY=/opt/genieacs/certs/ui.key

.eslintignore

@@ -1,2 +0,0 @@
-!/bin/.*.ts
-!/tools/.*.ts

.eslintrc.json

@@ -1,27 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true
-  },
-  "parser": "@typescript-eslint/parser",
-  "parserOptions": {
-    "project": "./tsconfig.json"
-  },
-  "plugins": ["@typescript-eslint"],
-  "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended", "prettier", "prettier/@typescript-eslint"],
-  "rules": {
-    "no-shadow": ["error", { "allow": ["err"] }],
-    "handle-callback-err": "error",
-    "prefer-arrow-callback": "error",
-    "curly": ["error", "multi-or-nest", "consistent"],
-    "no-buffer-constructor": "error",
-    "prefer-const": ["error", {"destructuring": "all"}],
-    "eqeqeq": ["error", "always", {"null": "ignore"}],
-    "no-return-await": "error",
-    "@typescript-eslint/no-explicit-any": 0,
-    "@typescript-eslint/no-use-before-define": ["error", { "functions": false }],
-    "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
-    "require-atomic-updates": 0,
-    "no-prototype-builtins": 0
-  }
-}

.github/copilot-instructions.md

@@ -0,0 +1,50 @@
+GitHub Copilot Instructions: GenieACS -> UniFi UI Theme
+You are an expert senior software engineer and frontend architect focused on UI/UX modernization, maintainability, and clean architecture. Your primary goal is to help modify the GenieACS user interface to closely resemble the UniFi Controller, while strictly ensuring the codebase remains easy to synchronize (via merge or rebase) with the original GenieACS upstream repository for bug fixes and core updates.
+
+1. Core Principles (Non-Negotiable)
+Sync-Friendly Architecture (Crucial): Write code defensively to prevent future merge conflicts with the original GenieACS project. Prefer extending, overriding, or wrapping components rather than heavily modifying deeply ingrained original core files.
+
+Visual Fidelity (UniFi): Implement the clean, dashboard-centric, modern aesthetic of the UniFi Controller. Use comparable color palettes, typography, spacing, and network topology layouts.
+
+Safety First: Never modify the underlying TR-069 communication logic, backend models, or database schemas unless explicitly requested. The focus is strictly on the presentation layer (View).
+
+Minimal Footprint: When generating files, aim to isolate custom UI assets. Keep custom CSS/SCSS and new UI components in distinct directories or files whenever possible.
+
+2. GenieACS Customization & Upstream Integration
+CRITICAL: Whenever the user requests a modification to a page or component, you MUST prioritize the following strategies to keep the fork maintainable:
+
+Style Overrides: Prefer creating custom stylesheets (e.g., unifi-theme.scss) to override existing GenieACS classes instead of changing inline styles or the original CSS files directly.
+
+Component Wrapping: If an existing UI component needs significant structural changes, create a new "UniFi-styled" wrapper component that consumes the original GenieACS state/props, rather than gutting the original component.
+
+Clear Demarcation: When modifying an original GenieACS file is unavoidable, clearly mark the beginning and end of the custom code with comments (e.g., // --- BEGIN UNIFI THEME --- and // --- END UNIFI THEME ---). This makes resolving Git merge conflicts significantly easier.
+
+3. Code Generation Rules
+A. Refactoring & Modification
+Context Awareness: Read the surrounding code to understand the existing state management and API calls. Do not break the data flow between the GenieACS backend and the frontend.
+
+Preserve Logic: When changing how data is displayed (e.g., changing a simple table to a UniFi-style card grid), ensure all original data points remain accessible or are intentionally hidden via CSS, rather than deleting the data-fetching logic.
+
+B. New Features (UI Components)
+Modularity: Build reusable UI components (Buttons, Cards, Navbars) that match the UniFi design system, so they can be applied globally across the GenieACS interface.
+
+Responsiveness: Ensure all new UI elements are fully responsive, as network management dashboards are often viewed on various screen sizes.
+
+C. Testing
+Prioritize visual regression checks and ensure that frontend unit tests (if applicable) are updated to reflect structural DOM changes. Ensure the connection to the GenieACS API remains mocked and functional in tests.
+
+4. Response Format
+Plan First: Before generating code, briefly outline your strategy. Specifically state which files will be modified and how you plan to implement the change while minimizing upstream merge conflicts.
+
+Complete Code: Output the full, functional block of code needed for the file. Do not use placeholders like // ... rest of code unless the file is massive and the change is highly localized.
+
+Git Context: If a change requires creating a new file to override an old one, explain how to integrate it into the project's build system.
+
+5. "Do No Harm" Checklist
+Before outputting the final response, ask yourself:
+
+Does this modification break the core GenieACS device management logic?
+
+Will this change cause a massive merge conflict if the upstream repository updates this specific component? (If yes, rethink the approach).
+
+Does the resulting UI genuinely reflect the UniFi Controller design language?

.gitignore

@@ -1,6 +1,5 @@
 *~
 node_modules
 dist
-bin/.*.ts
-tools/.*.ts
 docs/_build
+.env

.npmignore

@@ -1,25 +0,0 @@
-*~
-.npmignore
-
-config/config.json
-config/cwmp.key
-config/cwmp.crt
-config/cwmp.ca-bundle
-config/nbi.key
-config/nbi.crt
-config/nbi.ca-bundle
-config/fs.key
-config/fs.crt
-config/fs.ca-bundle
-config/ext
-
-debug/*
-!debug/.gitkeep
-
-dist
-.eslintrc.json
-.eslintignore
-ava.config.js
-tsconfig.json
-bin/.*
-tools/.*

.prettierignore

@@ -0,0 +1,2 @@
+dist
+npm-shrinkwrap.json

AGENTS.md

@@ -0,0 +1,82 @@
+# AGENTS.md — GenieACS
+
+GenieACS is a TR-069 Auto Configuration Server for remote management of CPE
+devices (routers, modems, gateways). TypeScript codebase compiled with esbuild,
+backed by MongoDB.
+
+## Architecture Overview
+
+Four services share a single MongoDB instance:
+
+- **CWMP** (port 7547) — TR-069 protocol handler; manages device sessions
+- **NBI** (port 7557) — Northbound REST API for external consumers
+- **FS** (port 7567) — File server for firmware/config (GridFS-backed)
+- **UI** (port 3000) — Web interface (Koa backend + Mithril.js SPA frontend)
+
+Key subsystems: expression engine (`lib/common/expression/`) compiles a
+Lisp-like DSL used for queries, config, and authorization; session engine
+(`lib/session.ts`) drives CWMP interactions via declarations rather than
+imperative RPCs; sandbox (`lib/sandbox.ts`) runs user-defined provision scripts
+in `vm.Script` with deterministic replay.
+
+Read `ARCHITECTURE.md` for a full map of the codebase when working on unfamiliar
+areas. It covers service boundaries, the expression pipeline, the path system,
+the CWMP session state machine, the database layer, and architectural
+invariants.
+
+## Project Structure
+
+- `lib/` — Core server-side logic
+- `lib/common/` — Shared code (runs in both Node.js and browser)
+- `lib/db/` — MongoDB database layer
+- `lib/ui/` — UI backend helpers
+- `ui/` — Frontend SPA (Mithril.js)
+- `bin/` — Service entry points (5 executables)
+- `build/` — Build scripts (esbuild pipeline)
+- `test/` — Unit tests (node:test)
+- `docs/` — User docs (Sphinx/reStructuredText)
+- `public/` — Static assets (favicon, logo)
+
+## Build / Lint / Test Commands
+
+```bash
+npm run build # Production build (esbuild pipeline -> dist/)
+NODE_ENV=development npm run build # Dev build (no minification)
+npm run lint # Prettier + ESLint + tsc --noEmit in parallel
+npm test # Compile tests with esbuild, run with node --test
+```
+
+### Running a Single Test File
+
+```bash
+esbuild --log-level=warning --bundle --platform=node --target=node18 \
+  --packages=external --sourcemap=inline --outdir=test test/path.ts \
+  && node --test --enable-source-maps test/path.js \
+  && rm test/path.js
+```
+
+### Running a Single Test Case
+
+```bash
+esbuild --log-level=warning --bundle --platform=node --target=node18 \
+  --packages=external --sourcemap=inline --outdir=test test/path.ts \
+  && node --test --enable-source-maps --test-name-pattern="^parse$" test/path.js \
+  && rm test/path.js
+```
+
+### Lint Sub-commands
+
+```bash
+prettier --prose-wrap always --write .
+eslint 'bin/*.ts' 'lib/**/*.ts' 'ui/**/*.ts' 'test/**/*.ts' 'build/**/*.ts'
+tsc --noEmit
+```
+
+## Before Committing
+
+Read `CONTRIBUTING.md` and ensure your changes comply with it. In particular:
+
+- Run `npm run lint` and `npm test` and fix any failures.
+- Follow the code style, naming, import, and comment conventions documented
+  there.
+- Use the Conventional Commits format for commit messages.

ARCHITECTURE.md

@@ -0,0 +1,479 @@
+# Architecture
+
+This document describes the high-level architecture of GenieACS. If you want to
+familiarize yourself with the codebase, you are in the right place.
+
+## Bird's Eye View
+
+GenieACS is a TR-069 Auto Configuration Server (ACS). It manages CPE devices
+(routers, modems, gateways) using the CWMP protocol (TR-069). The system
+consists of four network-facing services that share a MongoDB database:
+
+```
+                  +------------+
+  CPE Devices --->| CWMP (7547)|-+
+                  +------------+ |
+                  +-----------+  |     +---------+
+  CPE Devices --->| FS (7567) |--+---->| MongoDB |
+                  +-----------+  |     +---------+
+                  +-----------+  |
+  OSS / Scripts ->| NBI (7557)|--+
+                  +-----------+  |
+                  +-----------+  |
+  Administrators->| UI (3000) |--+
+                  +-----------+
+```
+
+- **CWMP** -- The core TR-069 protocol server. CPE devices connect here to
+  report their state and receive configuration instructions via SOAP/XML over
+  HTTP.
+- **NBI** -- Northbound Interface. A REST API for external systems (OSS/BSS,
+  scripts, automation) to manage devices, tasks, presets, and configuration
+  programmatically.
+- **FS** -- File Server. Serves firmware images and configuration files to CPE
+  devices during download operations.
+- **UI** -- Web interface. A Koa-based backend serving a Mithril.js single-page
+  application for administrators to browse devices, manage configuration, and
+  trigger operations.
+
+All four services follow the same process model: a primary process forks
+configurable worker processes via Node.js `cluster` (see `cluster.ts`). Workers
+connect to MongoDB and start an HTTP(S) server.
+
+## Code Map
+
+```
+bin/                        Service entry points (5 executables)
+lib/                        All backend logic
+  common/                   Shared utilities (expression engine, Path, errors)
+    expression.ts           Expression class hierarchy (base + subclasses)
+    expression/             Expression parser, evaluator, normalizer, minimizer
+  cwmp/                     CWMP-service-specific DB and caching
+  db/                       Database layer (MongoDB collections, query synthesis)
+  ui/                       UI-service-specific API, DB, and caching
+  types/                    (empty, reserved)
+ui/                         Frontend SPA (Mithril.js)
+  components/               Reusable UI components (parameter, tags, ping, etc.)
+  css/                      Stylesheets (vanilla CSS)
+  icons/                    SVG icons (compiled into a sprite)
+test/                       Unit tests (Node.js native test runner)
+build/                      Build scripts (esbuild-based)
+public/                     Static assets (logo, favicon)
+```
+
+### Entry Points (`bin/`)
+
+Each file in `bin/` bootstraps one service. They are structurally identical:
+initialize logging, read config, fork workers in the primary process, connect to
+MongoDB and start the HTTP server in each worker.
+
+- `genieacs-cwmp.ts` -- CWMP service. Unique in that it disables HTTP keep-alive
+  (`keepAliveTimeout: 0`) and provides custom `onConnection` / `onClientError`
+  hooks for TR-069 session lifecycle management.
+- `genieacs-nbi.ts` -- NBI service. Straightforward REST API server.
+- `genieacs-fs.ts` -- File server. The leanest service; does not use the
+  extensions subsystem.
+- `genieacs-ui.ts` -- UI service. Wraps the Koa application as the HTTP
+  listener.
+- `genieacs-ext.ts` -- **Not a service.** This is a child process worker spawned
+  by the extensions subsystem. It communicates with its parent via IPC,
+  executing user-defined extension scripts in isolation.
+
+### The Expression System (`lib/common/expression.ts`, `lib/common/expression/`)
+
+The `Expression` class (defined in `lib/common/expression.ts`) is the most
+important abstraction in the codebase. It uses a typed class hierarchy:
+`Expression.Literal` (wraps `string | number | boolean | null`),
+`Expression.Parameter` (wraps a `Path`), `Expression.Binary` (operator +
+left/right), `Expression.Unary` (operator + operand), `Expression.FunctionCall`
+(name + args), and `Expression.Conditional` (condition/then/otherwise). For
+example, in the SQL-like text syntax:
+
+```
+Device.ModelName = "BrandX" AND Events.Inform > 1000
+```
+
+Expressions are used pervasively:
+
+- **Query/filter language** -- Database queries for devices, faults, presets,
+  etc. are all represented as expressions and compiled to MongoDB filters.
+- **Configuration values** -- Config entries and preset preconditions are
+  expressions, enabling dynamic evaluation.
+- **Authorization** -- Permission filters and validators are expressions.
+- **Pagination cursors** -- Keyset pagination boundaries are expressed as filter
+  expressions.
+
+The expression pipeline flows through four modules:
+
+1. `parser.ts` -- Parses a SQL-like text syntax into the AST using a hand-rolled
+   recursive descent parser with a `Cursor`-based scanner. Also provides
+   `stringifyExpression()` for serialization. The `map()` / `mapAsync()`
+   tree-walking primitives are abstract methods on the `Expression` class, and
+   `stringify()` is now `Expression.toString()`.
+2. `normalize.ts` -- Algebraic normalization using exact rational polynomial
+   arithmetic over native `bigint` values. The `Polynomial` class extends
+   `Expression` as an intermediate representation. Ensures equivalent
+   expressions have the same canonical form (e.g., `a + 2 > b` and `a > b - 2`
+   normalize identically).
+3. `synth.ts` -- Boolean logic minimization via the Espresso algorithm
+   (espresso-iisojs). Converts expressions to minimal sum-of-products form with
+   three-valued logic (true/false/null). Handles domain-specific constraints
+   like comparison ordering and LIKE pattern relationships.
+4. `evaluate.ts` -- Runtime evaluation. The `reduce()` function evaluates
+   operators on literal values and supports partial evaluation (returns a
+   reduced expression if some values are unknown). Parameter resolution is
+   handled by the `Expression.evaluate()` method (in `lib/common/expression.ts`)
+   which calls `reduce()` after mapping children via a user-supplied callback.
+
+`pagination.ts` implements cursor-based pagination by generating filter
+expressions from sort-key bookmarks.
+
+### The Path System (`lib/common/path.ts`, `lib/common/path-set.ts`)
+
+`Path` represents a TR-069 parameter path (e.g., `Device.WiFi.SSID.1.Name`).
+Paths can contain wildcards (`*`) and alias expressions (`[key:value]`) for
+query-based addressing.
+
+Key design decisions:
+
+- **Cached** -- `Path.parse()` caches instances in a two-generation LRU cache
+  rotated every 120 seconds. The constructor is public (used directly by the
+  parser and by methods like `slice()`, `concat()`, and `stripAlias()`).
+- **Bitmask encoding** -- The `wildcard` and `alias` fields are bitfields for
+  O(1) segment-type checking. This limits paths to 32 segments. A `colon` field
+  tracks the number of attribute path segments (after a `:` separator), enabling
+  `paramLength` and `attrLength` accessors.
+- **Immutable** -- Segment arrays are `Object.freeze()`-d.
+
+`PathSet` is a multi-indexed collection of paths supporting pattern-matching
+queries. It maintains separate `paramSegmentIndex` and `attrSegmentIndex` arrays
+(one `Map<string, Set<Path>>` per position), plus a `stringIndex` map. The
+`find()` method takes bitmasks to control which segments require exact matches
+vs. wildcard compatibility, then uses set intersection across the smallest index
+sets. A higher-level `findCompat()` method computes the appropriate bitmasks for
+superset/subset matching.
+
+### CWMP Protocol Layer (`lib/cwmp.ts`, `lib/soap.ts`, `lib/xml-parser.ts`)
+
+`xml-parser.ts` is a custom single-pass XML parser (no DOM). It scans
+character-by-character with bitwise state flags, building a tree of elements
+with namespace support. Does not support CDATA.
+
+`soap.ts` handles both parsing CPE SOAP messages and generating ACS SOAP
+responses. It dispatches on the SOAP body's method name to type-specific parsers
+for Inform, TransferComplete, GetParameterNamesResponse, etc. Supports CWMP
+versions 1.0 through 1.4.
+
+`cwmp.ts` is the HTTP-level CWMP request handler. It manages the session state
+machine:
+
+1. **State 0** -- Expects an Inform. Authenticates the device (Basic or Digest
+   auth, configurable via expression). Acquires a distributed lock. Loads device
+   data from MongoDB. Sends InformResponse.
+2. **State 1** -- Waits for the CPE to send an empty POST (ready for ACS RPCs).
+   Processes any TransferComplete messages.
+3. **State 2** -- The ACS drives RPCs (GetParameterNames, GetParameterValues,
+   SetParameterValues, AddObject, DeleteObject, Download, Reboot, FactoryReset).
+   The CPE responds to each.
+
+Session persistence across TCP disconnects: when a socket closes mid-session,
+the entire `SessionContext` is serialized to Redis (the MongoDB `cache`
+collection) and restored when the CPE reconnects (identified by a session
+cookie).
+
+### Session Engine (`lib/session.ts`)
+
+The session engine implements the **declaration-driven data fetching** pattern.
+Rather than issuing RPCs imperatively, provisions create `Declaration` objects
+stating what paths and attributes they need to read or write. The engine then:
+
+1. Processes all declarations into a `SyncState` -- a structured plan of which
+   parameters to refresh, which values to set, which instances to create/delete,
+   etc.
+2. Generates the minimal set of CWMP RPCs needed to fulfill the plan.
+3. After each RPC response, updates `DeviceData` and re-evaluates.
+4. Iterates until all declarations are satisfied.
+
+The preset system (`applyPresets` in `cwmp.ts`) implements a policy engine:
+presets are rules with precondition expressions that, when matched, contribute
+provisions to the session. After provisions execute, if device data changed,
+presets are re-evaluated (up to 4 cycles to prevent infinite loops).
+
+### Provisions and Virtual Parameters
+
+**Provisions** are the unit of configuration intent. Built-in provisions
+(`default-provisions.ts`) include `refresh`, `value`, `tag`, `reboot`, `reset`,
+`download`, and `instances`. Custom provisions are user-defined JavaScript
+scripts stored in the `provisions` MongoDB collection.
+
+**Virtual parameters** are scripts that present computed/derived values as if
+they were real device parameters under the `VirtualParameters.*` namespace. They
+run in two phases: a "get" phase reads real parameters and returns a computed
+value; a "set" phase translates a desired value into real parameter changes.
+Virtual parameters can reference other virtual parameters (up to depth 8).
+
+### Sandbox (`lib/sandbox.ts`)
+
+The sandbox provides a secure execution environment for provision and virtual
+parameter scripts using `vm.Script` with a 50ms timeout. It uses a
+**replay-based execution model**:
+
+1. A script runs and calls `declare()` to request data.
+2. When `commit()` is called, the script throws a sentinel symbol and exits.
+3. The engine fetches the requested data via CWMP RPCs.
+4. The script is **re-run from the beginning** with the fetched data available.
+5. Earlier `declare()` calls return cached results; the script progresses
+   further.
+6. This repeats until the script completes without throwing.
+
+The sandbox API: `declare(path, timestamps, values)` returns a
+`ParameterWrapper` proxy; `clear(path, timestamp, attributes)` invalidates
+cached data; `ext(...args)` calls external extensions (results are cached per
+revision to survive replays); `commit()` explicitly triggers a fetch cycle.
+`Math.random()` is replaced with a seeded PRNG for determinism.
+
+### Device Data Model (`lib/types.ts`, `lib/device.ts`)
+
+`DeviceData` is the in-memory working copy of a device's parameter tree during a
+session:
+
+- `paths: PathSet` -- All known parameter paths.
+- `timestamps: VersionedMap<Path, number>` -- When each path was last refreshed.
+- `attributes: VersionedMap<Path, Attributes>` -- Per-path attributes (object,
+  writable, value, notification, accessList), each paired with a timestamp.
+- `trackers` / `changes` -- Change tracking for re-evaluation.
+
+`VersionedMap` (in `versioned-map.ts`) provides multi-revision snapshots,
+enabling the sandbox replay model where scripts may be re-run at different
+revision levels.
+
+`device.ts` handles setting and clearing parameter data with invariant
+enforcement (e.g., if `value` is set, `object` is forced to 0; parent paths are
+ensured to exist). The `unpack()` function resolves wildcards and alias paths
+against concrete device data.
+
+### NBI (`lib/nbi.ts`)
+
+A raw Node.js HTTP listener (no framework) with regex-based URL routing.
+Endpoints include CRUD for presets, provisions, virtual parameters, objects, and
+files; device task management (with optional synchronous execution via
+connection request); fault management; generic collection querying; and ping.
+
+The NBI uses MongoDB-style JSON queries directly. For the `devices` collection,
+`query.ts` expands user-friendly queries by auto-appending `._value` to
+parameter paths and generating multi-type interpretations (string, number, date,
+regex) for filter values.
+
+### File Server (`lib/fs.ts`)
+
+A minimal HTTP file server reading from MongoDB GridFS. Supports GET/HEAD with
+full HTTP caching (ETag, Last-Modified, If-None-Match, If-Modified-Since) and
+Range requests (HTTP 206) for partial content downloads. Files are cached
+in-memory via memoization.
+
+### UI Backend (`lib/ui.ts`, `lib/ui/`)
+
+A Koa application with JWT authentication, role-based authorization, and a rich
+CRUD API under `/api/`. The root route serves an HTML shell that bootstraps the
+Mithril.js SPA with injected config, user info, and hashed asset filenames.
+
+`lib/ui/api.ts` defines generic CRUD endpoints for all resource types (devices,
+presets, provisions, files, config, users, permissions, faults, tasks) with
+authorization checks at every level. Specialized endpoints handle file
+upload/download, synchronous task execution, tag management, password changes,
+and CSV export.
+
+`lib/ui/db.ts` translates between the UI's flat parameter representation and
+MongoDB's nested document structure. The `flattenDevice()` function is the key
+transformation: it recursively walks the nested device document and produces a
+flat key-value map with colon-delimited attribute keys (e.g.,
+`"Device.WiFi.SSID.1.Name" -> value`,
+`"Device.WiFi.SSID.1.Name:type" -> "xsd:string"`,
+`"Device.WiFi.SSID.1.Name:writable" -> true`). The `FlatDevice` type is
+`Record<string, Value>` where `Value = string | number | boolean | null`.
+
+`lib/ui/local-cache.ts` caches permissions, users, and config in-process with
+hash-based revision tracking. The `getConfig()` function uses typed overloads --
+it takes a typed default value (`string`, `number`, or `boolean`) and an
+expression evaluation callback, returning a value of the same type as the
+default.
+
+### Database Layer (`lib/db/`)
+
+`db/db.ts` is the single entry point to MongoDB. It manages 14 collections:
+
+| Collection          | Purpose                          |
+| ------------------- | -------------------------------- |
+| `devices`           | CPE device parameter trees       |
+| `presets`           | Configuration rules              |
+| `provisions`        | Provision scripts                |
+| `virtualParameters` | Virtual parameter scripts        |
+| `objects`           | Generic objects                  |
+| `tasks`             | Queued device management tasks   |
+| `faults`            | Error records with retry state   |
+| `operations`        | In-flight async operations       |
+| `files` (GridFS)    | Firmware images and config files |
+| `permissions`       | RBAC rules                       |
+| `users`             | User accounts                    |
+| `config`            | Key-value configuration          |
+| `cache`             | Distributed cache (TTL index)    |
+| `locks`             | Distributed locks (TTL index)    |
+
+`db/synth.ts` is the sophisticated expression-to-MongoDB query compiler. It
+normalizes expressions, converts them to a Boolean satisfiability representation
+using the `Clause` hierarchy, minimizes via Espresso, and emits MongoDB
+`$and`/`$or`/`$not` filter objects. This is used by the UI backend.
+
+`cwmp/db.ts` handles CWMP-specific persistence: loading the nested device
+document into `DeviceData` (`fetchDevice`), diffing changes back into MongoDB
+update operations (`saveDevice`), and managing faults, tasks, and operations.
+
+### Query Systems
+
+There are two separate query paths:
+
+1. **NBI queries** (`lib/query.ts`) -- Processes MongoDB-style JSON queries from
+   external API consumers. Expands parameter names, generates multi-type value
+   interpretations, and passes through to MongoDB directly.
+
+2. **UI/Expression queries** (`lib/db/synth.ts`) -- Compiles the internal
+   expression language into optimized MongoDB filters via Boolean minimization.
+   This is the more sophisticated path, used by the UI backend.
+
+### Frontend (`ui/`)
+
+A Mithril.js SPA with hash-based routing (`#!/`). Key modules:
+
+- `app.ts` -- Route definitions. The `pagify()` function wraps each page into a
+  `RouteResolver` that handles initialization, error boundaries, and data
+  fulfillment.
+- `layout.ts` -- Top-level layout (header, navigation, content, overlay).
+- `store.ts` -- Centralized data store. Implements a query-based reactive cache
+  with deduplication and incremental fetching. The `fulfill()` method (called
+  after every render) batches pending queries, computes filter diffs via
+  `unionDiff()`, and fetches only missing data. Connection monitoring polls
+  every 3 seconds for server health, clock skew, and config changes.
+- `components.ts` -- Component registry with a context propagation system. The
+  proxied `m()` function resolves string component names and the `m.context()`
+  API passes data (like the current device object) down the component tree
+  without explicit prop threading.
+- `smart-query.ts` -- Translates user-friendly `Label: value` searches into
+  filter expressions with type-aware matching (string, number, timestamp, MAC
+  address, tag).
+- `task-queue.ts` -- Two-stage task pipeline: staging (user configures tasks)
+  then queue (tasks are committed and executed via the backend).
+- `dynamic-loader.ts` -- Lazy loading of heavy libraries (CodeMirror, YAML) via
+  dynamic `import()`.
+
+### Build System (`build/`)
+
+The build is a self-bootstrapping esbuild pipeline (`npm run build` pipes
+`build/build.ts` through esbuild then node). It produces:
+
+- **Backend binaries** -- 5 entry points bundled for Node.js 12+ with shebang
+  banners, executable permissions, and `.js` extension stripped.
+- **Frontend bundle** -- `ui/app.ts` bundled for browsers (ESM, code-split) with
+  content-hashed filenames.
+- **CSS** -- `ui/css/app.css` bundled and minified with content-hashed output.
+- **SVG sprite** -- All icons in `ui/icons/` optimized via SVGO and combined
+  into a single sprite.
+
+`build/assets.ts` is a compile-time bridge: at rest it contains placeholder
+filenames; during build, the `assetsPlugin` replaces them with actual
+content-hashed names so both backend and frontend reference the correct assets.
+
+Build metadata includes the date and a hash derived from the git state, appended
+to the package version.
+
+## Cross-Cutting Concerns
+
+### Multi-Level Caching
+
+```
+memoize.ts         In-process function cache (2-4 min, two-generation rotation)
+     |
+local-cache.ts     In-process snapshot cache (5s refresh, hash-based revisions)
+     |
+cache.ts           MongoDB-backed distributed cache (configurable TTL)
+     |
+MongoDB            TTL index auto-expiration
+```
+
+Each service has its own `local-cache` that periodically checks the distributed
+cache for staleness. Distributed locks (`lock.ts`) coordinate rebuilds so only
+one worker does the expensive computation.
+
+### Configuration (`lib/config.ts`)
+
+Three-tier priority: CLI args > environment variables (`GENIEACS_*`) > config
+file (`config.json`) > defaults. Supports per-device overrides by appending
+`-OUI-ProductClass-SerialNumber` to option names.
+
+### Distributed Locking (`lib/lock.ts`)
+
+MongoDB-based mutual exclusion using upsert + duplicate key detection. TTL
+indexes prevent deadlocks from crashed processes. Clock skew tolerance of 30
+seconds.
+
+### Authentication
+
+- **CWMP devices** -- HTTP Basic or Digest auth, configurable per-device via
+  expressions (`auth.ts`).
+- **UI users** -- JWT tokens in cookies. Passwords hashed with PBKDF2-SHA512
+  (10000 iterations). Role-based authorization via `Authorizer` with
+  expression-based filters and validators.
+
+### Connection Requests (`lib/connection-request.ts`)
+
+Three methods to ask a CPE device to initiate a CWMP session:
+
+- **HTTP** -- Standard TR-069 connection request with Digest/Basic auth.
+- **UDP** -- For NAT traversal (STUN-based) with HMAC-SHA1 signed messages.
+- **XMPP** -- TR-069 Annex K via a full XMPP client (`xmpp-client.ts`).
+
+### Extensions (`lib/extensions.ts`)
+
+User-defined scripts executed in long-lived child processes (`genieacs-ext.ts`)
+communicating via IPC. Processes are lazily spawned per script name and reused.
+Each request gets a unique ID; responses are matched by ID with a configurable
+timeout.
+
+### Logging (`lib/logger.ts`)
+
+Dual-stream structured logging (application + access logs). Supports simple and
+JSON formats, systemd journal integration, and automatic log rotation detection.
+Protocol traces can be written to a debug file in YAML or JSON format
+(`debug.ts`).
+
+### Three-Valued Logic
+
+The system consistently implements SQL-style three-valued logic
+(true/false/null). This is visible in expression evaluation, the `Clause`
+hierarchy's separate true/false/null methods, and the 2-bit minterm encoding in
+the Boolean minimizer. NULL means "unknown" and propagates through operations
+following SQL semantics.
+
+## Architectural Invariants
+
+- **The expression system does not depend on any service-specific code.** The
+  `lib/common/expression/` modules are pure and shared across all services.
+
+- **The sandbox is deterministic across replays.** `Math.random()` is seeded
+  from the device ID, `Date.now()` is controlled, and extension results are
+  cached. A script re-run with the same inputs produces the same outputs.
+
+- **Services share no in-process state.** All cross-process coordination goes
+  through MongoDB (the `cache` and `locks` collections). Each worker process is
+  independent.
+
+- **Device data is never mutated in place during a session.** `VersionedMap`
+  provides revision-based snapshots. The sandbox writes to new revisions;
+  earlier revisions remain readable for re-evaluation.
+
+- **CWMP session exclusivity.** A distributed lock (`cwmp_session_<deviceId>`)
+  ensures only one session exists per device at a time. The lock is refreshed
+  periodically and released at session end.
+
+- **The UI backend never queries MongoDB with raw user input.** All queries go
+  through the expression-to-MongoDB compiler (`db/synth.ts`), which validates
+  and normalizes expressions before generating filters.

CHANGELOG.md

@@ -1,48 +1,286 @@
 # Change Log
 
+## 1.2.14 (2026-03-12)
+
+- Prevent UI crash when a malformed URL is sent to the server.
+
+- Fix potential edge-case bugs in expression evaluation and session
+  serialization.
+
+## 1.2.13 (2024-06-06)
+
+- Increase connection timeout for UI and NBI from 30 to 120 seconds to avoid
+  timeouts when running unindexed queries in large deployments.
+
+- Fix race condition causing 503 error when deleting multiple faults at once.
+
+- Fix some UI config options not being evaluated as dynamic expressions.
+
+- Fix an issue where certain edge-case query expressions were not being
+  correctly converted to MongoDB queries, resulting in inaccurate search
+  results.
+
+## 1.2.12 (2024-03-28)
+
+- Fix broken XMPP support in the previous release.
+
+- Fix regression causing CSV downloads to be buffered in memory before being
+  streamed to the client.
+
+## 1.2.11 (2024-03-21)
+
+- Resolved an issue from the previous release that caused incompatibility with
+  Node.js versions 12 through 15.
+
+## 1.2.10 (2024-03-18)
+
+- Add support for XMPP connection requests. Use the environment variables
+  `XMPP_JID` and `XMPP_PASSWORD` to configure the XMPP connection for the ACS.
+
+- The environment variables `CWMP_SSL_CERT` and `CWMP_SSL_KEY`, as well as their
+  counterparts for UI, NBI, and FS, now accept PEM-encoded certificates and keys
+  in addition to file paths.
+
+- The UI no longer requires users to refresh the page after modifying presets,
+  provisions, or virtual parameters. Refreshing is now only necessary for
+  changes to users, permissions, or UI configurations.
+
+- Improved conversion of GenieACS expressions into MongoDB queries for more
+  optimized queries and better index utilization.
+
+- Refactor UI pagination and sorting to fix issues from the previous approach,
+  especially with sorting by rapidly changing parameters such as 'last inform'
+  time.
+
+- The file server now supports HEAD requests, the Range header, and conditional
+  requests.
+
+- Addressed an issue causing file server disconnections for slow clients over
+  HTTPS.
+
+- Fix bug preventing users from changing their own passwords.
+
+- Introduced a new 'locks' collection for database locking, replacing the
+  previous use of the 'cache' collection for this purpose.
+
+- Various other minor fixes and enhancements.
+
+## 1.2.9 (2022-08-22)
+
+- New config option `cwmp.skipRootGpn` to enable a workaround for some
+  problematic CPEs that reject GPN requests on data model root.
+- Stream query results and CSV downloads as data becomes available instead of
+  buffering the entire response.
+- Log HTTP/HTTPS client errors in debug log.
+- Fix occasional lock expired errors after updating presets, etc.
+- Fix bug where queries containing `<>` operator may return incorrect results.
+- Fix a performance hit caused by DB calls containing the entire CPE data model
+  rather then just the updated parameters.
+- Fix bug where tags containing special characters are saved in their encoded
+  form when set via a Provision script.
+- Fix issue causing invalid `xsd:dateTime` values to be saved in DB as `NaN`
+  rather than maintain the original string value.
+- Fix bug using `$regex` operator with a numeric or a datetime string.
+- Fix ping not working on certain platforms.
+- Ping requests are now authenticated.
+- Fix error when the `FORWARDED_HEADER` environment variable contains IPv4 CIDR
+  while the listening interface is IPv6 and vice versa.
+- Fix false warning "unexpected parameter in response" showing in
+  `genieacs-cwmp` logs.
+- Other minor fixes and stability improvements.
+
+## 1.2.8 (2021-10-27)
+
+- Fix a remote code execution security vulnerability in genieacs-ui.
+- All UI components can now be configured using fully dynamic expressions.
+  Previously some component types only accept fixed string values as properties.
+- The `container` UI component can now be configured with an `element` property
+  that's either a string or a nested pair of `tag` and `attributes` properties.
+  The various attributes under the `attributes` property can now make use of a
+  new function `ENCODEURICOMPONENT()` to facilitate creating custom hyperlinks
+  in the UI.
+- Improve sorting buttons' behavior in the device listing page and other pages.
+  Sorting by multiple columns should now feel more intuitive.
+- Support the modulo (%) operator in expressions.
+- Fix a regression in the previous release where the config option 'ui.pageSize'
+  no longer works.
+- Fix process crash when a CPE sends an unsupported ACS method.
+
+## 1.2.7 (2021-09-18)
+
+- Fix regression causing frequent invalid session errors.
+- Fix regression breaking digest authentication in connection requests.
+
+## 1.2.6 (2021-09-16)
+
+- New config option `cwmp.downloadSuccessOnTimeout` to enable a workaround for
+  CPEs that neglect to send a TransferComplete message after a successful
+  download.
+- Display a progress bar when uploading new files.
+- Default to dual stack interface binding (i.e. `::` instead of `0.0.0.0`).
+  Unless the binding interface is explicitly set, this will cause IPv4 addresses
+  in the logs to be displayed as IPv4-mapped IPv6 addresses (e.g.
+  `::ffff:127.0.0.1` instead of `127.0.0.1`).
+- Detect and correct for client side (browser) clock skew that would otherwise
+  alter the numbers in the pie charts.
+- Various improvements relating to dealing with buggy TR-069 client
+  implementations.
+- Fix a bug causing "lock expired" exceptions when a CWMP session remains open
+  for a very long time due to slow clients.
+- Fix metadata of uploaded files going missing due to nginx stripping away what
+  it considers to be invalid headers. The nginx directive
+  `ignore_invalid_headers` is no longer required.
+- Fix crash when a CPE is assigned a tag containing a dot.
+- Fix bug preventing the user from closing the preset pop-up after being
+  presented with unsupported preset message.
+- Fix exceptions raised from ext scripts manifesting as timeout faults.
+- Fix download getting triggered repeatedly when the value passed to `Download`
+  parameter is greater than the current timestamp.
+- Fix crash when passing invalid attributes to `declare`.
+- Fix crash in NBI when pushing tasks to non-existing devices.
+- Fix crash in NBI when passing invalid JSON to various API endpoints.
+- Fix crash when the output from `ping` command cannot be parsed in some rare
+  edge cases.
+- A number of other fixes and stability improvements.
+
+## 1.2.5 (2021-03-12)
+
+- Support specifying custom types when uploading files.
+- Fix JS compatibility issue with Safari browser.
+
+## 1.2.4 (2021-02-24)
+
+- The data model state of a CPE is no longer forgotten after unsuccessful
+  session termination (e.g. timeout). This addresses a number of undesired side
+  effects that arise when a CPE does not terminate the session properly.
+- Executing tasks that take a long time to complete (e.g. refreshing the entire
+  data model) no longer shows a timeout error while the task is still being
+  processed.
+- New function `ROUND()` available to expressions. It works similar to the
+  function by the same name in SQLite and PostgreSQL.
+- Log access events for `genieacs-nbi` service.
+- Pipe stdout/stderr from extension scripts to the `genieacs-cwmp` process log.
+- Parameter values of type `xsd:dateTime` are now displayed in the UI and CSV
+  downloads as a date string rather than a numeric value.
+- Add file download link in the files listing page.
+- Display spinner loading animation throughout the UI.
+- Display GenieACS version and build number underneath the logo.
+- New option to specify how many parameters are displayed at a time in the
+  all-parameters component. Simply set `limit` property in the component config.
+- Reduce overly strong Brotli compression level which was causing significant
+  page load slowdown when Brotli is used.
+- Retire dump-data-model tool. `genieacs-sim` can now use a CSV file as its data
+  model.
+- Reduce the number of concurrent database connections from each process.
+- Remove dependency on 'hostInfo' MongoDB command which is a privileged action.
+  It is now possible to use shared MongoDB instances with limited privileged.
+- Fix bug in NBI where querying files returns 404 error.
+- Fix ping not working for devices with an IPv6 address.
+- Fix an elusive memory leak in `genieacs-fs` that slowly eats up memory and can
+  go unnoticed for long periods of time.
+- Fix a rare edge case where a `declare()` call to set a parameter value may not
+  work as intended if the parameter was originally received as part of the
+  Inform message.
+- A number of other fixes and stability improvements.
+
+## 1.2.3 (2020-10-26)
+
+- New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
+  report writable parameters as non-writable. When set to true, the scripts will
+  no longer respect the 'writable' attribute of the CPE parameters and will send
+  a SetParamteerValues, AddObject, or DeleteObject request anyway.
+- Tags no longer restrict what characters are allowed. Any character other than
+  alphanumeric characters, hyphen, or underscore is now encoded in the data
+  model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
+- Ask for a confirmation before closing a pop-up dialog with unsaved changes.
+- Better XML validation to avoid crashes caused by invalid CPE requests.
+- Fix confusing 404 error message when the user attempts to modify a resource
+  when they don't have the necessary permissions.
+- Fix a rare issue where genieacs-cwmp stops accepting new connections after
+  running for a few weeks.
+- Fix exception when IS NULL operator is used in certain situations.
+
+## 1.2.2 (2020-10-03)
+
+- Added button to push files to selected devices from device listing page.
+- A few minor UI improvements.
+- Fix exception that can happen and persist after a Download request.
+- Fix validation bug preventing running refreshObject task on data model root.
+- Fix invalid arguments fault in refresh preset configuration when upgrading
+  from v1.1.
+
+## 1.2.1 (2020-09-08)
+
+- Fix bug causing faults to not be displayed in the UI.
+- Fix bug where deleting objects does not get reflected immediately in the UI.
+- Improve conversion between filters written in the expression format and
+  MongoDB queries. There should now be fewer edge cases where the two are not
+  equisatisfiable.
+
+## 1.2.0 (2020-09-01)
+
+- Support GetParameterAttributes and SetParameterAttributes TR-069 methods.
+- Support CASE statement and COALESCE function in expressions.
+- Provision arguments can now be a list of expressions that are dynamically
+  evaluated.
+- Support Forwarded HTTP header to display in the logs the correct IP of CPEs
+  behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
+- Config expressions can now access all available device parameters, not only
+  serial number, product class, and OUI.
+- Use relative URLs throughout the UI to allow serving from a subdirectory using
+  a reverse proxy.
+- Make Date.parse() and Date.UTC() available to provision scripts.
+- libxmljs has been entirely removed in favor of our bespoke XML parser.
+- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now used
+  to determine the TCP connection timeout.
+- The all-parameters component now limits the number of parameters displayed for
+  better performance.
+- The process genieacs-cwmp is now much less likely to throw exceptions as a
+  result of invalid requests from CPE.
+- A large number of bug fixes and stability improvements.
+
 ## 1.2.0-beta.0 (2019-07-30)
 
 - A brand new UI superseding genieacs-gui.
 - New initialization wizard on first run.
-- New expression/query language used in search filters and preset
-preconditions.
+- New expression/query language used in search filters and preset preconditions.
 - CPE -> ACS authentication is now supported.
 - New config option (CWMP_KEEP_ALIVE_TIMEOUT) to specify how long to wait for a
-reply from the CPE before closing the TCP connection.
+  reply from the CPE before closing the TCP connection.
 - Debug logging has been reimplemented utilizing YAML format for logs.
 - Handle 9005 faults (Invalid Parameter Name) gracefully by attempting to
-rediscover the path of the missing parameter recursively.
+  rediscover the path of the missing parameter recursively.
 - declare() statements not followed by an explicit commit() are now deferred
-until all currently active scripts have been executed.
+  until all currently active scripts have been executed.
 - FS_HOSTNAME now defaults to the server's hostname or IP.
 - The API now validates the structure of task objects before saving.
 - New XML parser implementation for better performance. You can revert to the
-old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
-v10.
+  old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
+  v10.
 - Performance optimizations. While performance has improved for the majority of
-use cases, there may be situations where performance has degraded. It's
-recommended to revisit your hardware requirements.
+  use cases, there may be situations where performance has degraded. It's
+  recommended to revisit your hardware requirements.
 - Connection request authentication no longer uses 'auth.js' file. Instead, the
-connection request authentication behavior can now be customized using an
-'expression'.
+  connection request authentication behavior can now be customized using an
+  'expression'.
 - The config file (config.json) has been deprecated. System configuration (e.g.
-listen ports, worker count) are now recommended to be passed as environment.
-variables. Other general configuration options are stored in the database so as
-to not require service restart for changes to take effect.
+  listen ports, worker count) are now recommended to be passed as environment.
+  variables. Other general configuration options are stored in the database so
+  as to not require service restart for changes to take effect.
 - Optional redis dependency has been removed completely.
 - Tags now allow only alphanumeric characters and underscore.
-- Supported versions of NodeJs and MongoDB are 10.x and up and 2.6 and up
-respectively.
+- Supported versions of Node.js and MongoDB are 10.x and up and 2.6 and up
+  respectively.
 
 ## 1.1.3 (2018-10-23)
 
 - New config option (MAX_COMMIT_ITERATIONS) to avoid max commit iterations
-faults for more complex scripts.
+  faults for more complex scripts.
 - Support base64 and hexBinary parameter types.
 - Strict parsing of number values in queries (e.g. "123abc" no longer accepted
-as 123).
+  as 123).
 - Mixing $ne and $not operators is not allowed. Now it throws an error instead
-of returning incorrect results.
+  of returning incorrect results.
 - When a task expires, any associated fault is also deleted.
 - API now accepts 'timeout' argument when posting a task.
 - A number of stability fixes.
@@ -52,66 +290,68 @@ of returning incorrect results.
 - A large number of bug fixes as well as stability and performance improvements.
 - Three security vulnerabilities disclosed by Maximilian Hils have been patched.
 - New config option UDP_CONNECTION_REQUEST_PORT to specify binding port for UDP
-connection requests.
+  connection requests.
 - New config option DATETIME_MILLISECONDS to strip milliseconds from dateTime
-values.
+  values.
 - New config option BOOLEAN_LITERAL to use 1/0 or true/false for boolean values.
 - Parameter values that cannot be parsed according to the reported type now show
-a warning message.
+  a warning message.
 - Virtual parameter scripts now use the variable 'args' instead of the special
-'TIMESTAMPS' and 'VALUES' variables. The content of the args array is: {declare
-timestamps}, {declare values}, {current timestamps}, {current values}.
+  'TIMESTAMPS' and 'VALUES' variables. The content of the args array is:
+  {declare timestamps}, {declare values}, {current timestamps}, {current
+  values}.
 - Virtual parameter value types are now inferred from the JavaScript type if the
-returned value attribute is not a value-type pair.
+  returned value attribute is not a value-type pair.
 - Show a fault when a virtual parameter script doesn't return the required
-attributes.
+  attributes.
 - Redis is now optional (and disabled by default), reducing the complexity of
-scalable deployments.
+  scalable deployments.
 - Better detection of cyclical presets resulting in fewer faults for complex
-provisioning scripts.
+  provisioning scripts.
 - Math.random() is now deterministic on per-device basis. A function has been
-added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
+  added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
 - Overload spikes are now handled gracefully by refusing to accept new sessions
-temporarily when under abnormal load.
+  temporarily when under abnormal load.
 - Added log messages for session timeouts, connection drops, and XML parsing
-errors.
+  errors.
 - Date.now() now takes an optional argument to specify "time steps" (in
-milliseconds). This can be used to ensure a group of parameters are all
-refreshed at the same time intervals.
+  milliseconds). This can be used to ensure a group of parameters are all
+  refreshed at the same time intervals.
 - Only the non-default configuration options are now logged at process start.
 - Faults caused by errors from extensions now show a cleaner stack trace.
 - Exit main process if there are too many worker crashes (e.g. when DB is down).
 - Updated dependencies and included a lockfile to ensure installations get the
-exact dependencies it was tested against.
+  exact dependencies it was tested against.
 
 ## 1.1.1 (2017-03-23)
 
 - Avoid crashing when connection request credentials are missing.
 - Show a warning instead of crashing when failing to parse parameter values
-according to the expected value type.
+  according to the expected value type.
 - Add missing "Registered" event.
 - Fix bug where in certain cases many more instances than declared are created.
 - Fix parameter discovery bug when declared path timestamp is 1 or is not set.
 - Fix preset precondition failing when testing against datetime parameters and
-certain other parameters like \_deviceId.\_ProductClass.
+  certain other parameters like \_deviceId.\_ProductClass.
 
 ## 1.1.0 (2017-03-10)
 
 - Provisions enable implementing dynamic device configuration or complex device
-provisioning work flow using arbitrary scripts.
+  provisioning work flow using arbitrary scripts.
 - Virtual parameters are user-defined parameters whose values are evaluated from
-a custom script.
+  a custom script.
 - Extensions are sandboxed Node.js scripts that are accessible from provision
-and virtual parameter scripts to facilitate integration with external entities.
-- Support for UDP/STUN based connection requests for reaching devices behind
-NAT (TR-069 Annex G).
+  and virtual parameter scripts to facilitate integration with external
+  entities.
+- Support for UDP/STUN based connection requests for reaching devices behind NAT
+  (TR-069 Annex G).
 - Presets can now be scheduled using a cron-like expression.
 - Presets can now be tied to specific device events (e.g. boot).
-- Presets precondition queries no longer support "$or" or other MongoDB logical
-operators.
+- Presets precondition queries no longer support "\$or" or other MongoDB logical
+  operators.
 - Faults are no longer a part of tasks but are now first class objects.
 - Presets are now assigned to channels. A fault in one channel only blocks
-presets in that channel.
+  presets in that channel.
 - New API CRUD functions for provisions, virtual parameters, and faults.
 - New config options for XML output.
 - API responses now include "GenieACS-Version" header.
@@ -120,19 +360,19 @@ presets in that channel.
 - Supported Node.js versions are 6.x and 7.x.
 - Supported MongoDB versions are 2.6 through 3.4.
 - Expect performance differences due to major under the hood changes. Some
-operations are faster and some are slower. Overall performance is improved.
+  operations are faster and some are slower. Overall performance is improved.
 - GenieACS will no longer fetch the entire device data model upon first contact
-but will instead only fetch the parameters it needs to fulfill the presets.
+  but will instead only fetch the parameters it needs to fulfill the presets.
 - Logs have been overhauled and split into two streams: process log (stderr) and
-access log (stdout). Also added config options to dump logs to files rather than
-standard streams.
+  access log (stdout). Also added config options to dump logs to files rather
+  than standard streams.
 - Connection request authentication credentials are picked up from the device
-data model if available. config/auth.js is still supported as a fallback and now
-supports an optional callback argument.
+  data model if available. config/auth.js is still supported as a fallback and
+  now supports an optional callback argument.
 - Custom commands have been removed. Use virtual parameters and/or extensions.
 - Aliases and value normalizers (config/parameters.json) have been removed. Use
-virtual parameters.
+  virtual parameters.
 - The API /devices/<device_id>/preset has been removed.
 - Rarely used RequestDownload method no longer supported.
 - The TR-069 client simulator has moved to its own repo at
-https://github.com/zaidka/genieacs-sim
+  https://github.com/zaidka/genieacs-sim

CONTRIBUTING.md

@@ -1,39 +1,352 @@
-# Have a question?
+# Contributing to GenieACS
 
-Please ask in the [Users mailing list][mailing-lists] instead of opening a
-Github Issue. The Github issue tracker is strictly for bug reports.
+## Questions and Support
 
-# Issues
+Please use the [forum](https://forum.genieacs.com) for questions, help requests,
+and general discussion. GitHub Issues are reserved for confirmed bug reports.
 
-We use GitHub issues to track bugs. Please ensure your bug description is
-clear and has sufficient instructions to be able to reproduce the issue.
+## Issues
 
-If you face interoperability issues with your device, more often than not it's
-a device related issue. Please consult the [Users mailing list][mailing-lists]
-before submitting an Issue.
+We use GitHub Issues to track bugs. When filing a bug report:
 
-# Pull Requests
+- Provide a clear description of the problem.
+- Include steps to reproduce the issue.
+- Note the GenieACS version, Node.js version, and MongoDB version.
+- Include relevant log output if applicable.
 
-Pull requests are welcome. If you have a fix for a bug, go ahead and create a
-pull request. For other kinds of enhancements, please discuss in the [Users
-mailing list][mailing-lists] first.
+If you face interoperability issues with a CPE, it is more often than not a
+device-specific issue. Please consult the [forum](https://forum.genieacs.com)
+before opening an Issue.
 
-Please follow the guidelines [here][commit-guidelines] when committing your
-changes.
+## Pull Requests
+
+Pull requests are welcome. For bug fixes, go ahead and open a PR directly. For
+new features or significant changes, please discuss in the
+[forum](https://forum.genieacs.com) first to ensure alignment with the project's
+direction.
+
+When submitting a PR:
+
+- Keep changes focused. One PR should address one concern.
+- Run `npm run lint` and `npm test` before submitting.
+- Update documentation in `docs/` if your change affects user-facing behavior.
+- Add tests where appropriate (see [Testing](#testing)).
+- Write a clear PR description explaining what the change does and why.
+
+## Code Style
+
+### Naming Conventions
+
+- **Variables and functions**: `camelCase`
+- **Classes**: `PascalCase`
+- **Interfaces and type aliases**: `PascalCase`, without an `I` or `T` prefix
+- **Module-level constants**: `UPPER_SNAKE_CASE`
+- **Private class members**: prefixed with underscore (`_name`, `_cache`)
+
+Choose descriptive, meaningful names. A longer clear name is better than a short
+ambiguous one.
+
+### Imports
+
+Organize imports in three groups, in this order:
+
+1. Node.js built-in modules, using the `node:` prefix
+2. External packages
+3. Local project modules
+
+Use `.ts` extensions for all local imports.
+
+```typescript
+import { readFileSync } from "node:fs";
+import * as http from "node:http";
+
+import { Collection } from "mongodb";
+
+import { connect, disconnect } from "./db.ts";
+import Path from "./common/path.ts";
+```
+
+### Functions and Exports
+
+Use the `function` keyword for all named and exported function declarations.
+Arrow functions should only be used for callbacks and short inline expressions.
+
+```typescript
+// Named/exported functions use function keyword
+export function processRequest(req: Request): Response {
+  // ...
+}
+
+// Arrow functions for callbacks
+items.filter((item) => item.active);
+promise.then((result) => {
+  // ...
+});
+```
+
+### TypeScript
+
+- All function declarations must have explicit return types. This is enforced by
+  ESLint.
+- Using `any` is permitted where full typing would be impractical, but prefer
+  more specific types when reasonable.
+- Prefer `Record<string, T>` over `{ [key: string]: T }` for mapped types.
+- Use type assertions (`as`) sparingly and only when you have stronger type
+  knowledge than the compiler.
+
+### Comments
+
+Code should be self-documenting. Use comments sparingly and only when the "why"
+is not obvious from the code itself. When you do comment, use inline `//` style.
+
+Do not use JSDoc (`/** */`) or block comments (`/* */`).
+
+```typescript
+// Good: explains a non-obvious reason
+// Escapes everything except alphanumerics and underscore
+function escapeRegExp(str: string): string {
+  /* ... */
+}
+
+// Good: links to an external reference
+// Source: http://stackoverflow.com/a/6969486
+
+// Good: flags a known limitation
+// TODO support "MD5-sess" algorithm directive
+
+// Bad: restates what the code does
+// Loop through each item and increment the counter
+for (const item of items) {
+  counter++;
+}
+```
+
+### Error Handling
+
+Use `== null` (not `=== null || === undefined`) for null/undefined checks.
+ESLint is configured to allow this.
+
+## Testing
+
+### When to Write Tests
+
+Tests are most valuable for pure logic: parsers, data transformations,
+algorithms, and utility functions where inputs and outputs are well defined. A
+regression test accompanying a bug fix is also worthwhile to prevent the same
+issue from resurfacing.
+
+Not every change needs a test. Use judgment and consider the likelihood and cost
+of breakage. Avoid writing tests that duplicate coverage already provided by
+existing tests, and resist the urge to test trivial code just for the sake of
+coverage numbers.
+
+### Conventions
+
+Tests use the Node.js built-in test runner (`node:test`) and assertion module
+(`node:assert`). No external test libraries or mocking frameworks are used.
+
+```typescript
+import test from "node:test";
+import assert from "node:assert";
+
+void test("parseValue returns correct type for integer strings", () => {
+  const result = parseValue("42");
+  assert.strictEqual(result, 42);
+});
+
+void test("parseValue throws on invalid input", () => {
+  assert.throws(() => parseValue(""), new Error("empty value"));
+});
+```
+
+Key conventions:
+
+- Prefix `test()` calls with `void` to satisfy the `no-floating-promises` lint
+  rule.
+- Keep tests flat. Do not nest `describe` blocks.
+- Use descriptive test names that state what is being tested and the expected
+  outcome.
+- Use `assert.strictEqual()` for value comparisons and
+  `assert.deepStrictEqual()` for objects and arrays.
+
+Test files live in the `test/` directory and are named after the module they
+test (e.g. `test/path.ts` tests `lib/common/path.ts`).
+
+## Dependencies
+
+This project deliberately keeps its dependency footprint small. Before adding a
+new dependency:
+
+- Prefer Node.js built-in APIs when they can do the job.
+- Consider whether the functionality is simple enough to implement directly.
+- Justify the addition in your PR description.
+
+Do not add development tool dependencies (linter plugins, editor integrations,
+etc.) without prior discussion.
+
+## File Organization
+
+| Directory     | Contents                                     |
+| ------------- | -------------------------------------------- |
+| `lib/`        | Core server-side application code            |
+| `lib/common/` | Code shared between server and browser       |
+| `lib/db/`     | Database layer (MongoDB)                     |
+| `lib/ui/`     | UI backend helpers                           |
+| `ui/`         | Frontend code (Mithril.js SPA)               |
+| `bin/`        | Service entry points                         |
+| `build/`      | Build tooling                                |
+| `test/`       | Test files                                   |
+| `docs/`       | User documentation (Sphinx/reStructuredText) |
+| `public/`     | Static assets (favicon, logo)                |
+
+Place new code in the directory that matches its purpose. Server-side logic
+belongs in `lib/`, code that must run in both Node.js and the browser belongs in
+`lib/common/`, and frontend-only code belongs in `ui/`.
+
+## Documentation
+
+User documentation lives in the `docs/` directory as reStructuredText files
+built with Sphinx and published to
+[docs.genieacs.com](https://docs.genieacs.com).
+
+When your change affects user-facing behavior:
+
+- Update the relevant documentation in `docs/`.
+- If adding a new feature, consider whether it warrants a new page or a section
+  in an existing page.
+- Keep documentation concise and practical. Match the existing tone: direct,
+  factual, no filler.
+
+Documentation changes should be included in the same PR as the code change, not
+submitted separately.
+
+### ARCHITECTURE.md
+
+`ARCHITECTURE.md` describes the high-level architecture of the project: the
+service boundaries, major subsystems, data flow, and key invariants. It is aimed
+at contributors who need a mental map of the codebase.
+
+This file has a different update cadence than `docs/`. It should be revisited a
+few times a year rather than kept in lockstep with every code change. When you
+do update it, follow these principles:
+
+- **Only describe things that are unlikely to change frequently.** Module
+  responsibilities, service boundaries, key data structures, and architectural
+  invariants belong here. Implementation details, function signatures, and
+  config option lists do not.
+- **Name important files, modules, and types but do not link them.** Links go
+  stale. Encourage the reader to use symbol search to find named entities; this
+  also helps them discover related, similarly named things.
+- **Keep it short.** Every recurring contributor will read it. A shorter
+  document is less likely to become stale and more likely to be maintained.
+- **Describe the "what" and "where", not the "how".** This is a map of the
+  country, not an atlas of its states. Pull detailed explanations into inline
+  code comments or separate documents.
+- **Call out architectural invariants explicitly.** Important invariants are
+  often expressed as the _absence_ of something (e.g., "the expression system
+  does not depend on service-specific code") and are hard to discover by reading
+  code alone.
+
+## Git Workflow
+
+### Commit Messages
+
+This project follows the
+[Conventional Commits](https://www.conventionalcommits.org/) format:
+
+    <type>(<scope>): <subject>
+
+    [optional body]
+
+- Use the imperative mood, present tense: "Fix bug", not "Fixed bug" or "Fixes
+  bug".
+- Do not capitalize the first letter of the subject (the type prefix handles
+  visual structure).
+- Do not end the subject line with a period.
+- Keep the subject line under 72 characters.
+- When more context is helpful, add a body separated from the subject by a blank
+  line, wrapped at 72 characters. The goal is to provide enough information for
+  someone scanning the commit history to find a specific change (e.g. for
+  troubleshooting or rebasing) or to draft changelog entries for a release.
+  Don't be verbose, but don't be cryptic either.
+
+#### Types
+
+| Type       | When to use                                                                 |
+| ---------- | --------------------------------------------------------------------------- |
+| `fix`      | Bug fixes                                                                   |
+| `feat`     | New features or capabilities                                                |
+| `refactor` | Code restructuring with no behavior change                                  |
+| `test`     | Adding or updating tests                                                    |
+| `docs`     | Documentation changes (`docs/`, `CONTRIBUTING.md`, `README.md`)             |
+| `build`    | Build system, dependencies, esbuild config, `package.json` scripts          |
+| `chore`    | Maintenance tasks that don't fit above (`.gitignore`, tooling config, etc.) |
+
+#### Scopes
+
+Scope is optional. Use it when it adds useful context; omit it when the change
+is cross-cutting or the subject already makes it obvious.
+
+| Scope  | Covers                              |
+| ------ | ----------------------------------- |
+| `cwmp` | CWMP service (including extensions) |
+| `nbi`  | Northbound REST API service         |
+| `fs`   | File service                        |
+| `ui`   | Web UI (frontend and backend)       |
+| `db`   | Database layer                      |
+
+If a change touches multiple scopes, either pick the primary one or omit the
+scope entirely.
+
+#### Examples
+
+    feat(nbi): add bulk device delete endpoint
+    fix(cwmp): handle missing ParameterKey in InformResponse
+    refactor(db): replace raw queries with parameterized calls
+    fix(ui): correct parameter table sort order
+    test: add XML parser edge case coverage
+    docs: update provisioning guide for new API
+    build: upgrade esbuild to v0.20
+
+    fix(cwmp): increase server timeout to 2 mins
+
+    To allow enough time for running unindexed queries in large
+    deployments.
+
+### Branches
+
+- `master` is the main development branch.
+- Create a feature or fix branch for your work and open a PR against `master`.
+- Use concise, descriptive branch names in lowercase with hyphens:
+  `fix-race-condition`, `support-xmpp-requests`.
+
+## Changelog
+
+The changelog (`CHANGELOG.md`) is maintained for each release and is written for
+users and system administrators, not developers.
+
+- Write entries as clear, user-facing prose. Do not copy commit messages
+  verbatim.
+- Each entry should describe what changed and, when helpful, why it matters.
+- Group entries under a version heading with the release date:
+  `## 1.2.13 (2024-06-06)`.
+- Start each entry with a verb: "Fix", "Add", "Improve", "Remove", etc.
+- Include enough context that a user can understand the impact without reading
+  the code.
+
+You do not need to update the changelog in your PR. The maintainer will add
+changelog entries during the release process.
 
 ## Contributor License Agreement
 
 By submitting a pull request to this repository, you acknowledge that, while
-maintaining copyright, you grant Zaid Abdulla a perpetual, worldwide, non-
-exclusive, no-charge, royalty-free, irrevocable license to reproduce, prepare
-derivative works of, publicly display, publicly perform, sublicense, and
-distribute your contributions and such derivative works under the AGPLv3
-license or any other license terms, including, but not limited to, proprietary
-or commercial license terms.
+maintaining copyright, you grant GenieACS Inc. a perpetual, worldwide,
+non-exclusive, no-charge, royalty-free, irrevocable license to reproduce,
+prepare derivative works of, publicly display, publicly perform, sublicense, and
+distribute your contributions and such derivative works under the AGPLv3 license
+or any other license terms, including, but not limited to, proprietary or
+commercial license terms.
 
-You confirm that you own or have rights to distribute and sublicense the
-source code contained therein, and that your content does not infringe upon
-the intellectual property rights of a third party.
-
-[commit-guidelines]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
-[mailing-lists]: http://lists.genieacs.com
+You confirm that you own or have rights to distribute and sublicense the source
+code contained therein, and that your content does not infringe upon the
+intellectual property rights of a third party.

Dockerfile

@@ -0,0 +1,68 @@
+# =========================================================
+# GenieACS — UniFi Theme Fork
+# Multi-stage build:
+#   Stage 1 (builder): instala devDeps + compila o projeto
+#   Stage 2 (runtime): imagem mínima só com dist/ + prodDeps
+# =========================================================
+
+# ── Stage 1: builder ─────────────────────────────────────
+FROM node:20-alpine AS builder
+
+# Dependências de sistema necessárias para o build
+# (git é exigido pelo build.ts para calcular o metadata de versão)
+RUN apk add --no-cache git python3 make g++
+
+WORKDIR /build
+
+# Copiar manifests primeiro para cache de camadas do npm install
+COPY package.json npm-shrinkwrap.json ./
+
+# Instalar TODAS as dependências (incluindo devDependencies)
+RUN npm ci --fund=false --audit=false
+
+# Copiar o restante do fonte
+COPY . .
+
+# Compilar o projeto (gera dist/)
+RUN npm run build
+
+# ── Stage 2: runtime ─────────────────────────────────────
+FROM node:20-alpine AS runtime
+
+# Timezone e utilitários mínimos
+RUN apk add --no-cache tzdata dumb-init
+
+WORKDIR /opt/genieacs
+
+# Copiar artefatos compilados
+COPY --from=builder /build/dist ./
+
+# Instalar apenas dependências de produção a partir do dist/
+RUN npm ci --omit=dev --fund=false --audit=false
+
+# Criar diretórios de configuração e extensões
+RUN mkdir -p \
+      /opt/genieacs/config/ext \
+      /opt/genieacs/logs \
+    && addgroup -S genieacs \
+    && adduser  -S -G genieacs genieacs \
+    && chown -R genieacs:genieacs /opt/genieacs
+
+USER genieacs
+
+# Variáveis de ambiente padrão (sobrescreva no compose ou .env)
+ENV GENIEACS_MONGODB_CONNECTION_URL=mongodb://mongo:27017/genieacs \
+    GENIEACS_EXT_DIR=/opt/genieacs/config/ext \
+    GENIEACS_LOG_FORMAT=json \
+    GENIEACS_ACCESS_LOG_FORMAT=json \
+    GENIEACS_UI_JWT_SECRET=change-me-in-production \
+    GENIEACS_CWMP_WORKER_PROCESSES=1 \
+    GENIEACS_NBI_WORKER_PROCESSES=1 \
+    GENIEACS_FS_WORKER_PROCESSES=1 \
+    GENIEACS_UI_WORKER_PROCESSES=1
+
+# Expor todas as portas dos serviços
+EXPOSE 3000 7547 7557 7567
+
+# dumb-init garante propagação correta de sinais
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]

README.md

@@ -1,5 +1,9 @@
 # GenieACS
 
+**This is the development branch for GenieACS v1.3. It is unstable and not ready
+for production use. For the latest stable release, see the
+[v1.2 branch](https://github.com/genieacs/genieacs/tree/v1.2).**
+
 GenieACS is a high performance Auto Configuration Server (ACS) for remote
 management of TR-069 enabled devices. It utilizes a declarative and fault
 tolerant configuration engine for automating complex provisioning scenarios at
@@ -8,35 +12,30 @@ millions of concurrent devices.
 
 ## Quick Start
 
-*Important: This is a pre-release branch. Use
-[v1.1](https://github.com/genieacs/genieacs/tree/v1.1) for production
-deployments.*
-
 Install [Node.js](http://nodejs.org/) and [MongoDB](http://www.mongodb.org/).
 Refer to their corresponding documentation for installation instructions. The
 supported versions are:
 
-- Node.js: 10.x through 12.x
-- MongoDB: 2.6 through 4.1
+- Node.js: 12.3+
+- MongoDB: 3.6+
 
-Clone this repo or download the source archive then *cd* into the source
-directory and install the required dependencies:
+Install GenieACS from NPM:
+
+    sudo npm install -g genieacs
+
+To build from source instead, clone this repo or download the source archive
+then _cd_ into the source directory then run:
 
     npm install
-
-Now build:
-
     npm run build
 
-Finally, run the following services from the build that's generated under
-'dist' directory:
+Finally, run the following services (found under `./dist/bin/` if building from
+source):
 
 ### genieacs-cwmp
 
-This is the service that the CPEs will communicate with. It listens on port
-7547 by default. Configure the ACS URL in your devices accordingly.
-
-    ./dist/bin/genieacs-cwmp
+This is the service that the CPEs will communicate with. It listens on port 7547
+by default. Configure the ACS URL in your devices accordingly.
 
 You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
 as a dummy TR-069 simulator if you don't have a CPE at hand.
@@ -47,40 +46,35 @@ This is the northbound interface module. It exposes a REST API on port 7557 by
 default. This one is only required if you have an external system integrating
 with GenieACS using this API.
 
-    ./dist/bin/genieacs-nbi
-
 ### genieacs-fs
 
 This is the file server from which the CPEs will download firmware images and
-such.
-
-    ./dist/bin/genieacs-fs
+such. It listens on port 7567 by default.
 
 ### genieacs-ui
 
 This serves the web based user interface. It listens on port 3000 by default.
+You must pass _--ui-jwt-secret_ argument to supply the secret key used for
+signing browser cookies:
 
-    ./dist/bin/genieacs-ui --ui-jwt-secret secret
-
-The argument *--ui-jwt-secret* supplies the key used for signing browser
-cookies.
+    genieacs-ui --ui-jwt-secret secret
 
 The UI has plenty of configuration options. When you open GenieACS's UI in a
 browser you'll be greeted with a database initialization wizard to help you
 populate some initial configuration.
 
+Visit [docs.genieacs.com](https://docs.genieacs.com) for more documentation and
+a complete installation guide for production deployments.
+
 ## Support
 
-The [forum](https://forum.genieacs.com) is a good place to get guidance and
-help from the community. Head on over and join the conversation! In addition,
-the [wiki](https://github.com/genieacs/genieacs/wiki) provides useful
-documentation and tips from GenieACS users.
+The [forum](https://forum.genieacs.com) is a good place to get guidance and help
+from the community. Head on over and join the conversation!
 
-For commercial support options and professional services, please visit
+For commercial support options, please visit
 [genieacs.com](https://genieacs.com/support/).
 
 ## License
 
-Copyright 2013-2019 GenieACS Inc. GenieACS is released under the [AGPLv3
-license
-terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).
+Copyright 2013-2026 GenieACS Inc. GenieACS is released under the
+[AGPLv3 license terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

ava.config.js

@@ -1,6 +0,0 @@
-export default {
-  "compileEnhancements": false,
-  "extensions": ["ts"],
-  "require": ["esm", "ts-node/register/transpile-only"]
-};
-

bin/genieacs-cwmp

@@ -1,129 +0,0 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import * as cwmp from "../lib/cwmp";
-import * as db from "../lib/db";
-import * as extensions from "../lib/extensions";
-import * as cache from "../lib/cache";
-import { version as VERSION } from "../package.json";
-
-logger.init("cwmp", VERSION);
-
-const SERVICE_ADDRESS = config.get("CWMP_INTERFACE");
-const SERVICE_PORT = config.get("CWMP_PORT");
-
-function exitWorkerGracefully(): void {
-  db.disconnect();
-  cache.disconnect();
-  extensions.killAll();
-  cluster.worker.disconnect();
-}
-
-function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
-    process.exit(1);
-  });
-}
-
-if (!cluster.worker) {
-  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES");
-
-  logger.info({
-    message: `genieacs-cwmp starting`,
-    pid: process.pid,
-    version: VERSION
-  });
-
-  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
-
-  process.on("SIGINT", () => {
-    logger.info({
-      message: "Received signal SIGINT, exiting",
-      pid: process.pid
-    });
-
-    cluster.stop();
-  });
-
-  process.on("SIGTERM", () => {
-    logger.info({
-      message: "Received signal SIGTERM, exiting",
-      pid: process.pid
-    });
-
-    cluster.stop();
-  });
-} else {
-  const ssl = {
-    key: config.get("CWMP_SSL_KEY"),
-    cert: config.get("CWMP_SSL_CERT")
-  };
-
-  const keepAliveTimeout = config.get("CWMP_KEEP_ALIVE_TIMEOUT") as number;
-
-  process.on("uncaughtException", err => {
-    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
-    logger.error({
-      message: "Uncaught exception",
-      exception: err,
-      pid: process.pid
-    });
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGINT", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
-    .then(() => {
-      server.start(
-        SERVICE_PORT,
-        SERVICE_ADDRESS,
-        ssl,
-        cwmp.listener,
-        cwmp.onConnection,
-        keepAliveTimeout
-      );
-    })
-    .catch(err => {
-      setTimeout(() => {
-        throw err;
-      });
-    });
-}

bin/genieacs-cwmp.ts

@@ -0,0 +1,118 @@
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import * as cwmp from "../lib/cwmp.ts";
+import * as db from "../lib/db/db.ts";
+import * as extensions from "../lib/extensions.ts";
+import { version as VERSION } from "../package.json";
+
+logger.init("cwmp", VERSION);
+
+const SERVICE_ADDRESS = config.get("CWMP_INTERFACE") as string;
+const SERVICE_PORT = config.get("CWMP_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  void extensions.killAll().finally(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-cwmp starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("CWMP_SSL_KEY") as string;
+  const cert = config.get("CWMP_SSL_CERT") as string;
+
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    onConnection: cwmp.onConnection,
+    onClientError: cwmp.onClientError,
+    timeout: 30000,
+    keepAliveTimeout: 0,
+    requestTimeout: cwmp.REQUEST_TIMEOUT,
+  };
+
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    server.stop(false).then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, cwmp.listener);
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    void initPromise.finally(() => {
+      server
+        .stop(false)
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    void initPromise.finally(() => {
+      server
+        .stop(false)
+        .then(exitWorkerGracefully)
+        .catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-ext.ts

@@ -1,25 +1,4 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { Fault } from "../lib/types";
+import { Fault } from "../lib/types.ts";
 
 const jobs = new Set();
 const fileName = process.argv[2];
@@ -35,20 +14,20 @@ function errorToFault(err: Error): Fault {
     message: err.message,
     detail: {
       name: err.name,
-      message: err.message
-    }
+      message: err.message,
+    },
   };
 
   if (err.stack) {
     fault.detail["stack"] = err.stack;
     // Trim the stack trace
     const stackTrimIndex = fault.detail["stack"].match(
-      /\s+at\s[^\s]+\s\(.*genieacs-ext:.+\)/
+      /\s+at\s[^\s]+\s\(.*genieacs-ext:.+\)/,
     );
     if (stackTrimIndex) {
       fault.detail["stack"] = fault.detail["stack"].slice(
         0,
-        stackTrimIndex.index
+        stackTrimIndex.index,
       );
     }
   }
@@ -56,21 +35,27 @@ function errorToFault(err: Error): Fault {
   return fault;
 }
 
-process.on("uncaughtException", err => {
+// Need this for Node < 15
+process.on("unhandledRejection", (err) => {
+  throw err;
+});
+
+process.on("uncaughtException", (err) => {
   const fault = errorToFault(err);
-  jobs.forEach(jobId => {
+  jobs.forEach((jobId) => {
     process.send([jobId, fault, null]);
   });
   jobs.clear();
   process.disconnect();
 });
 
-process.on("message", message => {
+process.on("message", (message) => {
   jobs.add(message[0]);
 
   if (!script) {
     const cwd = process.env["GENIEACS_EXT_DIR"];
     process.chdir(cwd);
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
     script = require(`${cwd}/${fileName}`);
   }
 
@@ -79,7 +64,7 @@ process.on("message", message => {
   if (!script[funcName]) {
     const fault = {
       code: "ext",
-      message: `No such function '${funcName}' in extension '${fileName}'`
+      message: `No such function '${funcName}' in extension '${fileName}'`,
     };
     process.send([message[0], fault, null]);
     return;
@@ -93,4 +78,6 @@ process.on("message", message => {
 });
 
 // Ignore SIGINT
-process.on("SIGINT", () => {});
+process.on("SIGINT", () => {
+  // Ignore
+});

bin/genieacs-fs

@@ -1,116 +0,0 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/fs";
-import * as db from "../lib/db";
-import * as cache from "../lib/cache";
-import { version as VERSION } from "../package.json";
-
-logger.init("fs", VERSION);
-
-const SERVICE_ADDRESS = config.get("FS_INTERFACE");
-const SERVICE_PORT = config.get("FS_PORT");
-
-function exitWorkerGracefully(): void {
-  db.disconnect();
-  cache.disconnect();
-  cluster.worker.disconnect();
-}
-
-function exitWorkerUngracefully(): void {
-  process.exit(1);
-}
-
-if (!cluster.worker) {
-  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES");
-
-  logger.info({
-    message: `genieacs-fs starting`,
-    pid: process.pid,
-    version: VERSION
-  });
-
-  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
-
-  process.on("SIGINT", () => {
-    logger.info({
-      message: "Received signal SIGINT, exiting",
-      pid: process.pid
-    });
-
-    cluster.stop();
-  });
-
-  process.on("SIGTERM", () => {
-    logger.info({
-      message: "Received signal SIGTERM, exiting",
-      pid: process.pid
-    });
-
-    cluster.stop();
-  });
-} else {
-  const ssl = {
-    key: config.get("FS_SSL_KEY"),
-    cert: config.get("FS_SSL_CERT")
-  };
-
-  process.on("uncaughtException", err => {
-    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
-    logger.error({
-      message: "Uncaught exception",
-      exception: err,
-      pid: process.pid
-    });
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGINT", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
-    .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
-    })
-    .catch(err => {
-      setTimeout(() => {
-        throw err;
-      });
-    });
-}

bin/genieacs-fs.ts

@@ -0,0 +1,100 @@
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import { listener } from "../lib/fs.ts";
+import * as db from "../lib/db/db.ts";
+import { version as VERSION } from "../package.json";
+
+logger.init("fs", VERSION);
+
+const SERVICE_ADDRESS = config.get("FS_INTERFACE") as string;
+const SERVICE_PORT = config.get("FS_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([db.disconnect(), cluster.worker.disconnect()])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  process.exit(1);
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-fs starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("FS_SSL_KEY") as string;
+  const cert = config.get("FS_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
+  };
+
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, listener);
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-nbi

@@ -1,120 +0,0 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/nbi";
-import * as db from "../lib/db";
-import * as extensions from "../lib/extensions";
-import * as cache from "../lib/cache";
-import { version as VERSION } from "../package.json";
-
-logger.init("nbi", VERSION);
-
-const SERVICE_ADDRESS = config.get("NBI_INTERFACE");
-const SERVICE_PORT = config.get("NBI_PORT");
-
-function exitWorkerGracefully(): void {
-  db.disconnect();
-  cache.disconnect();
-  extensions.killAll();
-  cluster.worker.disconnect();
-}
-
-function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
-    process.exit(1);
-  });
-}
-
-if (!cluster.worker) {
-  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES");
-
-  logger.info({
-    message: `genieacs-nbi starting`,
-    pid: process.pid,
-    version: VERSION
-  });
-
-  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
-
-  process.on("SIGINT", () => {
-    logger.info({
-      message: "Received signal SIGINT, exiting",
-      pid: process.pid
-    });
-
-    cluster.stop();
-  });
-
-  process.on("SIGTERM", () => {
-    logger.info({
-      message: "Received signal SIGTERM, exiting",
-      pid: process.pid
-    });
-
-    cluster.stop();
-  });
-} else {
-  const ssl = {
-    key: config.get("NBI_SSL_KEY"),
-    cert: config.get("NBI_SSL_CERT")
-  };
-
-  process.on("uncaughtException", err => {
-    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
-    logger.error({
-      message: "Uncaught exception",
-      exception: err,
-      pid: process.pid
-    });
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGINT", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  Promise.all([db.connect(), cache.connect()])
-    .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, listener);
-    })
-    .catch(err => {
-      setTimeout(() => {
-        throw err;
-      });
-    });
-}

bin/genieacs-nbi.ts

@@ -0,0 +1,107 @@
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import { listener } from "../lib/nbi.ts";
+import * as db from "../lib/db/db.ts";
+import * as extensions from "../lib/extensions.ts";
+import { version as VERSION } from "../package.json";
+
+logger.init("nbi", VERSION);
+
+const SERVICE_ADDRESS = config.get("NBI_INTERFACE") as string;
+const SERVICE_PORT = config.get("NBI_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  void extensions.killAll().finally(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-nbi starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("NBI_SSL_KEY") as string;
+  const cert = config.get("NBI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 120000,
+  };
+
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, listener);
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-ui

@@ -1,132 +0,0 @@
-#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only
-
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/ui";
-import * as extensions from "../lib/extensions";
-import * as db from "../lib/ui/db";
-import * as db2 from "../lib/db";
-import * as cache from "../lib/cache";
-import { version as VERSION } from "../package.json";
-
-logger.init("ui", VERSION);
-
-const SERVICE_ADDRESS = config.get("UI_INTERFACE");
-const SERVICE_PORT = config.get("UI_PORT");
-
-function exitWorkerGracefully(): void {
-  db.disconnect();
-  db2.disconnect();
-  cache.disconnect();
-  extensions.killAll();
-  cluster.worker.disconnect();
-}
-
-function exitWorkerUngracefully(): void {
-  extensions.killAll().then(() => {
-    process.exit(1);
-  });
-}
-
-if (!cluster.worker) {
-  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES");
-
-  logger.info({
-    message: `genieacs-ui starting`,
-    pid: process.pid,
-    version: VERSION
-  });
-
-  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
-
-  process.on("SIGINT", () => {
-    logger.info({
-      message: "Received signal SIGINT, exiting",
-      pid: process.pid
-    });
-
-    cluster.stop();
-  });
-
-  process.on("SIGTERM", () => {
-    logger.info({
-      message: "Received signal SIGTERM, exiting",
-      pid: process.pid
-    });
-
-    cluster.stop();
-  });
-} else {
-  const ssl = {
-    key: config.get("UI_SSL_KEY"),
-    cert: config.get("UI_SSL_CERT")
-  };
-
-  let stopping = false;
-
-  process.on("uncaughtException", err => {
-    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
-    logger.error({
-      message: "Uncaught exception",
-      exception: err,
-      pid: process.pid
-    });
-    stopping = true;
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGINT", () => {
-    stopping = true;
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  process.on("SIGTERM", () => {
-    stopping = true;
-    server
-      .stop()
-      .then(exitWorkerGracefully)
-      .catch(exitWorkerUngracefully);
-  });
-
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
-  };
-
-  Promise.all([db2.connect(), cache.connect()])
-    .then(() => {
-      server.start(SERVICE_PORT, SERVICE_ADDRESS, ssl, _listener);
-    })
-    .catch(err => {
-      setTimeout(() => {
-        throw err;
-      });
-    });
-}

bin/genieacs-ui.ts

@@ -0,0 +1,107 @@
+import * as config from "../lib/config.ts";
+import * as logger from "../lib/logger.ts";
+import * as cluster from "../lib/cluster.ts";
+import * as server from "../lib/server.ts";
+import { listener } from "../lib/ui.ts";
+import * as extensions from "../lib/extensions.ts";
+import * as db from "../lib/db/db.ts";
+import { version as VERSION } from "../package.json";
+
+logger.init("ui", VERSION);
+
+const SERVICE_ADDRESS = config.get("UI_INTERFACE") as string;
+const SERVICE_PORT = config.get("UI_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ])
+    .then(logger.close)
+    .catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  void extensions.killAll().finally(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-ui starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("UI_SSL_KEY") as string;
+  const cert = config.get("UI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 120000,
+  };
+
+  // Need this for Node < 15
+  process.on("unhandledRejection", (err) => {
+    throw err;
+  });
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, async (req, res) => listener(req, res));
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    void initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+}

build/assets.ts

@@ -0,0 +1,5 @@
+export const APP_JS = "app.js";
+export const APP_CSS = "app.css";
+export const ICONS_SVG = "icons.svg";
+export const LOGO_SVG = "logo.svg";
+export const FAVICON_PNG = "favicon.png";

build/build.ts

@@ -1,97 +1,46 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
+import path from "node:path";
+import fs from "node:fs";
+import { createHash } from "node:crypto";
+import { promisify } from "node:util";
+import { exec } from "node:child_process";
+import * as esbuild from "esbuild";
+import { optimize } from "svgo";
+import * as xmlParser from "../lib/xml-parser.ts";
 
-import * as path from "path";
-import * as fs from "fs";
-import { promisify } from "util";
-import { rollup, WarningHandler } from "rollup";
-import rollupReplace from "rollup-plugin-replace";
-import rollupJson from "rollup-plugin-json";
-import typescript from "rollup-plugin-typescript";
-import { terser } from "rollup-plugin-terser";
-import webpack from "webpack";
-import postcss from "postcss";
-import postcssImport from "postcss-import";
-import postcssPresetEnv from "postcss-preset-env";
-import cssnano from "cssnano";
-import SVGO from "svgo";
-import * as xmlParser from "../lib/xml-parser";
+const fsAsync = {
+  readdir: promisify(fs.readdir),
+  readFile: promisify(fs.readFile),
+  writeFile: promisify(fs.writeFile),
+  copyFile: promisify(fs.copyFile),
+  rename: promisify(fs.rename),
+  chmod: promisify(fs.chmod),
+  lstat: promisify(fs.lstat),
+  exists: promisify(fs.exists),
+  rmdir: promisify(fs.rmdir),
+  unlink: promisify(fs.unlink),
+  mkdir: promisify(fs.mkdir),
+};
+
+const execAsync = promisify(exec);
 
 const MODE = process.env["NODE_ENV"] || "production";
 
-const BUILD_METADATA = new Date()
-  .toISOString()
-  .split(".")[0]
-  .replace(/[^0-9]/g, "");
+const INPUT_DIR = process.cwd();
+const OUTPUT_DIR = path.join(INPUT_DIR, "dist");
 
-const INPUT_DIR = path.resolve(__dirname, "..");
-const OUTPUT_DIR = path.resolve(__dirname, "../dist");
-
-const externals = [
-  "path",
-  "fs",
-  "cluster",
-  "os",
-  "tls",
-  "http",
-  "https",
-  "zlib",
-  "crypto",
-  "util",
-  "mongodb",
-  "libxmljs",
-  "vm",
-  "later",
-  "parsimmon",
-  "seedrandom",
-  "querystring",
-  "child_process",
-  "dgram",
-  "url",
-  "koa",
-  "koa-router",
-  "koa-compress",
-  "koa-bodyparser",
-  "koa-jwt",
-  "koa-static",
-  "jsonwebtoken",
-  "stream",
-  "mithril",
-  "parsimmon",
-  "yaml",
-  "codemirror",
-  "codemirror/mode/javascript/javascript",
-  "codemirror/mode/yaml/yaml"
-];
-
-function rmDirSync(dirPath): void {
-  if (!fs.existsSync(dirPath)) return;
-  const files = fs.readdirSync(dirPath);
+async function rmDir(dirPath: string): Promise<void> {
+  if (!(await fsAsync.exists(dirPath))) return;
+  const files = await fsAsync.readdir(dirPath);
 
   for (const file of files) {
-    const filePath = `${dirPath}/${file}`;
-    if (fs.statSync(filePath).isFile()) fs.unlinkSync(filePath);
-    else rmDirSync(filePath);
+    const filePath = path.join(dirPath, file);
+    if ((await fsAsync.lstat(filePath)).isDirectory()) await rmDir(filePath);
+    else await fsAsync.unlink(filePath);
   }
-  fs.rmdirSync(dirPath);
+  await fsAsync.rmdir(dirPath);
 }
 
+// For lockfileVersion = 1
 function stripDevDeps(deps): void {
   if (!deps["dependencies"]) return;
   for (const [k, v] of Object.entries(deps["dependencies"])) {
@@ -101,6 +50,15 @@ function stripDevDeps(deps): void {
   if (!Object.keys(deps["dependencies"]).length) delete deps["dependencies"];
 }
 
+// For lockfileVersion = 2
+function stripDevDeps2(deps): void {
+  if (!deps["packages"]) return;
+  for (const [k, v] of Object.entries(deps["packages"])) {
+    delete v["devDependencies"];
+    if (v["dev"]) delete deps["packages"][k];
+  }
+}
+
 function xmlTostring(xml): string {
   const children = [];
   for (const c of xml.children || []) children.push(xmlTostring(c));
@@ -110,6 +68,82 @@ function xmlTostring(xml): string {
     : `<${xml.name} ${xml.attrs}>${children.join("")}</${xml.name}>`;
 }
 
+function assetHash(buffer: Buffer | string): string {
+  return createHash("md5").update(buffer).digest("hex").slice(0, 8);
+}
+
+const ASSETS = {} as {
+  APP_JS?: string;
+  APP_CSS?: string;
+  ICONS_SVG?: string;
+  LOGO_SVG?: string;
+  FAVICON_PNG?: string;
+};
+
+const assetsPlugin = {
+  name: "assets",
+  setup(build) {
+    build.onLoad({ filter: /\/build\/assets.ts$/ }, () => {
+      const lines = Object.entries(ASSETS).map(
+        ([k, v]) => `export const ${k} = ${JSON.stringify(v)};`,
+      );
+      return { contents: lines.join("\n") };
+    });
+  },
+} as esbuild.Plugin;
+
+const seedPlugin = {
+  name: "seed",
+  setup(build) {
+    build.onLoad({ filter: /\/seed\// }, (args) => {
+      if (args.with?.["type"] !== "text") return undefined;
+      let contents = fs.readFileSync(args.path, "utf8");
+      // Strip TypeScript directives that are only needed for type-checking
+      contents = contents.replace(/^\s*\/\/\s*@ts-.*\n/gm, "");
+      return { contents, loader: "text" };
+    });
+  },
+} as esbuild.Plugin;
+
+const packageDotJsonPlugin = {
+  name: "packageDotJson",
+  setup(build) {
+    const sourcePath = path.join(INPUT_DIR, "package.json");
+    build.onResolve({ filter: /\/package.json$/ }, (args) => {
+      const p = path.join(args.resolveDir, args.path);
+      if (p !== sourcePath) return undefined;
+      return { path: path.join(OUTPUT_DIR, "package.json") };
+    });
+  },
+} as esbuild.Plugin;
+
+const inlineDepsPlugin = {
+  name: "inlineDeps",
+  setup(build) {
+    const deps = ["espresso-iisojs", "@codemirror", "mithril", "yaml"];
+    const depFiles = new Set();
+    build.onResolve({ filter: /^[^.]/ }, async (args) => {
+      if (args.pluginData === "inlineDeps") return undefined;
+      if (
+        depFiles.has(args.importer) ||
+        deps.some((d) => args.path.startsWith(d))
+      ) {
+        const res = await build.resolve(args.path, {
+          importer: args.importer,
+          namespace: args.namespace,
+          resolveDir: args.resolveDir,
+          kind: args.kind,
+          with: args.with,
+          pluginData: "inlineDeps",
+        });
+        depFiles.add(res.path);
+        return res;
+      }
+      return { sideEffects: false, external: true };
+    });
+  },
+} as esbuild.Plugin;
+
 function generateSymbol(id: string, svgStr: string): string {
   const xml = xmlParser.parseXml(svgStr);
   const svg = xml.children[0];
@@ -122,48 +156,65 @@ function generateSymbol(id: string, svgStr: string): string {
     }
   }
   const symbolBody = xml.children[0].children
-    .map(c => {
-      return xmlTostring(c);
-    })
+    .map((c) => xmlTostring(c))
     .join("");
   return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
 }
 
+async function getBuildMetadata(): Promise<string> {
+  const date = new Date().toISOString().slice(2, 10).replaceAll("-", "");
+
+  const [commit, diff, newFiles] = await Promise.all([
+    execAsync("git rev-parse HEAD"),
+    execAsync("git diff HEAD"),
+    execAsync("git ls-files --others --exclude-standard"),
+  ]).then((res) => res.map((r) => r.stdout.trim()));
+
+  if (!diff && !newFiles) return date + commit.slice(0, 4);
+
+  const hash = createHash("md5");
+  hash.update(commit).update(diff).update(newFiles);
+  for (const file of newFiles.split("\n").filter((f) => f))
+    hash.update(await fsAsync.readFile(file));
+  return date + hash.digest("hex").slice(0, 4);
+}
+
 async function init(): Promise<void> {
-  // Delete any old output directory
-  rmDirSync(OUTPUT_DIR);
-
-  // Create output directory layout
-  fs.mkdirSync(OUTPUT_DIR);
-  fs.mkdirSync(OUTPUT_DIR + "/bin");
-  fs.mkdirSync(OUTPUT_DIR + "/public");
-  fs.mkdirSync(OUTPUT_DIR + "/tools");
-
-  // Create package.json
-  const packageJson = JSON.parse(
-    fs.readFileSync(path.resolve(INPUT_DIR, "package.json")).toString()
+  const [buildMetadata, packageJsonFile, npmShrinkwrapFile] = await Promise.all(
+    [
+      getBuildMetadata(),
+      fsAsync.readFile(path.join(INPUT_DIR, "package.json")),
+      fsAsync.readFile(path.join(INPUT_DIR, "npm-shrinkwrap.json")),
+    ],
   );
+
+  const packageJson = JSON.parse(packageJsonFile.toString());
   delete packageJson["devDependencies"];
-  packageJson["scripts"] = {
-    install: packageJson["scripts"].install,
-    configure: packageJson["scripts"].configure
-  };
-  packageJson["version"] = `${packageJson["version"]}+${BUILD_METADATA}`;
-  fs.writeFileSync(
-    path.resolve(OUTPUT_DIR, "package.json"),
-    JSON.stringify(packageJson, null, 2)
-  );
+  delete packageJson["private"];
+  delete packageJson["scripts"];
+  packageJson["version"] = `${packageJson["version"]}+${buildMetadata}`;
 
-  // Create npm-shrinkwrap.json
-  const npmShrinkwrapJson = JSON.parse(
-    fs.readFileSync(path.resolve(INPUT_DIR, "npm-shrinkwrap.json")).toString()
-  );
-  npmShrinkwrapJson["version"] = packageJson["version"];
-  stripDevDeps(npmShrinkwrapJson);
-  fs.writeFileSync(
-    path.resolve(OUTPUT_DIR, "npm-shrinkwrap.json"),
-    JSON.stringify(npmShrinkwrapJson, null, 2)
-  );
+  const npmShrinkwrap = JSON.parse(npmShrinkwrapFile.toString());
+  npmShrinkwrap["version"] = packageJson["version"];
+  stripDevDeps(npmShrinkwrap);
+  stripDevDeps2(npmShrinkwrap);
+
+  await rmDir(OUTPUT_DIR);
+
+  await fsAsync.mkdir(OUTPUT_DIR);
+
+  await Promise.all([
+    fsAsync.mkdir(path.join(OUTPUT_DIR, "bin")),
+    fsAsync.mkdir(path.join(OUTPUT_DIR, "public")),
+    fsAsync.writeFile(
+      path.join(OUTPUT_DIR, "package.json"),
+      JSON.stringify(packageJson, null, 2),
+    ),
+    fsAsync.writeFile(
+      path.join(OUTPUT_DIR, "npm-shrinkwrap.json"),
+      JSON.stringify(npmShrinkwrap, null, 2),
+    ),
+  ]);
 }
 
 async function copyStatic(): Promise<void> {
@@ -172,219 +223,174 @@ async function copyStatic(): Promise<void> {
     "README.md",
     "CHANGELOG.md",
     "public/logo.svg",
-    "public/favicon.png"
+    "public/favicon.png",
   ];
 
-  for (const file of files) {
-    fs.copyFileSync(
-      path.resolve(INPUT_DIR, file),
-      path.resolve(OUTPUT_DIR, file)
-    );
-  }
+  const [logo, favicon] = await Promise.all([
+    fsAsync.readFile(path.join(INPUT_DIR, "public/logo.svg")),
+    fsAsync.readFile(path.join(INPUT_DIR, "public/favicon.png")),
+  ]);
+
+  ASSETS.LOGO_SVG = `logo-${assetHash(logo)}.svg`;
+  ASSETS.FAVICON_PNG = `favicon-${assetHash(favicon)}.png`;
+
+  const filenames = {} as Record<string, string>;
+  filenames["public/logo.svg"] = path.join("public", ASSETS.LOGO_SVG);
+  filenames["public/favicon.png"] = path.join("public", ASSETS.FAVICON_PNG);
+
+  await Promise.all(
+    files.map((f) =>
+      fsAsync.copyFile(
+        path.join(INPUT_DIR, f),
+        path.join(OUTPUT_DIR, filenames[f] || f),
+      ),
+    ),
+  );
 }
 
 async function generateCss(): Promise<void> {
-  const cssInPath = path.resolve(INPUT_DIR, "ui/css/app.css");
-  const cssOutPath = path.resolve(OUTPUT_DIR, "public/app.css");
-  const cssIn = fs.readFileSync(cssInPath);
-  const cssOut = await postcss([
-    postcssImport,
-    postcssPresetEnv({
-      stage: 3,
-      features: {
-        "nesting-rules": true,
-        "color-mod-function": true
-      }
-    }),
-    cssnano
-  ]).process(cssIn, { from: cssInPath, to: cssOutPath });
-  fs.writeFileSync(cssOutPath, cssOut.css);
-}
+  const tailwindPlugin = {
+    name: "tailwind",
+    setup(build) {
+      build.onLoad({ filter: /\/ui\/css\/app.css$/ }, async (args) => {
+        const res = await execAsync(`npx @tailwindcss/cli -i ${args.path}`);
+        return { loader: "css", contents: res.stdout };
+      });
+    },
+  } as esbuild.Plugin;
 
-async function generateToolsJs(): Promise<void> {
-  for (const bin of ["dump-data-model"]) {
-    const inputFile = path.resolve(INPUT_DIR, `tools/${bin}`);
-    const outputFile = path.resolve(OUTPUT_DIR, `tools/${bin}`);
-    const bundle = await rollup({
-      input: inputFile,
-      external: externals,
-      acorn: {
-        allowHashBang: true
-      },
-      plugins: [
-        rollupReplace({
-          delimiters: ["", ""],
-          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
-        }),
-        typescript({
-          tsconfig: "./tsconfig.json",
-          include: [`tools/${bin}`, "lib/**/*.ts"]
-        }),
-        MODE === "production" ? terser() : null
-      ]
-    });
+  const res = await esbuild.build({
+    bundle: true,
+    absWorkingDir: INPUT_DIR,
+    minify: MODE === "production",
+    sourcemap: "linked",
+    sourcesContent: false,
+    entryPoints: ["ui/css/app.css"],
+    entryNames: "[dir]/[name]-[hash]",
+    outfile: path.join(OUTPUT_DIR, "public/app.css"),
+    plugins: [tailwindPlugin],
+    loader: {
+      ".woff2": "dataurl",
+    },
+    target: ["chrome111", "safari16.4", "firefox128"],
+    metafile: true,
+  });
 
-    await bundle.write({
-      format: "cjs",
-      preferConst: true,
-      sourcemap: "inline",
-      sourcemapExcludeSources: true,
-      banner: "#!/usr/bin/env node",
-      file: outputFile
-    });
-
-    // Mark as executable
-    const mode = fs.statSync(outputFile).mode;
-    fs.chmodSync(outputFile, mode | 73);
+  for (const [k, v] of Object.entries(res.metafile.outputs)) {
+    if (v.entryPoint === "ui/css/app.css") {
+      ASSETS.APP_CSS = path.relative(
+        path.join(OUTPUT_DIR, "public"),
+        path.join(INPUT_DIR, k),
+      );
+      break;
+    }
   }
 }
 
 async function generateBackendJs(): Promise<void> {
-  for (const bin of [
+  const services = [
     "genieacs-cwmp",
     "genieacs-ext",
     "genieacs-nbi",
     "genieacs-fs",
-    "genieacs-ui"
-  ]) {
-    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}`);
-    const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
-    const bundle = await rollup({
-      input: inputFile,
-      external: externals,
-      acorn: {
-        allowHashBang: true
-      },
-      treeshake: {
-        propertyReadSideEffects: false,
-        pureExternalModules: true
-      },
-      plugins: [
-        rollupReplace({
-          delimiters: ["", ""],
-          "#!/usr/bin/env -S node -r esm -r ts-node/register/transpile-only": ""
-        }),
-        rollupJson({ preferConst: true }),
-        {
-          resolveId: (importee, importer) => {
-            if (importee.endsWith("/package.json")) {
-              const p = path.resolve(path.dirname(importer), importee);
-              if (p === path.resolve(INPUT_DIR, "package.json"))
-                return path.resolve(OUTPUT_DIR, "package.json");
-            }
-            return null;
-          }
-        },
-        typescript({
-          tsconfig: "./tsconfig.json",
-          include: [`bin/${bin}`, "lib/**/*.ts"]
-        }),
-        MODE === "production" ? terser() : null
-      ]
-    });
+    "genieacs-ui",
+  ];
 
-    await bundle.write({
-      format: "cjs",
-      preferConst: true,
-      sourcemap: "inline",
-      sourcemapExcludeSources: true,
-      banner: "#!/usr/bin/env node",
-      file: outputFile
-    });
+  await esbuild.build({
+    bundle: true,
+    absWorkingDir: INPUT_DIR,
+    minify: MODE === "production",
+    sourcemap: "inline",
+    sourcesContent: false,
+    platform: "node",
+    target: "node12.13.0",
+    packages: "external",
+    banner: { js: "#!/usr/bin/env node" },
+    entryPoints: services.map((s) => `bin/${s}.ts`),
+    outdir: path.join(OUTPUT_DIR, "bin"),
+    plugins: [packageDotJsonPlugin, assetsPlugin, seedPlugin],
+  });
 
+  for (const bin of services) {
+    const p = path.join(OUTPUT_DIR, "bin", bin);
+    await fsAsync.rename(`${p}.js`, p);
     // Mark as executable
-    const mode = fs.statSync(outputFile).mode;
-    fs.chmodSync(outputFile, mode | 73);
+    const mode = (await fsAsync.lstat(p)).mode;
+    await fsAsync.chmod(p, mode | 73);
   }
 }
 
 async function generateFrontendJs(): Promise<void> {
-  const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
-  const outputFile = path.resolve(OUTPUT_DIR, "public/app.js");
-
-  const bundle = await rollup({
-    input: inputFile,
-    external: externals,
-    plugins: [
-      rollupJson({ preferConst: true }),
-      typescript({ tsconfig: "./tsconfig.json" })
-    ],
-    inlineDynamicImports: true,
-    treeshake: {
-      propertyReadSideEffects: false,
-      pureExternalModules: true
-    },
-    onwarn: ((warning, warn) => {
-      // Ignore circular dependency warnings
-      if (warning.code !== "CIRCULAR_DEPENDENCY") warn(warning);
-    }) as WarningHandler
-  });
-
-  await bundle.write({
-    preferConst: true,
+  const res = await esbuild.build({
+    bundle: true,
+    absWorkingDir: INPUT_DIR,
+    splitting: true,
+    minify: MODE === "production",
+    sourcemap: "linked",
+    sourcesContent: false,
+    platform: "browser",
     format: "esm",
-    sourcemap: "inline",
-    sourcemapExcludeSources: true,
-    file: outputFile
+    target: ["chrome111", "safari16.4", "firefox128"],
+    entryPoints: ["ui/app.ts"],
+    entryNames: "[dir]/[name]-[hash]",
+    outdir: path.join(OUTPUT_DIR, "public"),
+    plugins: [packageDotJsonPlugin, inlineDepsPlugin, assetsPlugin],
+    metafile: true,
   });
 
-  const webpackConf = {
-    mode: MODE,
-    entry: outputFile,
-    resolve: {
-      aliasFields: ["module"]
-    },
-    devtool: "nosources-source-map",
-    module: {
-      rules: [
-        {
-          test: /\.js$/,
-          use: ["source-map-loader"],
-          enforce: "pre"
-        }
-      ]
-    },
-    output: {
-      path: path.resolve(OUTPUT_DIR, "public"),
-      filename: "app.js"
-    }
-  };
+  for (const [k, v] of Object.entries(res.metafile.outputs)) {
+    for (const imp of v.imports)
+      if (imp.external) throw new Error(`External import found: ${imp.path}`);
 
-  const stats = await promisify(webpack)(webpackConf);
-  process.stdout.write(stats.toString({ colors: true }) + "\n");
+    if (v.entryPoint === "ui/app.ts") {
+      ASSETS.APP_JS = path.relative(
+        path.join(OUTPUT_DIR, "public"),
+        path.join(INPUT_DIR, k),
+      );
+    }
+  }
 }
 
 async function generateIconsSprite(): Promise<void> {
-  const svgo = new SVGO({ plugins: [{ removeViewBox: false }] });
-  const symbols = [];
-  const iconsDir = path.resolve(INPUT_DIR, "ui/icons");
-  for (const file of fs.readdirSync(iconsDir)) {
+  const symbols = [] as string[];
+  const iconsDir = path.join(INPUT_DIR, "ui/icons");
+  for (const file of await fsAsync.readdir(iconsDir)) {
     const id = path.parse(file).name;
     const filePath = path.join(iconsDir, file);
-    const { data } = await svgo.optimize(fs.readFileSync(filePath).toString());
+    const src = (await fsAsync.readFile(filePath)).toString();
+    const { data } = await optimize(src, {
+      plugins: [
+        {
+          name: "preset-default",
+          params: {
+            overrides: {
+              removeViewBox: false,
+            },
+          },
+        },
+      ],
+    });
     symbols.push(generateSymbol(id, data));
   }
-  fs.writeFileSync(
-    path.resolve(OUTPUT_DIR, "public/icons.svg"),
-    `<svg xmlns="http://www.w3.org/2000/svg">${symbols.join("")}</svg>`
+  const data = `<svg xmlns="http://www.w3.org/2000/svg">${symbols.join(
+    "",
+  )}</svg>`;
+  ASSETS.ICONS_SVG = `icons-${assetHash(data)}.svg`;
+  await fsAsync.writeFile(
+    path.join(OUTPUT_DIR, "public", ASSETS.ICONS_SVG),
+    data,
   );
 }
 
 init()
-  .then(() => {
+  .then(() =>
     Promise.all([
-      copyStatic(),
+      Promise.all([generateIconsSprite(), copyStatic()]).then(
+        generateFrontendJs,
+      ),
       generateCss(),
-      generateIconsSprite(),
-      generateToolsJs(),
-      generateBackendJs(),
-      generateFrontendJs()
-    ])
-      .then(() => {})
-      .catch(err => {
-        process.stderr.write(err.stack + "\n");
-      });
-  })
-  .catch(err => {
+    ]).then(generateBackendJs),
+  )
+  .catch((err) => {
     process.stderr.write(err.stack + "\n");
   });

build/generate-fonts.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"
+
+declare -A FONTS=(
+  [InterVariable]=https://rsms.me/inter/font-files/InterVariable.woff2
+  [InterVariable-Italic]=https://rsms.me/inter/font-files/InterVariable-Italic.woff2
+  [RobotoMono]=https://raw.githubusercontent.com/googlefonts/RobotoMono/main/fonts/variable/RobotoMono%5Bwght%5D.ttf
+)
+
+for name in "${!FONTS[@]}"
+do
+  url="${FONTS[$name]}"
+  echo "Downloading $url"
+  TMP=$(mktemp)
+  curl "$url" -s --output "$TMP"
+  pyftsubset "$TMP" --output-file="../ui/css/$name.woff2" --flavor=woff2 --unicodes="U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD" --layout-features+="tnum"
+done

build/lint.ts

@@ -0,0 +1,55 @@
+import { exec } from "node:child_process";
+import { promisify } from "node:util";
+
+const execPromise = promisify(exec);
+
+async function runEslint(): Promise<string> {
+  const CMD =
+    "eslint 'bin/*.ts' 'lib/**/*.ts' 'ui/**/*.ts' 'test/**/*.ts' 'build/**/*.ts' 'seed/*'";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  try {
+    const { stdout, stderr } = await execPromise(CMD, { env });
+    if (stderr) throw new Error(stderr);
+    return stdout;
+  } catch (err) {
+    if (err.killed || err.signal || err.stderr || err.code !== 1) throw err;
+    return err.stdout;
+  }
+}
+
+async function runTsc(): Promise<string> {
+  const CMD = "tsc --noEmit && tsc -p seed/tsconfig.json";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  const { stdout, stderr } = await execPromise(CMD, { env });
+  if (stderr) throw new Error(stderr);
+  return stdout;
+}
+
+async function runPrettier(): Promise<string> {
+  const CMD = "prettier --prose-wrap always --write .";
+  const env = {
+    ...(process.stdout.isTTY && { FORCE_COLOR: "1" }),
+    ...process.env,
+  };
+  const { stdout, stderr } = await execPromise(CMD, { env });
+  if (stderr) throw new Error(stderr);
+  return stdout;
+}
+
+async function runAll(): Promise<void> {
+  const prom1 = runPrettier();
+  const prom2 = runEslint();
+  const prom3 = runTsc();
+
+  console.log(await prom1);
+  console.log(await prom2);
+  console.log(await prom3);
+}
+
+runAll().catch(console.error);

build/spellcheck-dict.pws

@@ -0,0 +1,180 @@
+personal_ws-1.1 en 0
+genieacs
+GenieACS
+javascript
+sudo
+config
+cwmp
+CPE
+systemctl
+nbi
+CWMP
+NBI
+ACS
+SSL
+xsd
+args
+DeviceID
+GENIEACS
+env
+param
+arg
+AUTH
+igd
+TCP
+toctree
+auth
+EnvironmentFile
+JSON
+json
+parameterValues
+setParameterValues
+stderr
+stdout
+attr
+cfg
+CPEs
+ExecStart
+faq
+FileType
+GenieACS's
+JWT
+maxdepth
+npm
+objectName
+refreshObject
+SerialNumber
+systemd
+usr
+WantedBy
+yaml
+chown
+Config
+getParameterValues
+https
+InternetGatewayDevice
+mkdir
+OUI
+sql
+TODO
+UDP
+addObject
+AGPLv
+const
+cpe
+cron
+deleteObject
+FactoryReset
+factoryReset
+FileName
+fileType
+Github
+hostname
+HTTPS
+jwt
+latlong
+NPM
+oui
+parameterNames
+ProductClass
+productClass
+repo
+SSID
+sublicense
+TLS
+WANIPConnection
+WPA
+YourProvisionName
+abc
+Abdulla
+accessors
+AddObject
+AddressingType
+api
+APIs
+chmod
+cwmp's
+dateext
+DATETIME
+dateTime
+datetime
+declaratively
+delaycompress
+DeleteObject
+deviceId
+DHCP
+dir
+distro
+equisatisfiable
+ExternalIPAddress
+failover
+favor
+func
+GetParameterAttributes
+getPassword
+gte
+gui
+hexBinary
+Hils
+HOSTNAME
+init
+journalctl
+journald
+LastFileName
+LastFileType
+lastInform
+LIBXMLJS
+libxmljs
+literalinclude
+lockfile
+logrotate
+lte
+mipsbe
+normalizers
+pre
+quickstart
+README
+redis
+reimplemented
+RequestDownload
+RESTful
+rollout
+sandboxed
+scalable
+SetParameterAttributes
+SetParamteerValues
+SIGINT
+SIGTERM
+skipWritableCheck
+stringify
+subdirectory
+TargetFileName
+useradd
+VirtualParameters
+vparam
+xml
+YAML
+Zaid
+cacheExpire
+iss
+http
+statusCode
+rawData
+pos
+Brotli
+CSV
+hostInfo
+IPv
+downloadSuccessOnTimeout
+TransferComplete
+nginx
+ENCODEURICOMPONENT
+pageSize
+skipRootGpn
+GPN
+NaN
+CIDR
+XMPP
+PEM
+JID
+unindexed

build/spellcheck.sh

@@ -0,0 +1,12 @@
+#! /bin/sh
+
+cd "$(dirname "$0")"
+
+FILES=`ls ../docs/*.rst ../docs/*.js ../*.md`
+
+for FILE in $FILES
+do
+    echo $FILE
+    cat $FILE | aspell list --lang=en --add-extra-dicts=./spellcheck-dict.pws --ignore 2
+    echo
+done

docker-compose.yml

@@ -0,0 +1,134 @@
+services:
+
+  # ── MongoDB ──────────────────────────────────────────────────────────────
+  mongo:
+    image: mongo:6
+    container_name: genieacs-mongo
+    restart: unless-stopped
+    volumes:
+      - mongo_data:/data/db
+      - mongo_config:/data/configdb
+    networks:
+      - genieacs
+    healthcheck:
+      test: ["CMD", "mongosh", "--quiet", "--eval", "db.adminCommand('ping').ok"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 20s
+
+  # ── CWMP — TR-069 protocol handler (porta 7547) ──────────────────────────
+  cwmp:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: runtime
+    image: genieacs-unifi:latest
+    container_name: genieacs-cwmp
+    restart: unless-stopped
+    command: ["node", "bin/genieacs-cwmp"]
+    ports:
+      - "7547:7547"
+    env_file: .env
+    environment:
+      GENIEACS_MONGODB_CONNECTION_URL: mongodb://mongo:27017/genieacs
+    depends_on:
+      mongo:
+        condition: service_healthy
+    volumes:
+      - ext_dir:/opt/genieacs/config/ext
+      - logs:/opt/genieacs/logs
+    networks:
+      - genieacs
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:7547/ || true"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
+
+  # ── NBI — Northbound REST API (porta 7557) ───────────────────────────────
+  nbi:
+    image: genieacs-unifi:latest
+    container_name: genieacs-nbi
+    restart: unless-stopped
+    command: ["node", "bin/genieacs-nbi"]
+    ports:
+      - "7557:7557"
+    env_file: .env
+    environment:
+      GENIEACS_MONGODB_CONNECTION_URL: mongodb://mongo:27017/genieacs
+    depends_on:
+      mongo:
+        condition: service_healthy
+      cwmp:
+        condition: service_started
+    volumes:
+      - ext_dir:/opt/genieacs/config/ext
+      - logs:/opt/genieacs/logs
+    networks:
+      - genieacs
+
+  # ── FS — File Server / GridFS (porta 7567) ───────────────────────────────
+  fs:
+    image: genieacs-unifi:latest
+    container_name: genieacs-fs
+    restart: unless-stopped
+    command: ["node", "bin/genieacs-fs"]
+    ports:
+      - "7567:7567"
+    env_file: .env
+    environment:
+      GENIEACS_MONGODB_CONNECTION_URL: mongodb://mongo:27017/genieacs
+    depends_on:
+      mongo:
+        condition: service_healthy
+      cwmp:
+        condition: service_started
+    volumes:
+      - ext_dir:/opt/genieacs/config/ext
+      - logs:/opt/genieacs/logs
+    networks:
+      - genieacs
+
+  # ── UI — Web Interface (porta 3000) ──────────────────────────────────────
+  ui:
+    image: genieacs-unifi:latest
+    container_name: genieacs-ui
+    restart: unless-stopped
+    command: ["node", "bin/genieacs-ui"]
+    ports:
+      - "3000:3000"
+    env_file: .env
+    environment:
+      GENIEACS_MONGODB_CONNECTION_URL: mongodb://mongo:27017/genieacs
+    depends_on:
+      mongo:
+        condition: service_healthy
+      nbi:
+        condition: service_started
+    volumes:
+      - ext_dir:/opt/genieacs/config/ext
+      - logs:/opt/genieacs/logs
+    networks:
+      - genieacs
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:3000/ || exit 1"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 20s
+
+volumes:
+  mongo_data:
+    driver: local
+  mongo_config:
+    driver: local
+  ext_dir:
+    driver: local
+  logs:
+    driver: local
+
+networks:
+  genieacs:
+    driver: bridge

docs/.readthedocs.yaml

@@ -0,0 +1,18 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+
+sphinx:
+  configuration: docs/conf.py
+
+formats: all
+
+python:
+  install:
+    - requirements: docs/requirements.txt

docs/administration-faq.rst

@@ -0,0 +1,64 @@
+.. _administration-faq:
+
+Administration FAQ
+==================
+
+.. _administration-faq-duplicate-log-entries:
+
+Duplicate log entries when using :func:`log()` function
+-------------------------------------------------------
+
+Because GenieACS uses a full fledged scripting language for device
+configuration, the only way to guarantee that it has satisfied the 'desired
+state' is by repeatedly executing the script until there's no more
+discrepancies with the current device state. Though it may seem like this will
+cause duplicate requests going to the device, this isn't actually the case
+because device configuration are stated declaratively and that the scripts
+themselves are pure functions in the context of a session (e.g. Date.now()
+always returns the same value within the session).
+
+To illustrate with an example, consider the following script:
+
+.. code:: javascript
+
+  log("Executing script");
+  declare("Device.param", null, {value: 1});
+  commit();
+  declare("Device.param", null, {value: 2});
+
+This will set the value of the 'Device.param' to 1, then to 2. Then as the
+script is run again the value is set back to 1 and so on. A stable state will
+never be reached so GenieACS will execute the script a few times until it gives
+up and throws a fault. This is an edge case that should be avoided. A more
+typical case is where the script is run once or twice. Essentially if an
+execution doesn't result in any request to the CPE or a change in the data
+model then a stable state is deemed to have been reached.
+
+Configurations not pushed to device after factory reset
+---------------------------------------------------------
+
+After a device is reset to its factory default state, the cached data model in
+GenieACS's database needs to be invalidated to force rediscovery. Ensure the
+following lines are called on ``0 BOOTSTRAP`` event:
+
+.. code:: javascript
+
+  const now = Date.now();
+
+  // Clear cached data model to force a refresh
+  clear("Device", now);
+  clear("InternetGatewayDevice", now);
+
+
+Most device parameters are missing
+----------------------------------
+
+For performance reasons (server, client, and network), GenieACS by default only
+fetches parts of the data model that are necessary to satisfy the declarations
+in your provision scripts. Create declarations for any parameters you need
+fetched by default.
+
+If you're unsure and want to explore the available parameters exposed by the
+device, refresh the root parameter (e.g. ``InternetGatewayDevice``) from
+GenieACS's UI. You typically only need to do that one time for a given CPE
+model.

docs/api-reference.rst

@@ -1,7 +1,7 @@
 API Reference
 =============
 
-GenieACS expoeses a rich RESTful API through its NBI component. This document
+GenieACS exposes a rich RESTful API through its NBI component. This document
 serves as a reference for the available APIs.
 
 This API makes use of MongoDB's query language in some of its endpoints. Refer
@@ -16,7 +16,7 @@ to MongoDB's documentation for details.
 
 .. warning::
 
-  A common pitfll is not properly percent-encoding special characters in the
+  A common pitfall is not properly percent-encoding special characters in the
   device ID or query in the URL.
 
 Endpoints
@@ -41,11 +41,11 @@ Examples
 
 .. code:: javascript
 
-  query = {"_id": "202BC1-BM632w-0000000"}
+  query = {"_id": "202BC1-BM632w-000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D'
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D'
 
 - Find a device by its MAC address:
 
@@ -77,21 +77,21 @@ Examples
 
 .. code:: javascript
 
-  query = {"device": "202BC1-BM632w-0000000"}
+  query = {"device": "202BC1-BM632w-000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-0000000%22%7D'
+  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-000000%22%7D'
 
 - Return specific parameters for a given device:
 
 .. code:: javascript
 
-  query = {"_id": "202BC1-BM632w-0000000"}
+  query = {"_id": "202BC1-BM632w-000000"}
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-0000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
+  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
 
 The ``projection`` URL param is a comma-separated list of the parameters to receive.
 
@@ -106,7 +106,7 @@ if the tasks have been queued to be executed at the next inform.
 *device_id*: The ID of the device.
 
 *connection_request*: Indicates that a connection request will be triggered to
-execute the tasks immediatly. Otherwise, the tasks will be queued and be
+execute the tasks immediately. Otherwise, the tasks will be queued and be
 processed at the next inform.
 
 The response body is the task object as it is inserted in the database. The
@@ -120,7 +120,7 @@ Examples
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
   -X POST \
   --data '{"name": "refreshObject", "objectName": ""}'
 
@@ -138,7 +138,7 @@ Examples
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tasks?connection_request' \
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
   -X POST \
   --data '{"name":"setParameterValues", "parameterValues": [["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]]}'
 
@@ -183,7 +183,7 @@ Example
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/faults/202BC1-BM632w-0000000:default' -X DELETE
+  curl -i 'http://localhost:7557/faults/202BC1-BM632w-000000:default' -X DELETE
 
 DELETE /devices/\<device_id\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -218,7 +218,7 @@ Assign the tag "testing" to a device:
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X POST
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X POST
 
 DELETE /devices/\<device_id\>/tags/\<tag\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -236,7 +236,7 @@ Remove the tag "testing" from a device:
 
 .. code:: bash
 
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-0000000/tags/testing' -X DELETE
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X DELETE
 
 PUT /presets/\<preset_name\>
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -299,7 +299,7 @@ The following file metadata may be sent as request headers:
 - ``fileType``: For firmware images it should be "1 Firmware Upgrade Image".
   Other common types are "2 Web Content" and "3 Vendor Configuration File".
 
-- ``oui``: The OUI of the device model that this file belogs to.
+- ``oui``: The OUI of the device model that this file belongs to.
 
 - ``productClass``: The product class of the device.
 
@@ -344,7 +344,7 @@ Find files using a query.
 Tasks
 -----
 
-Find the different availabe tasks and their object structure.
+Find the different available tasks and their object structure.
 
 ``getParameterValues``
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -415,6 +415,15 @@ parameterValues key. For example:
   -X POST \
   --data '{"name":"addObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection"}'
 
+``deleteObject``
+~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name":"deleteObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection.1"}'
+
 ``reboot``
 ~~~~~~~~~~
 

docs/conf.py

@@ -19,7 +19,7 @@ import json
 # -- Project information -----------------------------------------------------
 
 project = 'GenieACS Documentation'
-copyright = '2019, GenieACS Inc.'
+copyright = '2024, GenieACS Inc.'
 author = 'GenieACS Inc.'
 
 # The full version, including alpha/beta/rc tags
@@ -58,7 +58,7 @@ html_theme_options = {
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 master_doc = 'index'
 

docs/cpe-authentication.rst

@@ -3,4 +3,70 @@
 CPE Authentication
 ==================
 
+CPE to ACS
+----------
+
+.. note::
+
+  By default GenieACS will accept any incoming connection via HTTP/HTTPS and
+  respond to it.
+
+The following parameters are used to set and get (password is redacted but
+can be set) the username/password used to authenticate against the ACS:
+
+Username: ``Device.ManagementServer.Username`` or ``InternetGatewayDevice.ManagementServer.Username``
+
+Password: ``Device.ManagementServer.Password`` or ``InternetGatewayDevice.ManagementServer.Password``
+
+Enable CPE to ACS Authentication
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+CPE to ACS authentication can be configured in the web interface by using the
+`Config` option in the `Admin` tab.
+
+Go to the `Admin` -> `Config` page and click on `New config` button at the
+bottom of the page. This will open pop-up which requires you to fill in a key
+and value. The key should be ``cwmp.auth``. The value accepts a boolean.
+Setting the value to ``true`` makes it so that GenieACS accepts any incoming
+connection, setting it to ``false`` makes GenieACS deny all incoming
+connections. This can be further configured using the ``AUTH()`` and ``EXT()``
+functions.
+
+The ``AUTH()`` function
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``AUTH()`` function accepts two parameters, username and password. It
+checks the given username and password with the incoming request to determine
+whether to return true or false.
+
+Basic usage of the ``AUTH()`` function could be as follows:
+
+.. code:: sql
+
+   AUTH("fixed-username", "fixed-password")
+
+This will only accept incoming request who authenticate with
+"fixed-username" and "fixed-password".
+
+The various device parameters can be referenced from within the ``cwmp.auth``
+expression. For example:
+
+.. code:: sql
+
+   AUTH(Device.ManagementServer.Username, Device.ManagementServer.Password)
+
+The ``EXT()`` function
+~~~~~~~~~~~~~~~~~~~~~~
+
+The ``EXT()`` function makes it possible to call an :ref:`extension
+<extensions>` script from the auth expression. This can be used to fetch
+the credentials from an external source:
+
+.. code:: sql
+
+   AUTH(DeviceID.SerialNumber, EXT("authenticate", "getPassword", DeviceID.SerialNumber))
+
+ACS to CPE
+----------
+
 TODO

docs/environment-variables.rst

@@ -3,4 +3,215 @@
 Environment Variables
 =====================
 
-TODO
+Configuring GenieACS services can be done through the following environment
+variables:
+
+.. attention::
+
+  All GenieACS environment variables must be prefixed with ``GENIEACS_``.
+
+MONGODB_CONNECTION_URL
+  MongoDB connection string.
+
+  Default: ``mongodb://127.0.0.1/genieacs``
+
+EXT_DIR
+  The directory from which to look up extension scripts.
+
+  Default: ``<installation dir>/config/ext``
+
+EXT_TIMEOUT
+  Timeout (in milliseconds) to allow for calls to extensions to return a
+  response.
+
+  Default: ``3000``
+
+DEBUG_FILE
+  File to dump CPE debug log.
+
+  Default: unset
+
+DEBUG_FORMAT
+  Debug log format. Valid values are 'yaml' and 'json'.
+
+  Default: ``yaml``
+
+LOG_FORMAT
+  The format used for the log entries in ``CWMP_LOG_FILE``, ``NBI_LOG_FILE``,
+  ``FS_LOG_FILE``, and ``UI_LOG_FILE``. Possible values are ``simple`` and
+  ``json``.
+
+  Default: ``simple``
+
+ACCESS_LOG_FORMAT
+  The format used for the log entries in ``CWMP_ACCESS_LOG_FILE``,
+  ``NBI_ACCESS_LOG_FILE``, ``FS_ACCESS_LOG_FILE``, and ``UI_ACCESS_LOG_FILE``.
+  Possible values are ``simple`` and ``json``.
+
+  Default: ``simple``
+
+CWMP_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-cwmp. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+CWMP_PORT
+  The TCP port that genieacs-cwmp listens on.
+
+  Default: ``7547``
+
+CWMP_INTERFACE
+  The network interface that genieacs-cwmp binds to.
+
+  Default: ``::``
+
+CWMP_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_LOG_FILE
+  File to log process related events for genieacs-cwmp. If omitted, logs will
+  go to stderr.
+
+  Default: unset
+
+CWMP_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-cwmp. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+NBI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-nbi. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+NBI_PORT
+  The TCP port that genieacs-nbi listens on.
+
+  Default: ``7557``
+
+NBI_INTERFACE
+  The network interface that genieacs-nbi binds to.
+
+  Default: ``::``
+
+NBI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+NBI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+NBI_LOG_FILE
+  File to log process related events for genieacs-nbi. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+NBI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-nbi. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-fs. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+FS_PORT
+  The TCP port that genieacs-fs listens on.
+
+  Default: ``7567``
+
+FS_INTERFACE
+  The network interface that genieacs-fs binds to.
+
+  Default: ``::``
+
+FS_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_LOG_FILE
+  File to log process related events for genieacs-fs. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+FS_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-fs. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_URL_PREFIX
+  The URL prefix (e.g. 'https://example.com:7567/') to use when generating the
+  file URL for TR-069 Download requests. Set this if genieacs-fs and
+  genieacs-cwmp are behind a proxy or running on different servers.
+
+  Default: auto generated based on the hostname from the ACS URL, FS_PORT
+  config, and whether or not SSL is enabled for genieacs-fs.
+
+UI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-ui. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+UI_PORT
+  The TCP port that genieacs-ui listens on.
+
+  Default: ``3000``
+
+UI_INTERFACE
+  The network interface that genieacs-ui binds to.
+
+  Default: ``::``
+
+UI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_LOG_FILE
+  File to log process related events for genieacs-ui. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+UI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-ui. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+UI_JWT_SECRET
+  The key used for signing JWT tokens that are stored in browser cookies. The
+  string can be up to 64 characters in length.
+
+  Default: unset

docs/ext-sample.js

@@ -1,4 +1,4 @@
-// This is an exmaple GenieACS extension to get the current latitude/longitude
+// This is an example GenieACS extension to get the current latitude/longitude
 // of the International Space Station. Why, you ask? Because why not.
 // To install, copy this file to config/ext/iss.js.
 
@@ -10,25 +10,28 @@ let cache = null;
 let cacheExpire = 0;
 
 function latlong(args, callback) {
-  if (Date.now() < cacheExpire)
-    return callback(null, cache);
+  if (Date.now() < cacheExpire) return callback(null, cache);
 
-  http.get("http://api.open-notify.org/iss-now.json", (res) => {
-    if (res.statusCode !== 200)
-      return callback(new Error(`Request failed (status code: ${res.statusCode})`));
+  http
+    .get("http://api.open-notify.org/iss-now.json", (res) => {
+      if (res.statusCode !== 200)
+        return callback(
+          new Error(`Request failed (status code: ${res.statusCode})`),
+        );
 
-    let rawData = "";
-    res.on("data", (chunk) => rawData += chunk);
+      let rawData = "";
+      res.on("data", (chunk) => (rawData += chunk));
 
-    res.on("end", () => {
-      let pos = JSON.parse(rawData)["iss_position"];
-      cache = [+pos["latitude"], +pos["longitude"]];
-      cacheExpire = Date.now() + 10000;
-      callback(null, cache);
+      res.on("end", () => {
+        let pos = JSON.parse(rawData)["iss_position"];
+        cache = [+pos["latitude"], +pos["longitude"]];
+        cacheExpire = Date.now() + 10000;
+        callback(null, cache);
+      });
+    })
+    .on("error", (err) => {
+      callback(err);
     });
-  }).on("error", (err) => {
-    callback(err);
-  });
 }
 
 exports.latlong = latlong;

docs/index.rst

@@ -30,6 +30,7 @@ Welcome to GenieACS's documentation!
 
   provisions
   virtual-parameters
+  administration-faq
 
 .. raw:: latex
 

docs/installation-guide.rst

@@ -4,7 +4,7 @@ Installation Guide
 This guide is for installing GenieACS on a single server on any Linux distro
 that uses *systemd* as its init system.
 
-The various GenieACS services are independant of each other and may be
+The various GenieACS services are independent of each other and may be
 installed on different servers. You may also run multiple instances of each in
 a load-balancing/failover setup.
 
@@ -19,7 +19,7 @@ Prerequisites
 
 .. topic:: Node.js
 
-  GenieACS requires Node.js 10.x and up. Refer to https://nodejs.org/ for
+  GenieACS requires Node.js 12.13 and up. Refer to https://nodejs.org/ for
   instructions.
 
 .. topic:: MongoDB
@@ -68,12 +68,19 @@ Configure systemd
 
   .. code:: bash
 
-    GENIEACS_CWMP_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-cmwp-access.log
+    GENIEACS_CWMP_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-cwmp-access.log
     GENIEACS_NBI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-nbi-access.log
     GENIEACS_FS_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-fs-access.log
     GENIEACS_UI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-ui-access.log
     GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
-    GENIEACS_UI_JWT_SECRET=secret
+    NODE_OPTIONS=--enable-source-maps
+    GENIEACS_EXT_DIR=/opt/genieacs/ext
+
+  Generate a secure JWT secret and append to :file:`/opt/genieacs/genieacs.env`:
+
+  .. code:: bash
+
+    node -e "console.log(\"GENIEACS_UI_JWT_SECRET=\" + require('crypto').randomBytes(128).toString('hex'))" >> /opt/genieacs/genieacs.env
   
   Set file ownership and permissions:
 
@@ -82,7 +89,7 @@ Configure systemd
     sudo chown genieacs:genieacs /opt/genieacs/genieacs.env
     sudo chmod 600 /opt/genieacs/genieacs.env
 
-.. topic:: Create logs directory  
+.. topic:: Create logs directory
 
   .. code:: bash
     
@@ -100,11 +107,16 @@ Configure systemd
   :file:`/var/log/genieacs/` while process logs go to *journald*. Use
   ``journalctl`` command to view process logs.
 
+  .. attention::
+
+    If the command :command:`systemctl edit --force --full` fails, you can
+    create the unit file manually.
+
   1. Run the following command to create ``genieacs-cwmp`` service:
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-cwmp
+      sudo systemctl edit --force --full genieacs-cwmp
     
     Then paste the following in the editor and save:
 
@@ -117,8 +129,7 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-cwmp
-      KillMode=process
+      ExecStart=/usr/bin/genieacs-cwmp
 
       [Install]
       WantedBy=default.target
@@ -127,7 +138,7 @@ Configure systemd
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-nbi
+      sudo systemctl edit --force --full genieacs-nbi
     
     Then paste the following in the editor and save:
 
@@ -140,8 +151,7 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-nbi
-      KillMode=process
+      ExecStart=/usr/bin/genieacs-nbi
 
       [Install]
       WantedBy=default.target
@@ -150,7 +160,7 @@ Configure systemd
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-fs
+      sudo systemctl edit --force --full genieacs-fs
     
     Then paste the following in the editor and save:
 
@@ -163,8 +173,7 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-fs
-      KillMode=process
+      ExecStart=/usr/bin/genieacs-fs
 
       [Install]
       WantedBy=default.target
@@ -173,7 +182,7 @@ Configure systemd
   
     .. code:: bash
 
-      sudo systemctl edit --force genieacs-ui
+      sudo systemctl edit --force --full genieacs-ui
     
     Then paste the following in the editor and save:
 
@@ -186,19 +195,18 @@ Configure systemd
       [Service]
       User=genieacs
       EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=genieacs-ui
-      KillMode=process
+      ExecStart=/usr/bin/genieacs-ui
 
       [Install]
       WantedBy=default.target
 
 .. topic:: Configure log file rotation using logrotate
 
-  Save the following under :file:`/etc/logrotate.d/genieacs`
+  Save the following as :file:`/etc/logrotate.d/genieacs`
 
   .. code::
   
-    /var/log/genieacs/*.{log,yaml} {
+    /var/log/genieacs/*.log /var/log/genieacs/*.yaml {
         daily
         rotate 30
         compress

docs/provisions.rst

@@ -128,6 +128,14 @@ script filename while second argument is the function name within that script.
 Any remaining arguments will be passed to that function. See :ref:`extensions`
 for more details.
 
+``log(message)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Prints out a string in genieacs-cwmp's access log. It's meant to be used for
+debugging. Note that you may see multiple log entries as the script can be
+executed multiple times in a session. See :ref:`this FAQ
+<administration-faq-duplicate-log-entries>`.
+
 .. _path-format:
 
 Path format

docs/requirements.txt

@@ -0,0 +1 @@
+sphinx_rtd_theme==2.0.0

docs/virtual-parameters.rst

@@ -18,8 +18,8 @@ differences between the scripts of provisions and virtual parameters are:
 
 .. code:: javascript
 
-  // [<current attr timestamps>, <current attr values>, <declared attr timestamps, declared attr values>]
-  [{path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}, {path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}]
+  // [<declared attr timestamps, declared attr values>, <current attr timestamps>, <current attr values>]
+  [{path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}, {path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}]
 
 - Virtual parameter scripts must return an object containing the attributes of
   this parameter.
@@ -69,9 +69,9 @@ Expose an external value as a virtual parameter
 
   // Example: Expose an external value as a virtual parameter
   let serial = declare("DeviceID.SerialNumber", {value: 1});
-  if (values.value) {
-    ext("example-ext", "set", serial.value[0], values.value[0]);
-    return {writable: true, value: [values.value[0], "xsd:string"]};
+  if (args[1].value) {
+    ext("example-ext", "set", serial.value[0], args[1].value[0]);
+    return {writable: true, value: [args[1].value[0], "xsd:string"]};
   }
   else {
     let v = ext("example-ext", "get", serial.value[0]);
@@ -85,8 +85,8 @@ Create an editable virtual parameter for WPA passphrase
 
   // Example: Create an editable virtual parameter for WPA passphrase
   let m = "";
-  if (VALUES && VALUES.value) {
-    m = VALUES.value[0];
+  if (args[1].value) {
+    m = args[1].value[0];
     declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", null, {value: m});
     declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", null, {value: m});
   }

eslint.config.mjs

@@ -0,0 +1,77 @@
+import { defineConfig } from "@eslint/config-helpers";
+import eslint from "@eslint/js";
+import tseslint from "typescript-eslint";
+import eslintConfigPrettier from "eslint-config-prettier";
+import globals from "globals";
+
+export default defineConfig(
+  {
+    ignores: ["dist/", "test/*.js"],
+  },
+  eslint.configs.recommended,
+  ...tseslint.configs.recommended,
+  eslintConfigPrettier,
+  {
+    languageOptions: {
+      globals: {
+        ...globals.es2022,
+        ...globals.node,
+      },
+      parserOptions: {
+        projectService: true,
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+    rules: {
+      "@typescript-eslint/no-shadow": ["error", { allow: ["err"] }],
+      "handle-callback-err": "error",
+      "prefer-arrow-callback": "error",
+      "no-buffer-constructor": "error",
+      "prefer-const": ["error", { destructuring: "all" }],
+      eqeqeq: ["error", "always", { null: "ignore" }],
+      "@typescript-eslint/no-explicit-any": "off",
+      "@typescript-eslint/no-use-before-define": [
+        "error",
+        { functions: false },
+      ],
+      "@typescript-eslint/explicit-function-return-type": [
+        "error",
+        { allowExpressions: true },
+      ],
+      "@typescript-eslint/no-floating-promises": "error",
+      "@typescript-eslint/no-misused-promises": "error",
+      "no-prototype-builtins": "off",
+    },
+  },
+  {
+    files: ["ui/**/*.ts", "ui/**/*.tsx"],
+    languageOptions: {
+      globals: {
+        ...globals.browser,
+      },
+    },
+  },
+  {
+    files: ["seed/*.js"],
+    languageOptions: {
+      parserOptions: {
+        project: null,
+      },
+      globals: {
+        declare: "readonly",
+        clear: "readonly",
+        commit: "readonly",
+        ext: "readonly",
+        log: "readonly",
+        args: "readonly",
+      },
+    },
+    rules: {
+      "@typescript-eslint/explicit-function-return-type": "off",
+      "@typescript-eslint/no-floating-promises": "off",
+      "@typescript-eslint/no-misused-promises": "off",
+      "@typescript-eslint/no-shadow": "off",
+      "no-shadow": ["error", { allow: ["err", "total"] }],
+    },
+  },
+);

lib/api-functions.ts

@@ -1,177 +1,249 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { parse } from "url";
-import * as db from "./db";
-import * as common from "./common";
-import * as cache from "./cache";
+import { ObjectId } from "mongodb";
+import { collections } from "./db/db.ts";
 import {
-  getCurrentSnapshot,
+  deleteConfig,
+  deleteFault as dbDeleteFault,
+  deleteFile,
+  deletePermission,
+  deletePreset,
+  deleteProvision,
+  deleteTask,
+  deleteUser,
+  deleteVirtualParameter,
+  putConfig,
+  putPermission,
+  putPreset,
+  putProvision,
+  putUser,
+  putVirtualParameter,
+} from "./ui/db.ts";
+import * as common from "./util.ts";
+import * as cache from "./cache.ts";
+import { acquireLock, getToken, releaseLock } from "./lock.ts";
+import {
+  getRevision,
   getConfig,
-  getConfigExpression
-} from "./local-cache";
+  getConfigExpression,
+  getUsers,
+} from "./ui/local-cache.ts";
 import {
   httpConnectionRequest,
-  udpConnectionRequest
-} from "./connection-request";
-import { Expression, Task } from "./types";
+  udpConnectionRequest,
+  xmppConnectionRequest,
+} from "./connection-request.ts";
+import { Task } from "./types.ts";
+import Expression, { Value } from "./common/expression.ts";
+import { hashPassword } from "./auth.ts";
+import { flattenDevice } from "./ui/db.ts";
+import { ResourceLockedError } from "./common/errors.ts";
+import * as config from "../lib/config.ts";
 
-export async function connectionRequest(deviceId): Promise<void> {
-  const options = {
-    projection: {
-      _deviceId: 1,
-      "Device.ManagementServer.ConnectionRequestURL._value": 1,
-      "Device.ManagementServer.UDPConnectionRequestAddress._value": 1,
-      "Device.ManagementServer.ConnectionRequestUsername._value": 1,
-      "Device.ManagementServer.ConnectionRequestPassword._value": 1,
-      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value": 1,
-      "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress._value": 1,
-      "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername._value": 1,
-      "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword._value": 1
-    }
-  };
+const XMPP_CONFIGURED = !!config.get("XMPP_JID");
 
-  const device = await db.devicesCollection.findOne({ _id: deviceId }, options);
-  if (!device) throw new Error("No such device");
+export async function connectionRequest(
+  deviceId: string,
+  device?: Record<string, Value>,
+): Promise<string> {
+  if (!device) {
+    const res = await collections.devices.findOne({ _id: deviceId });
+    if (!res) throw new Error("No such device");
+    device = flattenDevice(res);
+  }
 
-  let managementServer,
-    connectionRequestUrl,
+  let connectionRequestUrl,
     udpConnectionRequestAddress,
+    stunEnable,
+    connReqJabberId,
     username,
     password;
-  if (device.Device)
-    // TR-181 data model
-    managementServer = device.Device.ManagementServer;
-  // TR-098 data model
-  else managementServer = device.InternetGatewayDevice.ManagementServer;
 
-  if (managementServer.ConnectionRequestURL)
-    connectionRequestUrl = managementServer.ConnectionRequestURL._value;
-  if (managementServer.UDPConnectionRequestAddress) {
+  if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
+    connectionRequestUrl =
+      device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"] ||
+      "";
     udpConnectionRequestAddress =
-      managementServer.UDPConnectionRequestAddress._value;
+      device[
+        "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
+      ] || "";
+    stunEnable =
+      device["InternetGatewayDevice.ManagementServer.STUNEnable"] || "";
+    connReqJabberId =
+      device["InternetGatewayDevice.ManagementServer.ConnReqJabberID"] || "";
+    username =
+      device[
+        "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
+      ] || "";
+    password =
+      device[
+        "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword"
+      ] || "";
+  } else {
+    connectionRequestUrl =
+      device["Device.ManagementServer.ConnectionRequestURL"] || "";
+    udpConnectionRequestAddress =
+      device["Device.ManagementServer.UDPConnectionRequestAddress"] || "";
+    stunEnable = device["Device.ManagementServer.STUNEnable"] || "";
+    connReqJabberId = device["Device.ManagementServer.ConnReqJabberID"] || "";
+    username =
+      device["Device.ManagementServer.ConnectionRequestUsername"] || "";
+    password =
+      device["Device.ManagementServer.ConnectionRequestPassword"] || "";
+  }
+  let remoteAddress;
+  try {
+    remoteAddress = new URL(connectionRequestUrl).hostname;
+  } catch {
+    return "Invalid connection request URL";
   }
-  if (managementServer.ConnectionRequestUsername)
-    username = managementServer.ConnectionRequestUsername._value;
-  if (managementServer.ConnectionRequestPassword)
-    password = managementServer.ConnectionRequestPassword._value;
 
-  const context = {
-    id: device["_id"],
-    serialNumber: device["_deviceId"]["SerialNumber"],
-    productClass: device["_deviceId"]["ProductClass"],
-    oui: device["_deviceId"]["OUI"],
-    remoteAddress: connectionRequestUrl
-      ? parse(connectionRequestUrl).host
-      : null,
-    username: username || "",
-    password: password || ""
+  const snapshot = await getRevision();
+  const now = Date.now();
+
+  const evalCallback = (exp: Expression): Expression => {
+    if (exp instanceof Expression.Parameter) {
+      let name = exp.path.toString();
+      if (name === "id") name = "DeviceID.ID";
+      else if (name === "serialNumber") name = "DeviceID.SerialNumber";
+      else if (name === "productClass") name = "DeviceID.ProductClass";
+      else if (name === "oui") name = "DeviceID.OUI";
+      else if (name === "remoteAddress")
+        return new Expression.Literal(remoteAddress);
+      else if (name === "username") return new Expression.Literal(username);
+      else if (name === "password") return new Expression.Literal(password);
+      return new Expression.Literal(device[name] ?? null);
+    } else if (exp instanceof Expression.FunctionCall) {
+      if (exp.name === "NOW") return new Expression.Literal(now);
+      else if (exp.name === "REMOTE_ADDRESS")
+        return new Expression.Literal(remoteAddress);
+      else if (exp.name === "USERNAME") return new Expression.Literal(username);
+      else if (exp.name === "PASSWORD") return new Expression.Literal(password);
+    }
+    return exp;
   };
 
-  const snapshot = await getCurrentSnapshot();
-  const now = Date.now();
-  const UDP_CONNECTION_REQUEST_PORT = +getConfig(
+  const configCallback = (exp: Expression): Expression.Literal => {
+    const e = evalCallback(exp);
+    if (e instanceof Expression.Literal) return e;
+    return new Expression.Literal(null);
+  };
+
+  const UDP_CONNECTION_REQUEST_PORT = getConfig(
     snapshot,
-    "cwmp.udpConnectionRequestAuth",
-    context,
-    now
+    "cwmp.udpConnectionRequestPort",
+    0,
+    configCallback,
   );
-  const CONNECTION_REQUEST_TIMEOUT = +getConfig(
+  const CONNECTION_REQUEST_TIMEOUT = getConfig(
     snapshot,
     "cwmp.connectionRequestTimeout",
-    context,
-    now
+    2000,
+    configCallback,
   );
-  const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = !!getConfig(
+  const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = getConfig(
     snapshot,
     "cwmp.connectionRequestAllowBasicAuth",
-    context,
-    now
+    false,
+    configCallback,
   );
   let authExp: Expression = getConfigExpression(
     snapshot,
-    "cwmp.connectionRequestAuth"
+    "cwmp.connectionRequestAuth",
   );
 
-  if (authExp === undefined) {
-    authExp = [
-      "FUNC",
-      "AUTH",
-      ["PARAM", "username"],
-      ["PARAM", "password"]
-    ] as Expression;
+  if (!authExp) {
+    authExp = new Expression.FunctionCall("AUTH", [
+      new Expression.FunctionCall("USERNAME", []),
+      new Expression.FunctionCall("PASSWORD", []),
+    ]);
   }
 
-  const debug = !!getConfig(snapshot, "cwmp.debug", context, now);
+  authExp = authExp.evaluate(evalCallback);
 
-  let udpProm;
-  if (udpConnectionRequestAddress) {
-    udpProm = udpConnectionRequest(
-      udpConnectionRequestAddress,
+  const debug = getConfig(snapshot, "cwmp.debug", false, configCallback);
+
+  let udpProm = Promise.resolve(false);
+  if (udpConnectionRequestAddress && +stunEnable) {
+    try {
+      const u = new URL("udp://" + udpConnectionRequestAddress);
+      udpProm = udpConnectionRequest(
+        u.hostname,
+        parseInt(u.port || "80"),
+        authExp,
+        UDP_CONNECTION_REQUEST_PORT,
+        debug,
+        deviceId,
+      ).then(
+        () => true,
+        () => false,
+      );
+    } catch {
+      // Ignore invalid address
+    }
+  }
+
+  let status;
+
+  if (connReqJabberId && XMPP_CONFIGURED) {
+    status = await xmppConnectionRequest(
+      connReqJabberId,
       authExp,
-      context,
-      UDP_CONNECTION_REQUEST_PORT,
+      CONNECTION_REQUEST_TIMEOUT,
       debug,
-      deviceId
+      deviceId,
     );
-  }
-
-  try {
-    await httpConnectionRequest(
+  } else {
+    status = await httpConnectionRequest(
       connectionRequestUrl,
       authExp,
-      context,
       CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
       CONNECTION_REQUEST_TIMEOUT,
       debug,
-      deviceId
+      deviceId,
     );
-  } catch (err) {
-    if (!udpProm) throw err;
-    await udpProm;
   }
+
+  if (await udpProm) return "";
+
+  return status;
 }
 
-export async function watchTask(deviceId, taskId, timeout): Promise<string> {
-  await new Promise(resolve => setTimeout(resolve, 500));
-
-  const task = await db.tasksCollection.findOne(
-    { _id: taskId },
-    { projection: { _id: 1 } }
+export async function awaitSessionStart(
+  deviceId: string,
+  lastInform: number,
+  timeout: number,
+): Promise<boolean> {
+  const now = Date.now();
+  const device = await collections.devices.findOne(
+    { _id: deviceId },
+    { projection: { _lastInform: 1 } },
   );
-  if (!task) return "completed";
+  const li = (device["_lastInform"] as Date).getTime();
+  if (li > lastInform) return true;
+  const token = await getToken(`cwmp_session_${deviceId}`);
+  if (token?.startsWith("cwmp_session_")) return true;
+  if (timeout < 500) return false;
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionStart(deviceId, lastInform, timeout);
+}
 
-  const q = { _id: `${deviceId}:task_${taskId}` };
-  const fault = await db.faultsCollection.findOne(q, {
-    projection: { _id: 1 }
-  });
-  if (fault) return "fault";
-
-  if ((timeout -= 500) <= 0) return "timeout";
-
-  return watchTask(deviceId, taskId, timeout);
+export async function awaitSessionEnd(
+  deviceId: string,
+  timeout: number,
+): Promise<boolean> {
+  const now = Date.now();
+  const token = await getToken(`cwmp_session_${deviceId}`);
+  if (!token?.startsWith("cwmp_session_")) return true;
+  if (timeout < 500) return false;
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionEnd(deviceId, timeout);
 }
 
 function sanitizeTask(task): void {
   task.timestamp = new Date(task.timestamp || Date.now());
   if (task.expiry) {
-    if (common.typeOf(task.expiry) === common.DATE_TYPE || isNaN(task.expiry))
+    if (task.expiry instanceof Date || isNaN(task.expiry))
       task.expiry = new Date(task.expiry);
     else task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000);
   }
@@ -209,6 +281,10 @@ function sanitizeTask(task): void {
       break;
 
     case "refreshObject":
+      if (typeof task.objectName !== "string")
+        throw new Error("Missing 'objectName' property");
+      break;
+
     case "deleteObject":
       if (typeof task.objectName !== "string" || !task.objectName.length)
         throw new Error("Missing 'objectName' property");
@@ -226,11 +302,14 @@ function sanitizeTask(task): void {
       break;
 
     case "download":
-      if (typeof task.fileType !== "string" || !task.fileType.length)
-        throw new Error("Missing 'fileType' property");
+      // genieacs-gui sends file ID instead of fileName and fileType
+      if (!task.file) {
+        if (typeof task.fileType !== "string" || !task.fileType.length)
+          throw new Error("Missing 'fileType' property");
 
-      if (typeof task.fileName !== "string" || !task.fileName.length)
-        throw new Error("Missing 'fileName' property");
+        if (typeof task.fileName !== "string" || !task.fileName.length)
+          throw new Error("Missing 'fileName' property");
+      }
 
       if (
         task.targetFileName != null &&
@@ -238,42 +317,174 @@ function sanitizeTask(task): void {
       )
         throw new Error("Invalid 'targetFileName' property");
       break;
+
+    case "provisions":
+      if (
+        !Array.isArray(task.provisions) ||
+        !task.provisions.every((arr) =>
+          arr.every(
+            (s) =>
+              s == null || ["boolean", "number", "string"].includes(typeof s),
+          ),
+        )
+      )
+        throw new Error("Invalid 'provisions' property");
+      break;
+
+    case "reboot":
+      break;
+
+    case "factoryReset":
+      break;
+
+    default:
+      throw new Error("Invalid task name");
   }
 
   return task;
 }
 
-export async function insertTasks(tasks): Promise<Task[]> {
-  if (tasks && common.typeOf(tasks) !== common.ARRAY_TYPE) tasks = [tasks];
-  else if (!tasks || tasks.length === 0) return tasks || [];
+export async function insertTasks(tasks: any[]): Promise<Task[]> {
+  if (tasks && !Array.isArray(tasks)) tasks = [tasks];
+  else if (!tasks?.length) return tasks || [];
 
   for (const task of tasks) {
     sanitizeTask(task);
     if (task.uniqueKey) {
-      await db.tasksCollection.deleteOne({
+      await collections.tasks.deleteOne({
         device: task.device,
-        uniqueKey: task.uniqueKey
+        uniqueKey: task.uniqueKey,
       });
     }
   }
-  await db.tasksCollection.insertMany(tasks);
+  await collections.tasks.insertMany(tasks);
+  for (const task of tasks) task._id = task._id.toString();
   return tasks;
 }
 
-export async function deleteDevice(deviceId): Promise<void> {
-  await Promise.all([
-    db.tasksCollection.deleteMany({ device: deviceId }),
-    db.devicesCollection.deleteOne({ _id: deviceId }),
-    db.faultsCollection.deleteMany({
-      _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`
-      }
-    }),
-    db.operationsCollection.deleteMany({
-      _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`
-      }
-    }),
-    cache.del(`${deviceId}_tasks_faults_operations`)
-  ]);
+export async function deleteDevice(deviceId: string): Promise<void> {
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+  if (!token) throw new ResourceLockedError("Device is in session");
+  try {
+    await Promise.all([
+      collections.tasks.deleteMany({ device: deviceId }),
+      collections.devices.deleteOne({ _id: deviceId }),
+      collections.faults.deleteMany({
+        _id: {
+          $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+        },
+      }),
+      collections.operations.deleteMany({
+        _id: {
+          $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+        },
+      }),
+    ]);
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
+}
+
+export async function deleteFault(id: string): Promise<void> {
+  const deviceId = id.split(":", 1)[0];
+  const channel = id.slice(deviceId.length + 1);
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000);
+  if (!token) throw new ResourceLockedError("Device is in session");
+  try {
+    const proms = [dbDeleteFault(id)];
+    if (channel.startsWith("task_"))
+      proms.push(deleteTask(new ObjectId(channel.slice(5))));
+    await Promise.all(proms);
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
+}
+
+export async function deleteResource(
+  resource: string,
+  id: string,
+): Promise<void> {
+  if (resource === "devices") {
+    await deleteDevice(id);
+  } else if (resource === "files") {
+    await deleteFile(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "faults") {
+    await deleteFault(id);
+  } else if (resource === "provisions") {
+    await deleteProvision(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "presets") {
+    await deletePreset(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "virtualParameters") {
+    await deleteVirtualParameter(id);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "config") {
+    await deleteConfig(id);
+    await Promise.all([
+      cache.del("ui-local-cache-hash"),
+      cache.del("cwmp-local-cache-hash"),
+    ]);
+  } else if (resource === "permissions") {
+    await deletePermission(id);
+    await cache.del("ui-local-cache-hash");
+  } else if (resource === "users") {
+    await deleteUser(id);
+    await cache.del("ui-local-cache-hash");
+  } else {
+    throw new Error(`Unknown resource ${resource}`);
+  }
+}
+
+// TODO Implement validation
+export async function putResource(
+  resource: string,
+  id: string,
+  data: any,
+): Promise<void> {
+  if (resource === "presets") {
+    await putPreset(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "provisions") {
+    await putProvision(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "virtualParameters") {
+    await putVirtualParameter(id, data);
+    await cache.del("cwmp-local-cache-hash");
+  } else if (resource === "config") {
+    await putConfig(id, data);
+    await Promise.all([
+      cache.del("ui-local-cache-hash"),
+      cache.del("cwmp-local-cache-hash"),
+    ]);
+  } else if (resource === "permissions") {
+    await putPermission(id, data);
+    await cache.del("ui-local-cache-hash");
+  } else if (resource === "users") {
+    delete data["password"];
+    delete data["salt"];
+    await putUser(id, data);
+    await cache.del("ui-local-cache-hash");
+  } else {
+    throw new Error(`Unknown resource ${resource}`);
+  }
+}
+
+export function authLocal(
+  snapshot: string,
+  username: string,
+  password: string,
+): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    const users = getUsers(snapshot);
+    const user = users[username];
+    if (!user?.password) return void resolve(null);
+    hashPassword(password, user.salt)
+      .then((hash) => {
+        if (hash === user.password) resolve(true);
+        else resolve(false);
+      })
+      .catch(reject);
+  });
 }

lib/auth.ts

@@ -1,33 +1,16 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
+import { createHash, randomBytes, pbkdf2 } from "node:crypto";
 
-import { createHash, randomBytes, pbkdf2 } from "crypto";
-
-function parseHeaderFeilds(str: string): {} {
+function parseHeaderFeilds(str: string): Record<string, string> {
   const res = {};
   const parts = str.split(",");
 
-  let part;
+  let part: string;
   while ((part = parts.shift()) != null) {
     const name = part.split("=", 1)[0];
-    if (name.length === part.length)
+    if (name.length === part.length) {
+      if (!part.trim()) continue;
       throw new Error("Unable to parse auth header");
+    }
 
     let value = part.slice(name.length + 1);
     if (!/^\s*"/.test(value)) {
@@ -42,7 +25,7 @@ function parseHeaderFeilds(str: string): {} {
       try {
         value = JSON.parse(value);
       } catch (error) {
-        throw new Error("Unable to parse auth header");
+        throw new Error("Unable to parse auth header", { cause: error });
       }
     }
     res[name.trim()] = value;
@@ -50,7 +33,9 @@ function parseHeaderFeilds(str: string): {} {
   return res;
 }
 
-export function parseAuthorizationHeader(authHeader): { method: string } {
+export function parseAuthorizationHeader(authHeader: string): {
+  method: string;
+} {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };
@@ -59,7 +44,7 @@ export function parseAuthorizationHeader(authHeader): { method: string } {
     // Inspired by https://github.com/jshttp/basic-auth
     const USER_PASS_REGEX = /^([^:]*):(.*)$/;
     const creds = USER_PASS_REGEX.exec(
-      Buffer.from(authHeader.slice(method.length + 1), "base64").toString()
+      Buffer.from(authHeader.slice(method.length + 1), "base64").toString(),
     );
 
     if (!creds) throw new Error("Unable to parse auth header");
@@ -72,7 +57,9 @@ export function parseAuthorizationHeader(authHeader): { method: string } {
   return res;
 }
 
-export function parseWwwAuthenticateHeader(authHeader): {} {
+export function parseWwwAuthenticateHeader(
+  authHeader: string,
+): Record<string, string> {
   authHeader = authHeader.trim();
   const method = authHeader.split(" ", 1)[0];
   const res = { method: method };
@@ -80,7 +67,7 @@ export function parseWwwAuthenticateHeader(authHeader): {} {
   return res;
 }
 
-export function basic(username, password): string {
+export function basic(username: string, password: string): string {
   return "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
 }
 
@@ -94,23 +81,15 @@ export function digest(
   qop?: string | Buffer,
   body?: string | Buffer,
   cnonce?: string | Buffer,
-  nc?: string | Buffer
+  nc?: string | Buffer,
 ): string {
   const ha1 = createHash("md5");
-  ha1
-    .update(username)
-    .update(":")
-    .update(realm)
-    .update(":")
-    .update(password);
+  ha1.update(username).update(":").update(realm).update(":").update(password);
   // TODO support "MD5-sess" algorithm directive
   const ha1d = ha1.digest("hex");
 
   const ha2 = createHash("md5");
-  ha2
-    .update(httpMethod)
-    .update(":")
-    .update(uri);
+  ha2.update(httpMethod).update(":").update(uri);
 
   if (qop === "auth-int") {
     const bodyHash = createHash("md5")
@@ -122,10 +101,7 @@ export function digest(
   const ha2d = ha2.digest("hex");
 
   const hash = createHash("md5");
-  hash
-    .update(ha1d)
-    .update(":")
-    .update(nonce);
+  hash.update(ha1d).update(":").update(nonce);
   if (qop) {
     hash
       .update(":")
@@ -146,7 +122,7 @@ export function solveDigest(
   uri: string | Buffer,
   httpMethod: string | Buffer,
   body: string | Buffer,
-  authHeader
+  authHeader: Record<string, string>,
 ): string {
   const cnonce = randomBytes(8).toString("hex");
   const nc = "00000001";
@@ -168,7 +144,7 @@ export function solveDigest(
     qop,
     body,
     cnonce,
-    nc
+    nc,
   );
 
   let authString = `Digest username="${username}"`;
@@ -183,7 +159,7 @@ export function solveDigest(
   return authString;
 }
 
-export function generateSalt(length): Promise<string> {
+export function generateSalt(length: number): Promise<string> {
   return new Promise((resolve, reject) => {
     randomBytes(length, (err, rand) => {
       if (err) return void reject(err);
@@ -192,7 +168,7 @@ export function generateSalt(length): Promise<string> {
   });
 }
 
-export function hashPassword(pass, salt): Promise<string> {
+export function hashPassword(pass: string, salt: string): Promise<string> {
   return new Promise((resolve, reject) => {
     pbkdf2(pass, salt, 10000, 128, "sha512", (err, hash) => {
       if (err) return void reject(err);

lib/cache.ts

@@ -1,140 +1,35 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { MongoClient, Collection } from "mongodb";
-import * as config from "./config";
+import { collections } from "./db/db.ts";
+import * as config from "./config.ts";
 
+const CLOCK_SKEW_TOLERANCE = 30000;
 const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
 
-let mongoClient: MongoClient;
-let mongoCollection: Collection;
-let mongoTimeOffset = 0;
-
-export async function connect(): Promise<void> {
-  const MONGODB_CONNECTION_URL = "" + config.get("MONGODB_CONNECTION_URL");
-  mongoClient = await MongoClient.connect(MONGODB_CONNECTION_URL, {
-    useNewUrlParser: true
-  });
-  const db = mongoClient.db();
-  mongoCollection = db.collection("cache");
-  await mongoCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
-  const now = Date.now();
-  const res = await db.command({ hostInfo: 1 });
-  mongoTimeOffset = res.system.currentTime.getTime() - now;
+export async function get(key: string): Promise<string> {
+  const res = await collections.cache.findOne({ _id: key });
+  return res?.value;
 }
 
-export async function disconnect(): Promise<void> {
-  if (mongoClient) await mongoClient.close();
-}
-
-export async function get(key): Promise<any> {
-  const expire = new Date(Date.now() - mongoTimeOffset);
-  if (Array.isArray(key)) {
-    const res = await mongoCollection.find({ _id: { $in: key } }).toArray();
-
-    const indices = {};
-    key.forEach((v, i) => {
-      indices[v] = i;
-    });
-
-    const values = [];
-    res.forEach(r => {
-      if (r["expire"] > expire) values[indices[r["_id"]]] = r["value"];
-    });
-
-    return values;
-  } else {
-    const res = await mongoCollection.findOne({ _id: { $in: [key] } });
-    if (res && res["expire"] > expire) return res["value"];
-    return res;
-  }
-}
-
-export async function del(key): Promise<void> {
-  if (Array.isArray(key))
-    await mongoCollection.deleteMany({ _id: { $in: key } });
-  else await mongoCollection.deleteOne({ _id: key });
+export async function del(key: string): Promise<void> {
+  await collections.cache.deleteOne({ _id: key });
 }
 
 export async function set(
   key: string,
-  value: string | number,
-  ttl: number = MAX_CACHE_TTL
+  value: string,
+  ttl: number = MAX_CACHE_TTL,
 ): Promise<void> {
-  const expire = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
-  await mongoCollection.replaceOne(
+  const timestamp = new Date();
+  const expire = new Date(
+    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000,
+  );
+  await collections.cache.replaceOne(
     { _id: key },
-    { _id: key, value: value, expire: expire },
-    { upsert: true }
+    { value, expire, timestamp },
+    { upsert: true },
   );
 }
 
-export async function pop(key): Promise<any> {
-  const res = await mongoCollection.findOneAndDelete({ _id: key });
-
-  if (
-    res &&
-    res["value"] &&
-    +res["value"]["expire"] - (Date.now() - mongoTimeOffset)
-  )
-    return res["value"]["value"];
-
-  return null;
-}
-
-export async function lock(lockName, ttl): Promise<Function> {
-  const token = Math.random()
-    .toString(36)
-    .slice(2);
-
-  async function unlockOrExtend(extendTtl): Promise<void> {
-    if (!extendTtl) {
-      const res = await mongoCollection.deleteOne({
-        _id: lockName,
-        value: token
-      });
-      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
-    } else {
-      const expire = new Date(Date.now() - mongoTimeOffset + extendTtl * 1000);
-      const res = await mongoCollection.updateOne(
-        { _id: lockName, value: token },
-        { expire: expire }
-      );
-      if (res["result"]["n"] !== 1) throw new Error("Lock expired");
-    }
-  }
-
-  const expireTest = new Date(Date.now() - mongoTimeOffset);
-  const expireSet = new Date(Date.now() - mongoTimeOffset + ttl * 1000);
-
-  try {
-    await mongoCollection.updateOne(
-      { _id: lockName, expire: { $lte: expireTest } },
-      { $set: { value: token, expire: expireSet } },
-      { upsert: true }
-    );
-  } catch (err) {
-    if (err && err.code === 11000) {
-      await new Promise(resolve => setTimeout(resolve, 200));
-      return lock(lockName, ttl);
-    }
-  }
-
-  return unlockOrExtend;
+export async function pop(key: string): Promise<string> {
+  const res = await collections.cache.findOneAndDelete({ _id: key });
+  return res.value?.value;
 }

lib/cluster.ts

@@ -1,35 +1,28 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import cluster from "cluster";
-import { cpus } from "os";
-import * as logger from "./logger";
+import cluster, { Worker } from "node:cluster";
+import { cpus } from "node:os";
+import * as logger from "./logger.ts";
 
 let respawnTimestamp = 0;
-let tooManyCrashesTimestamp = 0;
+let crashes: number[] = [];
+
+function fork(): Worker {
+  const w = cluster.fork();
+  w.on("error", (err: NodeJS.ErrnoException) => {
+    // Avoid exception when attempting to kill the process just as it's exiting
+    if (err.code !== "EPIPE") throw err;
+    setTimeout(() => {
+      if (!w.isDead()) throw err;
+    }, 50);
+  });
+  return w;
+}
 
 function restartWorker(worker, code, signal): void {
   const msg = {
     message: "Worker died",
     pid: worker.process.pid,
     exitCode: null,
-    signal: null
+    signal: null,
   };
 
   if (code != null) msg.exitCode = code;
@@ -39,32 +32,49 @@ function restartWorker(worker, code, signal): void {
   logger.error(msg);
 
   const now = Date.now();
-  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
-  if (respawnTimestamp === now) {
-    tooManyCrashesTimestamp = now;
-    cluster.fork();
-    return;
-  }
+  crashes.push(now);
 
-  if (now - tooManyCrashesTimestamp > 60000) {
+  let min1 = 0,
+    min2 = 0,
+    min3 = 0;
+
+  crashes = crashes.filter((n) => {
+    if (n > now - 60000) ++min1;
+    else if (n > now - 120000) ++min2;
+    else if (n > now - 180000) ++min3;
+    else return false;
+    return true;
+  });
+
+  if (min1 > 5 && min2 > 5 && min3 > 5) {
     process.exitCode = 1;
     cluster.removeListener("exit", restartWorker);
     for (const pid in cluster.workers) cluster.workers[pid].kill();
 
     logger.error({
       message: "Too many crashes, exiting",
-      pid: process.pid
+      pid: process.pid,
     });
     return;
   }
 
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    fork();
+    return;
+  }
+
   setTimeout(() => {
     if (process.exitCode) return;
-    cluster.fork();
+    fork();
   }, respawnTimestamp - now);
 }
 
-export function start(workerCount, servicePort, serviceAddress): void {
+export function start(
+  workerCount: number,
+  servicePort: number,
+  serviceAddress: string,
+): void {
   cluster.on("listening", (worker, address) => {
     if (
       (address.addressType === 4 || address.addressType === 6) &&
@@ -75,7 +85,7 @@ export function start(workerCount, servicePort, serviceAddress): void {
         message: "Worker listening",
         pid: worker.process.pid,
         address: address.address,
-        port: address.port
+        port: address.port,
       });
     }
   });
@@ -84,7 +94,7 @@ export function start(workerCount, servicePort, serviceAddress): void {
 
   if (!workerCount) workerCount = Math.max(2, cpus().length);
 
-  for (let i = 0; i < workerCount; ++i) cluster.fork();
+  for (let i = 0; i < workerCount; ++i) fork();
 }
 
 export function stop(): void {

lib/common.ts

@@ -1,59 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-export const UNDEFINED_TYPE = "[object Undefined]";
-export const NULL_TYPE = "[object Null]";
-export const BOOLEAN_TYPE = "[object Boolean]";
-export const NUMBER_TYPE = "[object Number]";
-export const STRING_TYPE = "[object String]";
-export const ARRAY_TYPE = "[object Array]";
-export const OBJECT_TYPE = "[object Object]";
-export const REGEXP_TYPE = "[object RegExp]";
-export const DATE_TYPE = "[object Date]";
-
-export const typeOf = (obj): string => Object.prototype.toString.call(obj);
-
-export function generateDeviceId(deviceIdStruct): string {
-  // Escapes everything except alphanumerics and underscore
-  function esc(str): string {
-    return str.replace(/[^A-Za-z0-9_]/g, chr => {
-      const buf = Buffer.from(chr, "utf8");
-      let rep = "";
-      for (const b of buf) rep += "%" + b.toString(16).toUpperCase();
-      return rep;
-    });
-  }
-
-  // Guaranteeing globally unique id as defined in TR-069
-  if (deviceIdStruct["ProductClass"]) {
-    return (
-      esc(deviceIdStruct["OUI"]) +
-      "-" +
-      esc(deviceIdStruct["ProductClass"]) +
-      "-" +
-      esc(deviceIdStruct["SerialNumber"])
-    );
-  }
-  return esc(deviceIdStruct["OUI"]) + "-" + esc(deviceIdStruct["SerialNumber"]);
-}
-
-// Source: http://stackoverflow.com/a/6969486
-export function escapeRegExp(str): string {
-  return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
-}

lib/common/authorizer.ts

@@ -1,40 +1,23 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { PermissionSet, Expression } from "../types";
-import { evaluate } from "./expression";
+import { PermissionSet } from "../types.ts";
+import Expression from "./expression.ts";
 
 export default class Authorizer {
-  private permissionSets: PermissionSet[];
-  private validatorCache: WeakMap<
-    object,
+  declare private permissionSets: PermissionSet[];
+  declare private validatorCache: WeakMap<
+    any,
     (mutationType, mutation, any) => boolean
   >;
-  private hasAccessCache: Map<string, boolean>;
+  declare private hasAccessCache: Map<string, boolean>;
+  declare private getFilterCache: Map<string, Expression>;
 
-  public constructor(permissionSets) {
+  public constructor(permissionSets: PermissionSet[]) {
     this.permissionSets = permissionSets;
     this.validatorCache = new WeakMap();
     this.hasAccessCache = new Map();
+    this.getFilterCache = new Map();
   }
 
-  public hasAccess(resourceType, access): boolean {
+  public hasAccess(resourceType: string, access: number): boolean {
     const cacheKey = `${resourceType}-${access}`;
     if (this.hasAccessCache.has(cacheKey))
       return this.hasAccessCache.get(cacheKey);
@@ -55,14 +38,33 @@ export default class Authorizer {
     return has;
   }
 
+  public getFilter(resourceType: string, access: number): Expression {
+    const cacheKey = `${resourceType}-${access}`;
+    if (this.getFilterCache.has(cacheKey))
+      return this.getFilterCache.get(cacheKey);
+
+    let filter: Expression = new Expression.Literal(false);
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (perm[resourceType]) {
+          if (perm[resourceType].access >= access)
+            filter = Expression.or(filter, perm[resourceType].filter);
+        }
+      }
+    }
+
+    this.getFilterCache.set(cacheKey, filter);
+    return filter;
+  }
+
   public getValidator(
-    resourceType,
-    resource
-  ): (mutationType: string, mutation: any, args: any) => boolean {
+    resourceType: string,
+    resource: unknown,
+  ): (mutationType: string, mutation?: any, args?: any) => boolean {
     if (this.validatorCache.has(resource))
       return this.validatorCache.get(resource);
 
-    const validators: Expression[] = [];
+    let validators: Expression = new Expression.Literal(false);
 
     for (const permissionSet of this.permissionSets) {
       for (const perm of permissionSet) {
@@ -71,53 +73,57 @@ export default class Authorizer {
           perm[resourceType].access >= 3 &&
           perm[resourceType].validate
         )
-          validators.push(perm[resourceType].validate);
+          validators = Expression.or(validators, perm[resourceType].validate);
       }
     }
 
     const validator = (
       mutationType: string,
       mutation: any,
-      any: any
+      any: any,
     ): boolean => {
-      if (!validators.length) return false;
-
       const object = {
         mutationType,
         mutation,
         resourceType,
         object: resource,
-        options: any
+        options: any,
       };
 
-      const valueFunction = (paramName): any => {
-        const entry = paramName.split(".", 1)[0];
-        paramName = paramName.slice(entry.length + 1);
-        let value = null;
-        if (["mutation", "options"].includes(entry)) {
-          value = object[entry];
-          for (const seg of paramName.split(".")) {
-            // typeof null is "object"
-            if (value != null && typeof value !== "object") value = null;
-            else value = value[seg];
-            if (value == null) break;
+      const now = Date.now();
+      const res = validators.evaluate((exp) => {
+        if (exp instanceof Expression.Literal) return exp;
+        if (exp instanceof Expression.Parameter) {
+          if (exp.path.colon) return new Expression.Literal(null);
+          const entry = exp.path.segments[0] as string;
+          const paramName = exp.path.slice(1);
+          let value = null;
+          if (["mutation", "options"].includes(entry)) {
+            value = object[entry];
+            for (const seg of paramName.segments) {
+              if (value == null) break;
+              if (typeof value !== "object") value = null;
+              else value = value[seg as string];
+            }
+          } else if (object[entry]) {
+            if (paramName.length) value = object[entry][paramName.toString()];
+            else value = object[entry];
           }
-        } else if (object[entry]) {
-          value = object[entry][paramName];
+          return new Expression.Literal(value);
+        } else if (exp instanceof Expression.FunctionCall) {
+          if (exp.name === "NOW") return new Expression.Literal(now);
         }
+        return new Expression.Literal(null);
+      }).value;
 
-        return value;
-      };
-
-      const res = evaluate(
-        validators.length > 1 ? ["OR", validators] : validators[0],
-        valueFunction,
-        Date.now()
-      );
-      return !Array.isArray(res) && !!res;
+      return !!res;
     };
 
     this.validatorCache.set(resource, validator);
     return validator;
   }
+
+  public getPermissionSets(): PermissionSet[] {
+    return this.permissionSets;
+  }
 }

lib/common/debounce.ts

@@ -0,0 +1,16 @@
+export default function debounce<T>(
+  func: (args: T[]) => void,
+  timeout: number,
+): (arg: T) => void {
+  let timer: ReturnType<typeof setTimeout>;
+  let args: T[] = [];
+  return (arg: T) => {
+    args.push(arg);
+    clearTimeout(timer);
+    timer = setTimeout(() => {
+      const argscopy = args;
+      args = [];
+      func(argscopy);
+    }, timeout);
+  };
+}

lib/common/errors.ts

@@ -0,0 +1,6 @@
+export class ResourceLockedError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ResourceLockedError";
+  }
+}

lib/common/expression-cnf.ts

@@ -1,285 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { map, parseLikePattern } from "./expression-parser";
-import { Expression } from "../types";
-
-type CNF = number[][];
-
-function or(cnf1: CNF, cnf2: CNF): CNF {
-  if (cnf1.length === 0) return cnf2;
-  else if (cnf2.length === 0) return cnf1;
-
-  if (cnf1.length > cnf2.length) {
-    const c = cnf1;
-    cnf1 = cnf2;
-    cnf2 = c;
-  }
-
-  const res = [];
-  for (let i = 0; i < cnf1.length; ++i)
-    for (let j = i; j < cnf2.length; ++j) res.push(cnf1[i].concat(cnf2[j]));
-
-  return res;
-}
-
-function likePatternIncludes(pat1, pat2, idx1 = 0, idx2 = 0): boolean {
-  while (idx1 < pat1.length && idx2 < pat2.length) {
-    if (pat1[idx1] === "\\%") {
-      for (let i = idx2; i <= pat2.length; ++i)
-        if (likePatternIncludes(pat1, pat2, idx1 + 1, i)) return true;
-      return false;
-    } else if (pat2[idx2] === "\\%") {
-      if (pat1[idx1] !== "\\%") return false;
-    } else if (pat1[idx1] === "\\_") {
-      // Ignore
-    } else if (pat2[idx2] === "\\_") {
-      return false;
-    } else if (pat2[idx2] !== pat1[idx1]) {
-      return false;
-    }
-    ++idx1;
-    ++idx2;
-  }
-
-  if (idx1 === pat1.length && idx2 === pat2.length) return true;
-  return false;
-}
-
-export function booleanCnf(exp: Expression): { vars: number; clauses: CNF } {
-  const ranges = new Map();
-  const likePatterns = new Map();
-  map(exp, e => {
-    if (!Array.isArray(e)) return e;
-    if (e[0] === "LIKE" || e[0] === "NOT LIKE") {
-      if (
-        typeof e[2] === "string" &&
-        (e.length >= 4 || (e[3] != null && !Array.isArray(e[3])))
-      ) {
-        const key = JSON.stringify(e[1]);
-        let s = likePatterns.get(key);
-        if (!s) likePatterns.set(key, (s = new Map()));
-        s.set(`${e[3] || ""}:${e[2]}`, parseLikePattern(e[2], e[3]));
-      }
-    } else if ([">", ">=", "<", "<=", "=", "<>"].includes(e[0])) {
-      if (!Array.isArray(e[2])) {
-        const key = `${JSON.stringify(e[1])}:${typeof e[2]}`;
-        let s = ranges.get(key);
-        if (!s) ranges.set(key, (s = new Set()));
-        s.add(e[2]);
-      } else if (!Array.isArray(e[1])) {
-        const key = `${JSON.stringify(e[2])}:${typeof e[1]}`;
-        let s = ranges.get(key);
-        if (!s) ranges.set(key, (s = new Set()));
-        s.add(e[2]);
-      }
-    }
-    return e;
-  });
-
-  for (const [k, v] of ranges) {
-    if (k.endsWith(":number")) {
-      ranges.set(
-        k,
-        Array.from(v as number[]).sort((a: number, b: number) => a - b)
-      );
-    } else {
-      ranges.set(k, Array.from(v).sort());
-    }
-  }
-
-  const expressions: Map<string, number> = new Map();
-  const mutuallyExclusive: Map<string, Set<number>> = new Map();
-
-  function getVariable(lhs, op, rhs = ""): number {
-    const key = `${lhs}:${op}:${rhs}`;
-    let f = expressions.get(key);
-    if (!f) {
-      expressions.set(key, (f = expressions.size + 1));
-      let m = mutuallyExclusive.get(lhs);
-      if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
-      m.add(f);
-      if (rhs || op !== "null") m.add(getVariable(lhs, "null"));
-    }
-    return f;
-  }
-
-  function getComparisonVariables(
-    lhs,
-    from,
-    fromInclusive,
-    to,
-    toInclusive
-  ): number[] {
-    const type = from != null ? typeof from : typeof to;
-    const range = ranges.get(`${lhs}:${type}`);
-    const vars = [];
-
-    if (from == null) vars.push(`${lhs}:--${range[0]}`);
-
-    for (let i = 0; i < range.length; ++i) {
-      if (
-        i > 0 &&
-        (from == null || range[i] > from) &&
-        (to == null || range[i] <= to)
-      )
-        vars.push(`${lhs}:${range[i - 1]}--${range[i]}`);
-
-      if (
-        (range[i] > from && range[i] < to) ||
-        (range[i] === from && fromInclusive) ||
-        (range[i] === to && toInclusive)
-      )
-        vars.push(`${lhs}:${range[i]}`);
-    }
-
-    if (to == null) vars.push(`${lhs}:${range[range.length - 1]}--`);
-
-    return vars.map(key => {
-      let f = expressions.get(key);
-      if (!f) {
-        expressions.set(key, (f = expressions.size + 1));
-        let m = mutuallyExclusive.get(lhs);
-        if (!m) mutuallyExclusive.set(lhs, (m = new Set()));
-        m.add(f);
-      }
-      return f;
-    });
-  }
-
-  function getLikeVariables(lhs, pat, esc): number[] {
-    const vars = [];
-    const pats = likePatterns.get(lhs);
-    pat = pats.get(`${esc || ""}:${pat}`);
-    for (const p of pats.values()) {
-      if (pat === p || likePatternIncludes(pat, p))
-        vars.push(`${lhs}:like:${JSON.stringify(p)}`);
-    }
-
-    return vars.map(key => {
-      let f = expressions.get(key);
-      if (!f) expressions.set(key, (f = expressions.size + 1));
-      return f;
-    });
-  }
-
-  interface ComparisonOperators {
-    [name: string]: (v: string) => [string, boolean, string, boolean];
-  }
-
-  const comparisonOperators: ComparisonOperators = {
-    "=": v => [v, true, v, true],
-    ">": v => [v, false, null, null],
-    ">=": v => [v, true, null, null],
-    "<": v => [null, null, v, false],
-    "<=": v => [null, null, v, true]
-  };
-
-  function recursive(clause, negate): CNF {
-    let op = clause[0];
-
-    if ((op === "AND" && !negate) || (op === "OR" && negate)) {
-      let res = [];
-      for (let i = 1; i < clause.length; ++i)
-        res = res.concat(recursive(clause[i], negate));
-      return res;
-    } else if ((op === "OR" && !negate) || (op === "AND" && negate)) {
-      let res = [];
-      for (let i = 1; i < clause.length; ++i)
-        res = or(res, recursive(clause[i], negate));
-      return res;
-    } else if (op === "NOT") {
-      return recursive(clause[1], !negate);
-    } else if (op === "<>") {
-      const vars = getComparisonVariables(
-        JSON.stringify(clause[1]),
-        ...comparisonOperators["="](clause[2])
-      );
-      if (!negate) return vars.map(x => [0 - x]);
-      else return [vars];
-    } else if (op === "IS NULL") {
-      const v = getVariable(JSON.stringify(clause[1]), "null");
-      if (negate) return [[0 - v]];
-      else return [[v]];
-    } else if (op === "IS NOT NULL") {
-      const v = getVariable(JSON.stringify(clause[1]), "null");
-      if (!negate) return [[0 - v]];
-      else return [[v]];
-    } else if (op in comparisonOperators) {
-      if (!Array.isArray(clause[2])) {
-        const vars = getComparisonVariables(
-          JSON.stringify(clause[1]),
-          ...comparisonOperators[op](clause[2])
-        );
-        if (negate) return vars.map(x => [0 - x]);
-        else return [vars];
-      } else if (!Array.isArray(clause[1])) {
-        op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
-        const vars = getComparisonVariables(
-          JSON.stringify(clause[2]),
-          ...comparisonOperators[op](clause[1])
-        );
-        if (negate) return vars.map(x => [0 - x]);
-        else return [vars];
-      } else {
-        let lhs = JSON.stringify(clause[1]);
-        let rhs = JSON.stringify(clause[2]);
-        // For consistency
-        if (rhs > lhs) {
-          const t = rhs;
-          rhs = lhs;
-          lhs = t;
-          op = { ">": "<", ">=": "<=", "<": ">", "<=": ">=" }[op] || op;
-        }
-
-        const v = getVariable(lhs, op, rhs);
-        if (negate) return [[0 - v]];
-        else return [[v]];
-      }
-    } else if (
-      (op === "LIKE" || op === "NOT LIKE") &&
-      !Array.isArray(clause[2]) &&
-      clause[2] != null &&
-      (clause.length >= 4 || clause[3] != null)
-    ) {
-      if (op === "NOT LIKE") negate = !negate;
-      const vars = getLikeVariables(
-        JSON.stringify(clause[1]),
-        clause[2],
-        clause[3]
-      );
-      if (negate) return vars.map(x => [0 - x]);
-      else return [vars];
-    } else {
-      const v = getVariable(JSON.stringify(clause), "");
-      if (negate) return [[0 - v]];
-      else return [[v]];
-    }
-  }
-
-  const cnf = recursive(exp, false);
-
-  for (const m of mutuallyExclusive.values()) {
-    const ar: number[] = Array.from(m);
-    for (let i = 0; i < ar.length; ++i)
-      for (let j = i + 1; j < ar.length; ++j) cnf.push([0 - ar[i], 0 - ar[j]]);
-  }
-
-  return { vars: expressions.size, clauses: cnf };
-}

lib/common/expression-parser.ts

@@ -1,412 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as parsimmon from "parsimmon";
-import { Expression } from "../types";
-
-// Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
-function interpretEscapes(str): string {
-  const escapes = {
-    b: "\b",
-    f: "\f",
-    n: "\n",
-    r: "\r",
-    t: "\t"
-  };
-  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
-    const type = escape.charAt(0);
-    const hex = escape.slice(1);
-    if (type === "u") return String.fromCharCode(parseInt(hex, 16));
-
-    if (escapes.hasOwnProperty(type)) return escapes[type];
-
-    return type;
-  });
-}
-
-export function map(exp, callback): Expression {
-  if (!Array.isArray(exp)) return callback(exp);
-
-  let clone;
-  for (let i = 1; i < exp.length; ++i) {
-    const sub = map(exp[i], callback);
-    if (sub !== exp[i]) {
-      clone = clone || exp.slice();
-      clone[i] = sub;
-    }
-  }
-
-  return callback(clone || exp);
-}
-
-export async function mapAsync(exp, callback): Promise<Expression> {
-  if (!Array.isArray(exp)) return callback(exp);
-
-  let clone;
-  for (let i = 1; i < exp.length; ++i) {
-    const sub = await mapAsync(exp[i], callback);
-    if (sub !== exp[i]) {
-      clone = clone || exp.slice();
-      clone[i] = sub;
-    }
-  }
-
-  return callback(clone || exp);
-}
-
-function binaryLeft(operatorsParser, nextParser): parsimmon.Parser<{}> {
-  return parsimmon.seqMap(
-    nextParser,
-    parsimmon.seq(operatorsParser, nextParser).many(),
-    (first, rest) =>
-      rest.reduce((acc, ch) => {
-        const [op, another] = ch;
-        if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
-        if (Array.isArray(another) && op === another[0])
-          return [op, acc].concat(another.slice(1));
-        return [op, acc, another];
-      }, first)
-  );
-}
-
-const lang = parsimmon.createLanguage({
-  ComparisonOperator: function() {
-    return parsimmon
-      .alt(
-        parsimmon.string(">="),
-        parsimmon.string("<>"),
-        parsimmon.string("<="),
-        parsimmon.string("="),
-        parsimmon.string(">"),
-        parsimmon.string("<")
-      )
-      .skip(parsimmon.optWhitespace);
-  },
-  LikeOperator: function() {
-    return parsimmon
-      .alt(
-        parsimmon
-          .regexp(/like/i)
-          .result("LIKE")
-          .desc("LIKE"),
-        parsimmon
-          .regexp(/not\s+like/i)
-          .result("NOT LIKE")
-          .desc("NOT LIKE")
-      )
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace);
-  },
-  IsNullOperator: function() {
-    return parsimmon
-      .alt(
-        parsimmon
-          .regexp(/is\s+null/i)
-          .result("IS NULL")
-          .desc("IS NULL"),
-        parsimmon
-          .regexp(/is\s+not\s+null/i)
-          .result("IS NOT NULL")
-          .desc("IS NOT NULL")
-      )
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace);
-  },
-  NotOperator: function() {
-    return parsimmon
-      .regexp(/not/i)
-      .result("NOT")
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace)
-      .desc("NOT");
-  },
-  AndOperator: function() {
-    return parsimmon
-      .regexp(/and/i)
-      .result("AND")
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace)
-      .desc("AND");
-  },
-  OrOperator: function() {
-    return parsimmon
-      .regexp(/or/i)
-      .result("OR")
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace)
-      .desc("OR");
-  },
-  Parameter: function(r) {
-    return parsimmon
-      .alt(
-        parsimmon.regexp(/[a-zA-Z0-9_.*-]+/),
-        r.Expression.wrap(
-          parsimmon.string("{").skip(parsimmon.optWhitespace),
-          parsimmon.string("}")
-        )
-      )
-      .atLeast(1)
-      .map(x => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
-      .skip(parsimmon.optWhitespace)
-      .desc("parameter");
-  },
-  StringValueSql: function() {
-    return parsimmon
-      .regexp(/'([^']*)'/, 1)
-      .atLeast(1)
-      .skip(parsimmon.optWhitespace)
-      .map(s => s.join("'"))
-      .desc("string");
-  },
-  StringValueJs: function() {
-    return parsimmon
-      .regexp(/"((?:\\.|.)*?)"/, 1)
-      .skip(parsimmon.optWhitespace)
-      .map(interpretEscapes)
-      .desc("string");
-  },
-  NumberValue: function() {
-    return parsimmon
-      .regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/)
-      .skip(parsimmon.optWhitespace)
-      .map(Number)
-      .desc("number");
-  },
-  BooleanValue: function() {
-    return parsimmon
-      .alt(
-        parsimmon
-          .regexp(/true/i)
-          .result(true)
-          .desc("TRUE"),
-        parsimmon
-          .regexp(/false/i)
-          .result(false)
-          .desc("FALSE")
-      )
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace);
-  },
-  NullValue: function() {
-    return parsimmon
-      .regexp(/null/i)
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace)
-      .result(null)
-      .desc("NULL");
-  },
-  FuncValue: function(r) {
-    return parsimmon.seqMap(
-      parsimmon
-        .regexp(/([a-zA-Z0-9_]+)/, 1)
-        .skip(parsimmon.optWhitespace)
-        .desc("function"),
-      r.ValueExpression.sepBy(
-        parsimmon.string(",").skip(parsimmon.optWhitespace)
-      ).wrap(
-        parsimmon.string("(").skip(parsimmon.optWhitespace),
-        parsimmon.string(")").skip(parsimmon.optWhitespace)
-      ),
-      (f, args) => ["FUNC", f.toUpperCase()].concat(args)
-    );
-  },
-  Value: function(r) {
-    return parsimmon.alt(
-      r.NullValue,
-      r.BooleanValue,
-      r.NumberValue,
-      r.StringValueSql,
-      r.StringValueJs,
-      r.FuncValue
-    );
-  },
-  ValueExpression: function(r) {
-    return binaryLeft(
-      parsimmon.string("||").skip(parsimmon.optWhitespace),
-      binaryLeft(
-        parsimmon
-          .alt(parsimmon.string("+"), parsimmon.string("-"))
-          .skip(parsimmon.optWhitespace),
-        binaryLeft(
-          parsimmon
-            .alt(parsimmon.string("*"), parsimmon.string("/"))
-            .skip(parsimmon.optWhitespace),
-          parsimmon.alt(
-            r.Value,
-            r.Parameter,
-            r.Expression.wrap(
-              parsimmon.string("(").skip(parsimmon.optWhitespace),
-              parsimmon.string(")").skip(parsimmon.optWhitespace)
-            )
-          )
-        )
-      )
-    );
-  },
-  Comparison: function(r) {
-    return parsimmon.alt(
-      parsimmon.seqMap(r.ValueExpression, r.IsNullOperator, (p, o) => [o, p]),
-      parsimmon.seqMap(
-        r.ValueExpression,
-        r.ComparisonOperator,
-        r.ValueExpression,
-        (p, o, v) => [o, p, v]
-      ),
-      parsimmon.seqMap(
-        r.ValueExpression,
-        r.LikeOperator,
-        r.ValueExpression.skip(
-          parsimmon
-            .regexp(/escape/i)
-            .result("ESCAPE")
-            .skip(parsimmon.whitespace)
-            .desc("ESCAPE")
-        ),
-        r.ValueExpression,
-        (a, b, c, d) => [b, a, c, d]
-      ),
-      parsimmon.seqMap(
-        r.ValueExpression,
-        r.LikeOperator,
-        r.ValueExpression,
-        (a, b, c) => [b, a, c]
-      )
-    );
-  },
-  Expression: function(r) {
-    function unary(operatorsParser, nextParser): parsimmon.Parser<{}> {
-      return parsimmon.seq(operatorsParser, nextParser).or(nextParser);
-    }
-
-    return binaryLeft(
-      r.OrOperator,
-      binaryLeft(
-        r.AndOperator,
-        unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
-      )
-    ).trim(parsimmon.optWhitespace);
-  }
-});
-
-export function parse(str: string): Expression {
-  if (!str) return null;
-  return lang.Expression.tryParse(str);
-}
-
-export function stringify(exp, level = 0): string {
-  if (!Array.isArray(exp)) return JSON.stringify(exp);
-
-  const opLevels = {
-    OR: 10,
-    AND: 11,
-    NOT: 12,
-    "=": 20,
-    "<>": 20,
-    ">": 20,
-    ">=": 20,
-    "<": 20,
-    "<=": 20,
-    LIKE: 20,
-    "NOT LIKE": 20,
-    "IS NULL": 20,
-    "IS NOT NULL": 20,
-    "||": 30,
-    "+": 31,
-    "-": 31,
-    "*": 32,
-    "/": 32
-  };
-
-  const op = exp[0].toUpperCase();
-
-  function wrap(e): string {
-    if (opLevels[op] <= level) return `(${e})`;
-    else return e;
-  }
-
-  if (op === "FUNC") {
-    return wrap(
-      `${exp[1]}(${exp
-        .slice(2)
-        .map(e => stringify(e))
-        .join(", ")})`
-    );
-  } else if (op === "PARAM") {
-    if (typeof exp[1] === "string") {
-      return wrap(exp[1]);
-    } else if (Array.isArray(exp[1]) && exp[1][0] === "||") {
-      return wrap(
-        exp[1]
-          .slice(1)
-          .map(p => {
-            if (typeof p === "string") return p;
-            else return `{${stringify(p)}}`;
-          })
-          .join("")
-      );
-    } else {
-      return wrap(`{${stringify(exp[1])}}`);
-    }
-  } else if (op === "IS NULL" || op === "IS NOT NULL") {
-    return wrap(`${stringify(exp[1], opLevels[op])} ${op}`);
-  } else if (op === "LIKE" || op === "NOT LIKE") {
-    if (exp[3]) {
-      return wrap(
-        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
-          exp[2],
-          opLevels[op]
-        )} ESCAPE ${stringify(exp[3], opLevels[op])}`
-      );
-    } else {
-      return wrap(
-        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
-          exp[2],
-          opLevels[op]
-        )}`
-      );
-    }
-  } else if (op in opLevels) {
-    const parts = exp.slice(1).map((e, i) => {
-      return stringify(e, opLevels[exp[0]] + Math.min(i - 1, 0));
-    });
-
-    if (op === "NOT") return wrap(`${op} ${parts[0]}`);
-    else return wrap(parts.join(` ${op} `));
-  } else {
-    throw new Error(`Unrecognized operator ${exp[0]}`);
-  }
-}
-
-export function parseLikePattern(pat, esc): string[] {
-  const chars = pat.split("");
-
-  for (let i = 0; i < chars.length; ++i) {
-    const c = chars[i];
-    if (c === esc) {
-      chars[i] = chars[i + 1] || "";
-      chars[i + 1] = "";
-    } else if (c === "_") {
-      chars[i] = "\\_";
-    } else if (c === "%") {
-      chars[i] = "\\%";
-      while (chars[i + 1] === "%") chars[++i] = "";
-    }
-  }
-  return chars.filter(c => c);
-}

lib/common/expression.ts

@@ -1,329 +1,291 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
+import {
+  Cursor,
+  parseExpression,
+  stringifyExpression,
+} from "./expression/parser.ts";
+import Path from "./path.ts";
+import { reduce } from "./expression/evaluate.ts";
 
-import { map, mapAsync, parseLikePattern } from "./expression-parser";
-import { booleanCnf } from "./expression-cnf";
-import { naiveDpll } from "./sat-solver";
-import { Expression } from "../types";
+export type Value = string | number | boolean | null;
 
-const isArray = Array.isArray;
+export abstract class Expression {
+  private _string: string;
 
-const regExpCache = new WeakMap<object, RegExp>();
+  abstract map(fn: (e: Expression, i: number) => Expression): Expression;
+  abstract mapAsync(
+    fn: (e: Expression, i: number) => Promise<Expression>,
+  ): Promise<Expression>;
 
-const REDUCE_SKIP = {};
-function reduce(exp, callback): Expression {
-  let loop = true;
-  while (loop) {
-    loop = false;
-    for (let i = 2; i < exp.length; ++i) {
-      const res = callback(exp[i - 1], exp[i], i - 2);
-      if (res !== REDUCE_SKIP) {
-        loop = true;
-        exp = exp.slice();
-        exp.splice(i - 1, 2, res);
+  toString(): string {
+    if (!this._string) this._string = stringifyExpression(this);
+    return this._string;
+  }
+
+  evaluate<T extends Expression>(fn: (e: Expression) => T = (e) => e as T): T {
+    return fn(reduce(this.map((e) => e.evaluate(fn))));
+  }
+
+  async evaluateAsync<T extends Expression>(
+    fn: (e: Expression) => Promise<T>,
+  ): Promise<T> {
+    return await fn(
+      reduce(await this.mapAsync(async (e) => await e.evaluateAsync(fn))),
+    );
+  }
+
+  static parse(input: string): Expression {
+    const cursor = new Cursor(input);
+    const exp = parseExpression(cursor);
+    if (cursor.charCode) throw new Error("Unexpected character");
+    return exp;
+  }
+
+  static and(left: Expression, right: Expression): Expression {
+    // Flatten same-operator tree into operands
+    const operands: Expression[] = [];
+    const stack: Expression[] = [right, left];
+    while (stack.length) {
+      const e = stack.pop()!;
+      if (e instanceof Expression.Binary && e.operator === "AND")
+        stack.push(e.right, e.left);
+      else operands.push(e);
+    }
+
+    // Fold literals using three-valued AND logic
+    let folded: boolean | null = true;
+    let i = 0;
+    while (i < operands.length) {
+      const e = operands[i];
+      if (e instanceof Expression.Literal) {
+        operands.splice(i, 1);
+        if (e.value == null) folded = folded === false ? false : null;
+        else if (!e.value) return new Expression.Literal(false);
+        // truthy is identity for AND; discard
+      } else {
+        i++;
       }
     }
+
+    // Rebuild: folded value + remaining non-literal operands
+    if (!operands.length) return new Expression.Literal(folded);
+    let result = operands.reduce((a, b) => new Expression.Binary("AND", a, b));
+    if (folded === null)
+      result = new Expression.Binary(
+        "AND",
+        new Expression.Literal(null),
+        result,
+      );
+    return result;
+  }
+
+  static or(left: Expression, right: Expression): Expression {
+    // Flatten same-operator tree into operands
+    const operands: Expression[] = [];
+    const stack: Expression[] = [right, left];
+    while (stack.length) {
+      const e = stack.pop()!;
+      if (e instanceof Expression.Binary && e.operator === "OR")
+        stack.push(e.right, e.left);
+      else operands.push(e);
+    }
+
+    // Fold literals using three-valued OR logic
+    let folded: boolean | null = false;
+    let i = 0;
+    while (i < operands.length) {
+      const e = operands[i];
+      if (e instanceof Expression.Literal) {
+        operands.splice(i, 1);
+        if (e.value == null) folded = folded === true ? true : null;
+        else if (e.value) return new Expression.Literal(true);
+        // falsy is identity for OR; discard
+      } else {
+        i++;
+      }
+    }
+
+    // Rebuild: folded value + remaining non-literal operands
+    if (!operands.length) return new Expression.Literal(folded);
+    let result = operands.reduce((a, b) => new Expression.Binary("OR", a, b));
+    if (folded === null)
+      result = new Expression.Binary(
+        "OR",
+        new Expression.Literal(null),
+        result,
+      );
+    return result;
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/no-namespace
+export namespace Expression {
+  export class Literal extends Expression {
+    constructor(public readonly value: Value) {
+      super();
+    }
+
+    map(): Literal {
+      return this;
+    }
+
+    async mapAsync(): Promise<Literal> {
+      return this;
+    }
   }
-  if (exp.length === 2) return exp[1];
-  return exp;
-}
 
-export function likePatternToRegExp(pat, esc = "", flags = ""): RegExp {
-  const convChars = {
-    "-": "\\-",
-    "/": "\\/",
-    "\\": "\\/",
-    "^": "\\^",
-    $: "\\$",
-    "*": "\\*",
-    "+": "\\+",
-    "?": "\\?",
-    ".": "\\.",
-    "(": "\\(",
-    ")": "\\)",
-    "|": "\\|",
-    "[": "\\[",
-    "]": "\\]",
-    "{": "\\{",
-    "}": "\\}",
-    "\\%": ".*",
-    "\\_": "."
-  };
-  let chars = parseLikePattern(pat, esc);
-  if (!chars.length) return new RegExp("^$", flags);
-  chars = chars.map(c => convChars[c] || c);
-  chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
-  const l = chars.length - 1;
-  chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
-  return new RegExp(chars.join(""), flags);
-}
+  export class Parameter extends Expression {
+    constructor(public readonly path: Path) {
+      super();
+    }
 
-function evalExp(e: Expression): Expression {
-  if (e[0] === "AND") {
-    return reduce(e, (a, b) => {
-      if (!isArray(a)) return a ? b : a;
-      if (!isArray(b)) return b ? a : b;
-      return REDUCE_SKIP;
-    });
-  } else if (e[0] === "OR") {
-    return reduce(e, (a, b) => {
-      if (!isArray(a)) return a ? a : b;
-      if (!isArray(b)) return b ? b : a;
-      return REDUCE_SKIP;
-    });
-  } else if (e[0] === "NOT") {
-    if (!isArray(e[1])) return !e[1];
-    else if (e[1][0] === "NOT") return e[1][1];
-  } else if (e[0] === "IS NULL") {
-    if (isArray(e[1])) return e;
-    else if (e[1] == null) return true;
-    else return null;
-  } else if (e[0] === "IS NOT NULL") {
-    if (isArray(e[1])) return e;
-    else if (e[1] != null) return true;
-    else return null;
-  } else if (e[0] === "LIKE") {
-    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
-    else if (
-      e[1] == null ||
-      e[2] == null ||
-      ((e as any[]).length >= 4 && e[3] == null)
-    )
-      return null;
-    let r = regExpCache.get(e as any[]);
-    if (!r) {
-      r = likePatternToRegExp(e[2], e[3]);
-      regExpCache.set(e as any[], r);
+    map(): Parameter {
+      return this;
     }
-    return r.test(e[1]);
-  } else if (e[0] === "NOT LIKE") {
-    if (isArray(e[1]) || isArray(e[2]) || isArray(e[3])) return e;
-    else if (
-      e[1] == null ||
-      e[2] == null ||
-      ((e as any[]).length >= 4 && e[3] == null)
-    )
-      return null;
-    let r = regExpCache.get(e as any[]);
-    if (!r) {
-      r = likePatternToRegExp(e[2], e[3]);
-      regExpCache.set(e as any[], r);
+
+    async mapAsync(): Promise<Parameter> {
+      return this;
+    }
+
+    toString(): string {
+      return this.path.toString();
     }
-    return !r.test(e[1]);
-  } else if (e[0] === "=") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] === e[2];
-  } else if (e[0] === "<>") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] !== e[2];
-  } else if (e[0] === ">") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] > e[2];
-  } else if (e[0] === ">=") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] >= e[2];
-  } else if (e[0] === "<") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] < e[2];
-  } else if (e[0] === "<=") {
-    if (isArray(e[1]) || isArray(e[2])) return e;
-    if (e[1] == null || e[2] == null) return null;
-    return e[1] <= e[2];
-  } else if (e[0] === "*") {
-    return reduce(e, (a, b) => {
-      if (!isArray(a) && !isArray(b)) {
-        if (a == null || b == null) return null;
-        return a * b;
-      }
-      return REDUCE_SKIP;
-    });
-  } else if (e[0] === "/") {
-    return reduce(e, (a, b, i) => {
-      if (!isArray(a) && !isArray(b)) {
-        if (a == null || b == null) return null;
-        return i === 0 ? a / b : a * b;
-      }
-      return REDUCE_SKIP;
-    });
-  } else if (e[0] === "+") {
-    return reduce(e, (a, b) => {
-      if (!isArray(a) && !isArray(b)) {
-        if (a == null || b == null) return null;
-        return a + b;
-      }
-      return REDUCE_SKIP;
-    });
-  } else if (e[0] === "-") {
-    return reduce(e, (a, b, i) => {
-      if (!isArray(a) && !isArray(b)) {
-        if (a == null || b == null) return null;
-        return i === 0 ? a - b : a + b;
-      }
-      return REDUCE_SKIP;
-    });
-  } else if (e[0] === "||") {
-    return reduce(e, (a, b) => {
-      if (!isArray(a) && !isArray(b)) {
-        if (a == null || b == null) return null;
-        return `${a}${b}`;
-      }
-      return REDUCE_SKIP;
-    });
   }
-  return e;
-}
 
-export function evaluate(
-  exp,
-  obj,
-  now: number,
-  cb?: Function
-): string | number | boolean | null;
-export function evaluate(exp, obj?, now?: number, cb?: Function): Expression;
-export function evaluate(exp, obj?, now?: number, cb?: Function): Expression {
-  return map(exp, e => {
-    if (cb) e = cb(e);
-    if (!isArray(e)) return e;
-
-    if (e[0] === "FUNC") {
-      if (e[1] === "NOW") {
-        if (now) return now;
-      } else if (e[1] === "UPPER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
-      } else if (e[1] === "LOWER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
-      }
-    } else if (e[0] === "PARAM") {
-      if (e[1] == null) return null;
-      if (obj && !isArray(e[1])) {
-        let v;
-        if (typeof obj === "function") v = obj(e[1]);
-        else v = obj[e[1]];
-
-        if (v == null) return null;
-        if (typeof v === "object") v = v.value ? v.value[0] : null;
-        return v;
-      }
+  export class Unary extends Expression {
+    constructor(
+      public readonly operator: string,
+      public readonly operand: Expression,
+    ) {
+      super();
     }
-    return evalExp(e);
-  });
-}
 
-export async function evaluateAsync(
-  exp,
-  obj,
-  now: number,
-  cb?: Function
-): Promise<string | number | boolean | null>;
-export async function evaluateAsync(
-  exp,
-  obj?,
-  now?: number,
-  cb?: Function
-): Promise<Expression>;
-export async function evaluateAsync(
-  exp,
-  obj?,
-  now?: number,
-  cb?: Function
-): Promise<Expression> {
-  return mapAsync(exp, async e => {
-    if (cb) e = await cb(e);
-    if (!isArray(e)) return e;
-
-    if (e[0] === "FUNC") {
-      if (e[1] === "NOW") {
-        if (now) return now;
-      } else if (e[1] === "UPPER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
-      } else if (e[1] === "LOWER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
-      }
-    } else if (e[0] === "PARAM") {
-      if (e[1] == null) return null;
-      if (obj && !isArray(e[1])) {
-        let v = obj[e[1]];
-        if (v == null) return null;
-        if (typeof v === "object") v = v.value ? v.value[0] : null;
-        return v;
-      }
+    map(fn: (e: Expression, i: number) => Expression): Unary {
+      const operand = fn(this.operand, 0);
+      if (operand === this.operand) return this;
+      return Reflect.construct(this.constructor, [this.operator, operand]);
     }
-    return evalExp(e);
-  });
+
+    async mapAsync(
+      fn: (e: Expression, i: number) => Promise<Expression>,
+    ): Promise<Unary> {
+      const operand = await fn(this.operand, 0);
+      if (operand === this.operand) return this;
+      return Reflect.construct(this.constructor, [this.operator, operand]);
+    }
+  }
+
+  export class Binary extends Expression {
+    constructor(
+      public readonly operator: string,
+      public readonly left: Expression,
+      public readonly right: Expression,
+    ) {
+      super();
+    }
+
+    map(fn: (e: Expression, i: number) => Expression): Binary {
+      const left = fn(this.left, 0);
+      const right = fn(this.right, 1);
+      if (left === this.left && right === this.right) return this;
+      return Reflect.construct(this.constructor, [this.operator, left, right]);
+    }
+
+    async mapAsync(
+      fn: (e: Expression, i: number) => Promise<Expression>,
+    ): Promise<Binary> {
+      const left = await fn(this.left, 0);
+      const right = await fn(this.right, 1);
+      if (left === this.left && right === this.right) return this;
+      return Reflect.construct(this.constructor, [this.operator, left, right]);
+    }
+  }
+
+  export class FunctionCall extends Expression {
+    constructor(
+      public readonly name: string,
+      public readonly args: Expression[],
+    ) {
+      super();
+    }
+
+    map(fn: (e: Expression, i: number) => Expression): FunctionCall {
+      const args = this.args.map(fn);
+      if (args.every((arg, i) => arg === this.args[i])) return this;
+      return new FunctionCall(this.name, args);
+    }
+
+    async mapAsync(
+      fn: (e: Expression, i: number) => Promise<Expression>,
+    ): Promise<FunctionCall> {
+      const args = await Promise.all(this.args.map(fn));
+      if (args.every((arg, i) => arg === this.args[i])) return this;
+      return new FunctionCall(this.name, args);
+    }
+  }
+
+  export class Conditional extends Expression {
+    constructor(
+      public readonly condition: Expression,
+      public readonly then: Expression,
+      public readonly otherwise: Expression,
+    ) {
+      super();
+    }
+
+    map(fn: (e: Expression, i: number) => Expression): Conditional {
+      const condition = fn(this.condition, 0);
+      const then = fn(this.then, 1);
+      const otherwise = fn(this.otherwise, 2);
+      if (
+        condition === this.condition &&
+        then === this.then &&
+        otherwise === this.otherwise
+      )
+        return this;
+      return new Conditional(condition, then, otherwise);
+    }
+
+    async mapAsync(
+      fn: (e: Expression, i: number) => Promise<Expression>,
+    ): Promise<Conditional> {
+      const condition = await fn(this.condition, 0);
+      const then = await fn(this.then, 1);
+      const otherwise = await fn(this.otherwise, 2);
+      if (
+        condition === this.condition &&
+        then === this.then &&
+        otherwise === this.otherwise
+      )
+        return this;
+      return new Conditional(condition, then, otherwise);
+    }
+  }
 }
 
-export function and(exp1, exp2): Expression {
-  if (!isArray(exp1)) return exp1 ? exp2 : exp1;
-  if (!isArray(exp2)) return exp2 ? exp1 : exp2;
-
-  let res: Expression = ["AND"];
-
-  if (exp1[0] === "AND") res = res.concat(exp1.slice(1));
-  else res.push(exp1);
-
-  if (exp2[0] === "AND") res = res.concat(exp2.slice(1));
-  else res.push(exp2);
-
-  return res;
-}
-
-export function or(exp1, exp2): Expression {
-  if (!isArray(exp1)) return exp1 ? exp1 : exp2;
-  if (!isArray(exp2)) return exp2 ? exp2 : exp1;
-
-  let res: Expression = ["OR"];
-
-  if (exp1[0] === "OR") res = res.concat(exp1.slice(1));
-  else res.push(exp1);
-
-  if (exp2[0] === "OR") res = res.concat(exp2.slice(1));
-  else res.push(exp2);
-
-  return res;
-}
-
-export function not(exp): Expression {
-  if (isArray(exp) && exp[0] === "NOT") return exp[1];
-  return ["NOT", exp];
-}
-
-export function subset(exp1, exp2): boolean {
-  const e = evaluate(["NOT", ["OR", ["NOT", exp1], exp2]]);
-  if (!isArray(e)) return !e;
-  const { vars, clauses } = booleanCnf(e);
-  return !naiveDpll(clauses, vars);
-}
-
-export function extractParams(exp): Expression[] {
-  const params = new Set<Expression>();
-  map(exp, e => {
-    if (isArray(e) && e[0] === "PARAM") params.add(e[1]);
+export function extractPaths(exp: Expression): Path[] {
+  if (exp instanceof Expression.Parameter) return [exp.path];
+  const paths: Path[] = [];
+  exp.map((e) => {
+    if (e instanceof Expression.Parameter) paths.push(e.path);
+    else paths.push(...extractPaths(e));
     return e;
   });
-  return Array.from(params);
+  return paths;
 }
+
+export function parseList(input: string): Expression[] {
+  const CHAR_COMMA = 44;
+  const res: Expression[] = [];
+  const cursor = new Cursor(input);
+  cursor.skipwhitespace();
+  if (!cursor.charCode) return res;
+  res.push(parseExpression(cursor));
+  while (cursor.charCode === CHAR_COMMA) {
+    cursor.step();
+    res.push(parseExpression(cursor));
+  }
+  if (cursor.charCode) throw new Error("Unexpected character");
+  return res;
+}
+
+export default Expression;

lib/common/expression/evaluate.ts

@@ -0,0 +1,218 @@
+import Expression from "../expression.ts";
+import { likePatternToRegExp } from "./parser.ts";
+
+function compare(
+  a: boolean | number | string,
+  b: boolean | number | string,
+): number {
+  if (typeof a === "boolean") a = +a;
+  if (typeof b === "boolean") b = +b;
+  if (typeof a !== typeof b) return typeof a === "string" ? 1 : -1;
+  return a > b ? 1 : a < b ? -1 : 0;
+}
+
+function toNumber(a: boolean | number | string): number {
+  switch (typeof a) {
+    case "number":
+      return a;
+    case "boolean":
+      return +a;
+    case "string":
+      return parseFloat(a) || 0;
+  }
+}
+
+function toString(a: boolean | number | string): string {
+  switch (typeof a) {
+    case "string":
+      return a;
+    case "number":
+      return a.toString();
+    case "boolean":
+      return (+a).toString();
+  }
+}
+
+const regExpCache: WeakMap<Expression.Literal, RegExp> = new WeakMap();
+
+export function reduce(exp: Expression): Expression {
+  if (exp instanceof Expression.Literal) return exp;
+
+  if (exp instanceof Expression.Unary) {
+    if (exp.operator === "NOT") {
+      if (exp.operand instanceof Expression.Literal) {
+        if (exp.operand.value == null) return exp.operand;
+        return new Expression.Literal(!exp.operand.value);
+      } else if (exp.operand instanceof Expression.Unary) {
+        if (exp.operand.operator === "NOT") return exp.operand.operand;
+      }
+    } else if (exp.operator === "IS NULL") {
+      if (exp.operand instanceof Expression.Literal) {
+        return new Expression.Literal(exp.operand.value == null);
+      }
+    } else if (exp.operator === "IS NOT NULL") {
+      if (exp.operand instanceof Expression.Literal) {
+        return new Expression.Literal(exp.operand.value != null);
+      }
+    }
+  } else if (exp instanceof Expression.Binary) {
+    if (exp.operator === "AND") {
+      return Expression.and(exp.left, exp.right);
+    } else if (exp.operator === "OR") {
+      return Expression.or(exp.left, exp.right);
+    } else if (["=", ">", "<", "<>", ">=", "<="].includes(exp.operator)) {
+      if (exp.left instanceof Expression.Literal && exp.left.value == null)
+        return exp.left;
+      if (exp.right instanceof Expression.Literal && exp.right.value == null)
+        return exp.right;
+      if (
+        exp.left instanceof Expression.Literal &&
+        exp.right instanceof Expression.Literal
+      ) {
+        const c = compare(exp.left.value, exp.right.value);
+        switch (exp.operator) {
+          case "=":
+            return new Expression.Literal(c === 0);
+          case ">":
+            return new Expression.Literal(c > 0);
+          case "<":
+            return new Expression.Literal(c < 0);
+          case "<>":
+            return new Expression.Literal(c !== 0);
+          case ">=":
+            return new Expression.Literal(c >= 0);
+          case "<=":
+            return new Expression.Literal(c <= 0);
+        }
+      }
+    } else if (["+", "-", "*", "/"].includes(exp.operator)) {
+      if (exp.left instanceof Expression.Literal && exp.left.value == null)
+        return exp.left;
+      if (exp.right instanceof Expression.Literal && exp.right.value == null)
+        return exp.right;
+      if (
+        exp.left instanceof Expression.Literal &&
+        exp.right instanceof Expression.Literal
+      ) {
+        const a = toNumber(exp.left.value);
+        const b = toNumber(exp.right.value);
+        switch (exp.operator) {
+          case "+":
+            return new Expression.Literal(a + b);
+          case "-":
+            return new Expression.Literal(a - b);
+          case "*":
+            return new Expression.Literal(a * b);
+          case "/":
+            return new Expression.Literal(a / b);
+        }
+      }
+    } else if (exp.operator === "%") {
+      if (exp.left instanceof Expression.Literal && exp.left.value == null)
+        return exp.left;
+      if (exp.right instanceof Expression.Literal && exp.right.value == null)
+        return exp.right;
+      if (
+        exp.left instanceof Expression.Literal &&
+        exp.right instanceof Expression.Literal
+      ) {
+        const a = toNumber(exp.left.value);
+        const b = Math.trunc(toNumber(exp.right.value));
+        if (b === 0) return new Expression.Literal(null);
+        return new Expression.Literal(a % b);
+      }
+    } else if (exp.operator === "||") {
+      if (exp.left instanceof Expression.Literal && exp.left.value == null)
+        return exp.left;
+      if (exp.right instanceof Expression.Literal && exp.right.value == null)
+        return exp.right;
+      if (
+        exp.left instanceof Expression.Literal &&
+        exp.right instanceof Expression.Literal
+      ) {
+        const a = toString(exp.left.value);
+        const b = toString(exp.right.value);
+        return new Expression.Literal(a + b);
+      }
+    } else if (exp.operator === "LIKE") {
+      if (exp.left instanceof Expression.Literal && exp.left.value == null)
+        return exp.left;
+      if (exp.right instanceof Expression.Literal && exp.right.value == null)
+        return exp.right;
+      if (
+        exp.left instanceof Expression.Literal &&
+        exp.right instanceof Expression.Literal
+      ) {
+        const s = toString(exp.left.value);
+        let r = regExpCache.get(exp.right);
+        if (!r) {
+          r = likePatternToRegExp(toString(exp.right.value));
+          regExpCache.set(exp.right, r);
+        }
+        return new Expression.Literal(r.test(s));
+      }
+    } else if (exp.operator === "NOT LIKE") {
+      if (exp.left instanceof Expression.Literal && exp.left.value == null)
+        return exp.left;
+      if (exp.right instanceof Expression.Literal && exp.right.value == null)
+        return exp.right;
+      if (
+        exp.left instanceof Expression.Literal &&
+        exp.right instanceof Expression.Literal
+      ) {
+        const s = toString(exp.left.value);
+        let r = regExpCache.get(exp.right);
+        if (!r) {
+          r = likePatternToRegExp(toString(exp.right.value));
+          regExpCache.set(exp.right, r);
+        }
+        return new Expression.Literal(!r.test(s));
+      }
+    }
+  } else if (exp instanceof Expression.FunctionCall) {
+    if (exp.name === "COALESCE") {
+      const args = exp.args.filter(
+        (arg) => !(arg instanceof Expression.Literal && arg.value == null),
+      );
+      if (!args.length) return new Expression.Literal(null);
+      if (args.length === 1) return args[0];
+      if (args[0] instanceof Expression.Literal) return args[0];
+      if (args.length !== exp.args.length)
+        return new Expression.FunctionCall("COALESCE", args);
+    } else if (exp.name === "UPPER") {
+      if (exp.args[0] instanceof Expression.Literal) {
+        if (exp.args[0].value == null) return exp.args[0];
+        const a = toString(exp.args[0].value);
+        return new Expression.Literal(a.toUpperCase());
+      }
+    } else if (exp.name === "LOWER") {
+      if (exp.args[0] instanceof Expression.Literal) {
+        if (exp.args[0].value == null) return exp.args[0];
+        const a = toString(exp.args[0].value);
+        return new Expression.Literal(a.toLowerCase());
+      }
+    } else if (exp.name === "ROUND") {
+      let p = 0;
+      if (exp.args.length > 1) {
+        if (exp.args[1] instanceof Expression.Literal) {
+          if (exp.args[1].value == null) return exp.args[1];
+          p = Math.trunc(toNumber(exp.args[1].value));
+        }
+      }
+      if (exp.args[0] instanceof Expression.Literal) {
+        if (exp.args[0].value == null) return exp.args[0];
+        const n = toNumber(exp.args[0].value);
+        const d = 10 ** p;
+        const m = n * d * (1 + Number.EPSILON);
+        return new Expression.Literal(Math.round(m) / d);
+      }
+    }
+  } else if (exp instanceof Expression.Conditional) {
+    if (exp.condition instanceof Expression.Literal) {
+      if (exp.condition.value) return exp.then;
+      return exp.otherwise;
+    }
+  }
+
+  return exp;
+}

lib/common/expression/normalize.ts

@@ -0,0 +1,490 @@
+import Expression from "../expression.ts";
+import { reduce } from "./evaluate.ts";
+
+class Indeterminates {
+  declare public map: Map<Expression, number>;
+  declare public sortedKeys: Expression[];
+
+  public constructor(exp?: Expression) {
+    this.map = new Map();
+    if (exp) {
+      this.map.set(exp, 1);
+      this.sortedKeys = [exp];
+    } else {
+      this.sortedKeys = [];
+    }
+  }
+
+  public reciprocal(): Indeterminates {
+    const res = new Indeterminates();
+    res.sortedKeys = this.sortedKeys;
+    res.map = new Map();
+    for (const [k, v] of this.map) res.map.set(k, 0 - v);
+    return res;
+  }
+
+  public static multiply(
+    indeterminates1: Indeterminates,
+    indeterminates2: Indeterminates,
+  ): Indeterminates {
+    const res = new Indeterminates();
+    res.sortedKeys = indeterminates1.sortedKeys.slice();
+    res.map = new Map(indeterminates1.map);
+    const strMap: Map<string, Expression> = new Map();
+    for (const k of res.map.keys()) strMap.set(k.toString(), k);
+
+    for (const [key, val] of indeterminates2.map) {
+      const k = strMap.get(key.toString());
+      if (!k) {
+        res.map.set(key, val);
+        res.sortedKeys.push(key);
+      } else {
+        const v2 = val + res.map.get(k);
+        if (!v2) {
+          res.map.delete(k);
+          res.sortedKeys = res.sortedKeys.filter((s) => s !== k);
+        } else {
+          res.map.set(k, v2);
+        }
+      }
+    }
+
+    res.sortedKeys.sort((a, b) => {
+      const str1 = a.toString();
+      const str2 = b.toString();
+      if (str1.length !== str2.length) return str2.length - str1.length;
+      else if (str1 > str2) return 1;
+      else if (str1 < str2) return -1;
+      return 0;
+    });
+
+    return res;
+  }
+
+  public static compare(a: Indeterminates, b: Indeterminates): number {
+    if (a.sortedKeys.length !== b.sortedKeys.length)
+      return b.sortedKeys.length - a.sortedKeys.length;
+    for (let i = 0; i < a.sortedKeys.length; ++i) {
+      const k1 = a.sortedKeys[i];
+      const w1 = a.map.get(k1);
+      const k2 = b.sortedKeys[i];
+      const w2 = b.map.get(k2);
+      if (w1 !== w2) return w2 - w1;
+      if (k1.toString().length > k2.toString().length) return -1;
+      else if (k1.toString().length < k2.toString().length) return 1;
+      else if (k1.toString() > k2.toString()) return 1;
+      else if (k1.toString() < k2.toString()) return -1;
+    }
+    return 0;
+  }
+}
+
+interface Term {
+  indeterminates: Indeterminates;
+  coefficientNumerator: bigint;
+  coefficientDenominator: bigint;
+}
+
+function findGcd(a: bigint, b: bigint): bigint {
+  while (b !== 0n) {
+    const t = b;
+    b = a % b;
+    a = t;
+  }
+  return a;
+}
+
+class Polynomial extends Expression {
+  declare public terms: Term[];
+
+  public constructor(terms: Term[]) {
+    super();
+    this.terms = terms;
+  }
+
+  map(): Polynomial {
+    return this;
+  }
+
+  async mapAsync(): Promise<Polynomial> {
+    return this;
+  }
+
+  public static simplifyTerms(terms: Term[]): Term[] {
+    const ts = terms
+      .slice()
+      .sort((a: Term, b: Term) =>
+        Indeterminates.compare(a.indeterminates, b.indeterminates),
+      );
+
+    for (let i = 1; i < ts.length; ++i) {
+      const t1 = ts[i - 1];
+      const t2 = ts[i];
+      if (Indeterminates.compare(t1.indeterminates, t2.indeterminates) === 0) {
+        const numerator =
+          t1.coefficientNumerator * t2.coefficientDenominator +
+          t2.coefficientNumerator * t1.coefficientDenominator;
+
+        const denominator =
+          t1.coefficientDenominator * t2.coefficientDenominator;
+
+        const gcd = findGcd(numerator, denominator);
+
+        ts[i] = {
+          indeterminates: t2.indeterminates,
+          coefficientNumerator: numerator / gcd,
+          coefficientDenominator: denominator / gcd,
+        };
+        ts[i - 1] = {
+          indeterminates: t1.indeterminates,
+          coefficientNumerator: 0n,
+          coefficientDenominator: t1.coefficientDenominator,
+        };
+      }
+    }
+    return ts.filter((v) => v.coefficientNumerator !== 0n);
+  }
+
+  public static fromIndeterminate(indeterminate: Expression): Polynomial {
+    const indeterminates = new Indeterminates(indeterminate);
+    const terms = [
+      {
+        indeterminates: indeterminates,
+        coefficientNumerator: 1n,
+        coefficientDenominator: 1n,
+      },
+    ];
+    return new Polynomial(terms);
+  }
+
+  public static fromConstant(constant: number): Polynomial {
+    const [int, frac] = Math.abs(constant).toString(2).split(".", 2);
+    let numerator = BigInt("0b" + int);
+    if (constant < 0) numerator = numerator / -1n;
+    let denominator = 1n;
+    if (frac) {
+      denominator = 2n ** BigInt(frac.length);
+      numerator = numerator * denominator + BigInt("0b" + frac);
+    }
+
+    const terms = [
+      {
+        indeterminates: new Indeterminates(),
+        coefficientNumerator: numerator,
+        coefficientDenominator: denominator,
+      },
+    ];
+
+    return new Polynomial(terms);
+  }
+
+  public negation(): Polynomial {
+    const terms = this.terms.map((t) => ({
+      indeterminates: t.indeterminates,
+      coefficientNumerator: t.coefficientNumerator * -1n,
+      coefficientDenominator: t.coefficientDenominator,
+    }));
+
+    return new Polynomial(terms);
+  }
+
+  public reciprocal(): Polynomial {
+    const terms = this.terms.map((t) => ({
+      indeterminates: t.indeterminates.reciprocal(),
+      coefficientNumerator: t.coefficientDenominator,
+      coefficientDenominator: t.coefficientNumerator,
+    }));
+    return new Polynomial(terms);
+  }
+
+  public constant(): Polynomial {
+    const terms = this.terms.filter((t) => !t.indeterminates.sortedKeys.length);
+    return new Polynomial(terms);
+  }
+
+  public add(rhs: Polynomial): Polynomial {
+    return new Polynomial(
+      Polynomial.simplifyTerms(this.terms.concat(rhs.terms)),
+    );
+  }
+
+  public subtract(rhs: Polynomial): Polynomial {
+    return this.add(rhs.negation());
+  }
+
+  public multiply(rhs: Polynomial): Polynomial {
+    const terms: Term[] = [];
+
+    for (const t1 of this.terms) {
+      for (const t2 of rhs.terms) {
+        const numerator = t1.coefficientNumerator * t2.coefficientNumerator;
+        const denominator =
+          t1.coefficientDenominator * t2.coefficientDenominator;
+        const gcd = findGcd(numerator, denominator);
+
+        terms.push({
+          indeterminates: Indeterminates.multiply(
+            t1.indeterminates,
+            t2.indeterminates,
+          ),
+          coefficientNumerator: numerator / gcd,
+          coefficientDenominator: denominator / gcd,
+        });
+      }
+    }
+
+    return new Polynomial(Polynomial.simplifyTerms(terms));
+  }
+
+  public divide(rhs: Polynomial): Polynomial {
+    return this.multiply(rhs.reciprocal());
+  }
+
+  public toExpression(): Expression {
+    const add: Expression[] = [];
+    for (const t of this.terms) {
+      const coefficient =
+        Number(t.coefficientNumerator) / Number(t.coefficientDenominator);
+
+      const mul: Expression[] = [];
+      if (t.indeterminates.sortedKeys.length) {
+        for (const k of t.indeterminates.sortedKeys) {
+          const w = t.indeterminates.map.get(k);
+          for (let i = Math.abs(w); i > 0; --i) {
+            if (w > 0) mul.push(k);
+            else
+              mul.push(
+                new Expression.Binary("/", new Expression.Literal(1), k),
+              );
+          }
+        }
+
+        if (coefficient !== 1) mul.push(new Expression.Literal(coefficient));
+
+        while (mul.length > 1) {
+          const r = mul.pop();
+          const l = mul.pop();
+          mul.push(new Expression.Binary("*", l, r));
+        }
+        add.push(mul[0]);
+      } else {
+        add.push(new Expression.Literal(coefficient));
+      }
+    }
+
+    while (add.length > 1) {
+      const r = add.pop();
+      const l = add.pop();
+      add.push(new Expression.Binary("+", l, r));
+    }
+
+    if (!add.length) return new Expression.Literal(0);
+    return add[0];
+  }
+}
+
+const SWAPPED_OPS = {
+  "=": "=",
+  "<>": "<>",
+  ">": "<",
+  ">=": "<=",
+  "<": ">",
+  "<=": ">=",
+};
+
+function cartesianProduct<T>(arrays: T[][]): T[][] {
+  return arrays.reduce<T[][]>(
+    (acc, cur) => acc.flatMap((a) => cur.map((item) => [...a, item])),
+    [[]],
+  );
+}
+
+function toPolynomial(e: Expression): Polynomial {
+  if (e instanceof Polynomial) return e;
+  if (e instanceof Expression.Literal) {
+    if (e.value == null) return null;
+    if (typeof e.value === "number") return Polynomial.fromConstant(e.value);
+    if (typeof e.value === "string")
+      return Polynomial.fromConstant(parseFloat(e.value) || 0);
+    if (typeof e.value === "boolean") return Polynomial.fromConstant(+e.value);
+  }
+  return Polynomial.fromIndeterminate(e);
+}
+
+function fromPolynomial(e: Expression): Expression {
+  if (e instanceof Polynomial) return e.toExpression();
+  if (e instanceof Expression.Conditional) return e.map(fromPolynomial);
+  return e;
+}
+
+function normalizeCallback(exp: Expression): Expression {
+  if (exp instanceof Expression.FunctionCall) {
+    if (exp.name === "COALESCE") {
+      let e: Expression = new Expression.Literal(null);
+      for (let i = exp.args.length - 1; i >= 0; --i) {
+        e = new Expression.Conditional(
+          normalizeCallback(new Expression.Unary("IS NOT NULL", exp.args[i])),
+          exp.args[i],
+          e,
+        );
+      }
+      return normalizeCallback(e);
+    }
+  }
+
+  if (exp instanceof Expression.Conditional) {
+    let e = exp;
+
+    if (e.condition instanceof Polynomial)
+      e = new Expression.Conditional(
+        e.condition.toExpression(),
+        e.then,
+        e.otherwise,
+      );
+
+    if (e.then instanceof Expression.Conditional) {
+      e = new Expression.Conditional(
+        Expression.and(e.condition, e.then.condition),
+        e.then.then,
+        new Expression.Conditional(e.condition, e.then.otherwise, e.otherwise),
+      );
+    }
+
+    return e;
+  }
+
+  const combs: [Expression, Expression][][] = [];
+  const callback: (e: Expression) => Expression = (e) => {
+    if (e instanceof Expression.Conditional) {
+      combs[combs.length - 1].push([e.condition, e.then]);
+      callback(e.otherwise);
+    } else {
+      combs[combs.length - 1].push([new Expression.Literal(true), e]);
+    }
+    return e;
+  };
+
+  exp.map((e) => {
+    combs.push([]);
+    callback(e);
+    return e;
+  });
+
+  if (combs.some((a) => a.length > 1)) {
+    let res: Expression = new Expression.Literal(null);
+    for (const p of cartesianProduct(combs).reverse()) {
+      let condition: Expression = new Expression.Literal(true);
+      const e = reduce(
+        normalizeCallback(
+          exp.map((_, i) => {
+            condition = Expression.and(condition, p[i][0]);
+            return p[i][1];
+          }),
+        ),
+      );
+
+      if (!(condition instanceof Expression.Literal))
+        res = new Expression.Conditional(condition, e, res);
+      else if (condition.value) res = e;
+    }
+    return res;
+  }
+
+  if (exp instanceof Expression.Binary) {
+    if (["+", "-", "*", "/"].includes(exp.operator)) {
+      const lhs = toPolynomial(exp.left);
+      const rhs = toPolynomial(exp.right);
+      if (lhs == null || rhs == null) return new Expression.Literal(null);
+      if (exp.operator === "+") return lhs.add(rhs);
+      if (exp.operator === "-") return lhs.subtract(rhs);
+      if (exp.operator === "*") return lhs.multiply(rhs);
+      if (exp.operator === "/") return lhs.divide(rhs);
+    } else if ([">", ">=", "<", "<=", "=", "<>"].includes(exp.operator)) {
+      let lhs: Polynomial, rhs: Polynomial;
+
+      if (exp.left instanceof Polynomial) lhs = exp.left;
+      else if (exp.left instanceof Expression.Literal) {
+        if (exp.left.value == null) return exp.left;
+        else if (typeof exp.left.value === "number")
+          lhs = Polynomial.fromConstant(exp.left.value);
+      }
+
+      if (exp.right instanceof Polynomial) rhs = exp.right;
+      else if (exp.right instanceof Expression.Literal) {
+        if (exp.right.value == null) return exp.right;
+        else if (typeof exp.right.value === "number")
+          rhs = Polynomial.fromConstant(exp.right.value);
+      }
+
+      if (lhs || rhs) {
+        if (!lhs) lhs = Polynomial.fromIndeterminate(exp.left);
+        if (!rhs) rhs = Polynomial.fromIndeterminate(exp.right);
+
+        lhs = lhs.subtract(rhs);
+        rhs = lhs.constant().negation();
+        lhs = lhs.add(rhs);
+
+        if (!lhs.terms.length) {
+          const l = lhs.toExpression() as Expression.Literal;
+          const r = rhs.toExpression() as Expression.Literal;
+
+          if (exp.operator === "=")
+            return new Expression.Literal(l.value === r.value);
+          else if (exp.operator === "<>")
+            return new Expression.Literal(l.value !== r.value);
+          else if (exp.operator === ">")
+            return new Expression.Literal(l.value > r.value);
+          else if (exp.operator === ">=")
+            return new Expression.Literal(l.value >= r.value);
+          else if (exp.operator === "<")
+            return new Expression.Literal(l.value < r.value);
+          else if (exp.operator === "<=")
+            return new Expression.Literal(l.value <= r.value);
+          else throw new Error("Invalid operator");
+        }
+
+        let flipOp = 1;
+
+        const n = lhs.terms[0].coefficientNumerator;
+        const d = lhs.terms[0].coefficientDenominator;
+
+        if (n < 0n || d < 0n) flipOp *= -1;
+
+        const reciprocal = new Polynomial([
+          {
+            indeterminates: new Indeterminates(),
+            coefficientNumerator: d,
+            coefficientDenominator: n,
+          },
+        ]);
+
+        lhs = lhs.multiply(reciprocal);
+        rhs = rhs.multiply(reciprocal);
+
+        const keys = lhs.terms[0].indeterminates.sortedKeys;
+        let invert = lhs.terms[0].indeterminates.map.get(keys[0]) < 0 ? -1 : 0;
+
+        for (const t of lhs.terms)
+          for (const v of t.indeterminates.map.values()) invert += v;
+
+        if (invert < 0) {
+          flipOp *= -1;
+          lhs = lhs.reciprocal();
+          rhs = rhs.reciprocal();
+        }
+
+        if (flipOp > 0) exp = new Expression.Binary(exp.operator, lhs, rhs);
+        else exp = new Expression.Binary(SWAPPED_OPS[exp.operator], lhs, rhs);
+      }
+    }
+  }
+
+  // Restore polynomial expressions
+  exp = exp.map(fromPolynomial);
+
+  return exp;
+}
+
+export default function normalize(exp: Expression): Expression {
+  return fromPolynomial(exp.evaluate(normalizeCallback));
+}

lib/common/expression/pagination.ts

@@ -0,0 +1,160 @@
+import { complement } from "espresso-iisojs";
+import Expression from "../expression.ts";
+import normalize from "./normalize.ts";
+import { Clause, SynthContext } from "./synth.ts";
+import Path from "../path.ts";
+
+type Bookmark = Record<string, null | boolean | number | string>;
+
+type Minterm = number[];
+
+export function toBookmark(
+  sort: Record<string, number>,
+  row: unknown,
+): Bookmark {
+  const bookmark: Bookmark = {};
+  for (const param of Object.keys(sort)) {
+    let v = row[param];
+    if (v != null && typeof v === "object") v = v.value?.[0];
+    bookmark[param] = v;
+  }
+  return bookmark;
+}
+
+export function bookmarkToExpression(
+  bookmark: Bookmark,
+  sort: Record<string, number>,
+): Expression {
+  return Object.entries(sort)
+    .reverse()
+    .reduce(
+      (cur, kv) => {
+        const [param, asc] = kv;
+        const p = new Expression.Parameter(Path.parse(param));
+        const b = new Expression.Literal(bookmark[param]);
+        if (asc < 0) {
+          if (bookmark[param] == null) {
+            return Expression.or(
+              new Expression.Unary("IS NOT NULL", p),
+              Expression.and(new Expression.Unary("IS NULL", p), cur),
+            );
+          }
+          return new Expression.Binary(
+            "OR",
+            new Expression.Binary(">", p, b),
+            new Expression.Binary("AND", new Expression.Binary("=", p, b), cur),
+          );
+        } else {
+          let f: Expression = new Expression.Unary("IS NULL", p);
+          if (bookmark[param] == null) return Expression.and(f, cur);
+          f = Expression.or(f, new Expression.Binary("<", p, b));
+          return Expression.or(
+            f,
+            Expression.and(new Expression.Binary("=", p, b), cur),
+          );
+        }
+      },
+      new Expression.Literal(true) as Expression,
+    );
+}
+
+function getCover(
+  context: SynthContext,
+  minterm: Minterm,
+  allSort: [string, number][],
+): Minterm[] {
+  if (!allSort.length) return [[]];
+  const [param, sort] = allSort[0];
+
+  const cov: Minterm = [];
+  const nextCov: Minterm = [];
+
+  if (sort > 0) {
+    const lhs = new Clause.Exp(new Expression.Parameter(Path.parse(param)));
+    const isNull = context.getVar(new Clause.IsNull(lhs));
+    cov.push((isNull << 2) ^ 2);
+  }
+
+  for (const m of minterm) {
+    const clause = context.getClause(m >>> 2);
+    if (clause instanceof Clause.IsNull) {
+      if (!(clause.operand instanceof Clause.Exp)) continue;
+      if (!(clause.operand.exp instanceof Expression.Parameter)) continue;
+      if (clause.operand.exp.path.toString() !== param) continue;
+      nextCov.push(m);
+      if (sort < 0 && m & 1) cov.push(m, m ^ 1);
+    } else if (clause instanceof Clause.Compare) {
+      if (!(clause.lhs instanceof Clause.Exp)) continue;
+      if (!(clause.lhs.exp instanceof Expression.Parameter)) continue;
+      if (clause.lhs.exp.path.toString() !== param) continue;
+      if (!(m & 1) && sort > 0) continue;
+      nextCov.push(m);
+
+      const negate = (m ^ (m >> 1)) & 1;
+
+      if (sort > 0) {
+        if (
+          (clause.op === "=" && !negate) ||
+          (clause.op === ">" && !negate) ||
+          (clause.op === "<" && negate)
+        ) {
+          const c = new Clause.Compare(clause.lhs, ">", clause.rhs);
+          const v = context.getVar(c);
+          cov.push((v << 2) ^ 3);
+        }
+      } else if (sort < 0) {
+        if (
+          (clause.op === "=" && !negate) ||
+          (clause.op === ">" && negate) ||
+          (clause.op === "<" && !negate)
+        ) {
+          const c = new Clause.Compare(clause.lhs, "<", clause.rhs);
+          const v = context.getVar(c);
+          cov.push((v << 2) ^ 0);
+        }
+      }
+    }
+  }
+
+  const next = getCover(context, minterm, allSort.slice(1));
+
+  return [cov, ...next.map((n) => [...nextCov, ...n])];
+}
+
+export function paginate(
+  fetched: Expression,
+  toFetch: Expression,
+  sort: Record<string, number>,
+): [Expression, Expression] {
+  fetched = normalize(fetched);
+  if (fetched instanceof Expression.Literal && !fetched.value)
+    return [new Expression.Literal(false), toFetch];
+
+  toFetch = normalize(toFetch);
+  if (toFetch instanceof Expression.Literal && !toFetch.value)
+    return [new Expression.Literal(false), toFetch];
+
+  const synth1 = Clause.fromExpression(fetched);
+  const synth2 = Clause.fromExpression(toFetch);
+
+  const context = new SynthContext();
+
+  const expr1Minterms = synth1.getMinterms(context, 0b100);
+  const expr2MintermsC = synth2.getMinterms(context, 0b011);
+
+  const gaps = context.sanitizeMinterms(
+    complement([
+      ...expr1Minterms,
+      ...expr2MintermsC,
+      ...context.getDcSet([...expr1Minterms, ...expr2MintermsC]),
+    ]),
+  );
+
+  const cover = gaps.flatMap((m) => getCover(context, m, Object.entries(sort)));
+  const minterms1 = context.minimize(complement([...cover, ...expr2MintermsC]));
+  const minterms2 = context.minimize(
+    complement([...minterms1, ...expr2MintermsC]),
+  );
+
+  return [context.toExpression(minterms1), context.toExpression(minterms2)];
+}

lib/common/expression/parser.ts

@@ -0,0 +1,557 @@
+import Path from "../path.ts";
+import Expression from "../expression.ts";
+
+export class Cursor {
+  input: string;
+  pos: number;
+  boundryStack: number[][];
+  boundry: number[];
+  charCode: number;
+
+  constructor(input: string) {
+    this.input = input;
+    this.pos = 0;
+    this.boundryStack = [];
+    this.boundry = [];
+    this.charCode = input.charCodeAt(0) || 0;
+  }
+
+  fork(): Cursor {
+    const cursor = new Cursor(this.input);
+    cursor.pos = this.pos;
+    cursor.boundryStack = this.boundryStack.slice();
+    cursor.boundry = this.boundry;
+    cursor.charCode = this.charCode;
+    return cursor;
+  }
+
+  sync(cursor: Cursor): void {
+    this.pos = cursor.pos;
+    this.boundryStack = cursor.boundryStack.slice();
+    this.boundry = cursor.boundry;
+    this.charCode = cursor.charCode;
+  }
+
+  read(cur: Cursor): string {
+    return this.input.slice(this.pos, cur.pos);
+  }
+
+  step(): Cursor {
+    if (this.charCode === 0) return this;
+    ++this.pos;
+    this.charCode = this.input.charCodeAt(this.pos) || 0;
+    if (this.boundry.includes(this.charCode)) this.charCode = 0;
+    return this;
+  }
+
+  walk(callback: (charCode: number) => boolean): Cursor {
+    while (this.charCode && callback(this.charCode)) this.step();
+    return this;
+  }
+
+  descend(chars: number[], override = true): void {
+    this.boundryStack.push(this.boundry);
+    if (override) this.boundry = chars;
+    else this.boundry = [...this.boundry, ...chars];
+    this.charCode = this.input.charCodeAt(this.pos) || 0;
+    if (this.boundry.includes(this.charCode)) this.charCode = 0;
+  }
+
+  ascend(): void {
+    if (!this.boundryStack.length) throw new Error("Unmatched boundry");
+    this.boundry = this.boundryStack.pop();
+    this.charCode = this.input.charCodeAt(this.pos) || 0;
+  }
+
+  skipwhitespace(): Cursor {
+    return this.walk((c) => c <= 32);
+  }
+}
+
+const CHAR_SINGLE_QUOTE = 39;
+const CHAR_DOUBLE_QUOTE = 34;
+const CHAR_OPEN_PAREN = 40;
+const CHAR_CLOSE_PAREN = 41;
+const CHAR_COMMA = 44;
+const CHAR_PERIOD = 46;
+const CHAR_COLON = 58;
+const CHAR_OPEN_BRACKET = 91;
+const CHAR_BACKSLASH = 92;
+const CHAR_CLOSE_BRACKET = 93;
+
+const BINARY_OPERATORS = [
+  ">=",
+  "<=",
+  "<>",
+  "=",
+  ">",
+  "<",
+  "LIKE",
+  "NOT LIKE",
+  "AND",
+  "OR",
+  "*",
+  "/",
+  "%",
+  "||",
+  "-",
+  "+",
+];
+
+const PRECEDENCE = {
+  OR: 10,
+  AND: 11,
+  NOT: 12,
+  "=": 20,
+  "<>": 20,
+  ">": 20,
+  ">=": 20,
+  "<": 20,
+  "<=": 20,
+  LIKE: 20,
+  "NOT LIKE": 20,
+  "IS NULL": 20,
+  "IS NOT NULL": 20,
+  "||": 30,
+  "+": 31,
+  "-": 31,
+  "*": 32,
+  "/": 32,
+  "%": 32,
+};
+
+function* range(s: number, e: number): Generator<number> {
+  for (let i = s; i < e; i++) yield i;
+}
+
+const PATH_CHARS = new Set<number>([
+  ...range(65, 91),
+  ...range(97, 123),
+  ...range(48, 58),
+  95,
+  45,
+  42,
+  123,
+  125,
+]);
+
+function findOperator(cursor: Cursor): string {
+  cursor.skipwhitespace();
+  let found = "";
+  let foundCursor = cursor;
+  let operators = [...BINARY_OPERATORS, "IS NULL", "IS NOT NULL"];
+  for (let i = 0; operators.length && cursor.charCode; ++i) {
+    let c = cursor.charCode;
+    cursor.step();
+    if (c >= 97 && c <= 122) c -= 32;
+    if (c <= 32) {
+      cursor.skipwhitespace();
+      c = 32;
+    }
+    operators = operators.filter((o) => {
+      if (o.charCodeAt(i) !== c) return false;
+      if (o.length === i + 1) {
+        found = o;
+        foundCursor = cursor.fork();
+      }
+      return true;
+    });
+  }
+
+  if (found) cursor.sync(foundCursor);
+  return found;
+}
+
+// Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
+function interpretEscapes(str): string {
+  const escapes = {
+    b: "\b",
+    f: "\f",
+    n: "\n",
+    r: "\r",
+    t: "\t",
+  };
+  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/g, (_, escape) => {
+    const type = escape.charAt(0);
+    const hex = escape.slice(1);
+    if (type === "u") return String.fromCharCode(parseInt(hex, 16));
+
+    if (escapes.hasOwnProperty(type)) return escapes[type];
+
+    return type;
+  });
+}
+
+export function parseExpression(cursor: Cursor, presedence = 0): Expression {
+  cursor.skipwhitespace();
+  const char = cursor.charCode;
+  let lhs: Expression;
+  if (char === CHAR_OPEN_PAREN) {
+    cursor.step();
+    cursor.descend([CHAR_CLOSE_PAREN]);
+    lhs = parseExpression(cursor, 0);
+    cursor.ascend();
+
+    cursor.skipwhitespace();
+    if (cursor.charCode !== CHAR_CLOSE_PAREN)
+      throw new Error("Expected ')'" + cursor.pos + " " + cursor.charCode);
+    cursor.step();
+  } else if (char === CHAR_DOUBLE_QUOTE) {
+    cursor.step();
+    const cursor2 = cursor.fork();
+    for (
+      cursor2.descend([]);
+      cursor2.charCode !== CHAR_DOUBLE_QUOTE;
+      cursor2.step()
+    ) {
+      if (!cursor2.charCode) throw new Error("Unterminated string");
+      if (cursor2.charCode === CHAR_BACKSLASH) cursor2.step();
+    }
+    cursor2.ascend();
+    const str = cursor.read(cursor2);
+    cursor.sync(cursor2);
+    cursor.step();
+    return new Expression.Literal(interpretEscapes(str));
+  } else if (char === CHAR_SINGLE_QUOTE) {
+    cursor.step();
+    const cursor2 = cursor.fork();
+    for (
+      cursor2.descend([]);
+      cursor2.charCode !== CHAR_SINGLE_QUOTE;
+      cursor2.step()
+    ) {
+      if (!cursor2.charCode) throw new Error("Unterminated string");
+      if (cursor2.charCode === CHAR_BACKSLASH) cursor2.step();
+    }
+    cursor2.ascend();
+    const str = cursor.read(cursor2);
+    cursor.sync(cursor2);
+    cursor.step();
+    return new Expression.Literal(str.replaceAll("''", "'"));
+  } else {
+    const cursor2 = cursor.fork();
+    cursor.walk(
+      (c) => c > 32 && c !== CHAR_OPEN_PAREN && c !== CHAR_OPEN_BRACKET,
+    );
+    const token = cursor2.read(cursor);
+    if (!token) throw new Error("Invalid expression");
+    if (/^true$/i.test(token)) lhs = new Expression.Literal(true);
+    else if (/^false$/i.test(token)) lhs = new Expression.Literal(false);
+    else if (/^null$/i.test(token)) lhs = new Expression.Literal(null);
+    else if (/^-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?$/.test(token))
+      lhs = new Expression.Literal(Number(token));
+    else if (/^not$/i.test(token)) {
+      lhs = new Expression.Unary(
+        "NOT",
+        parseExpression(cursor, PRECEDENCE["NOT"]),
+      );
+    } else if (/^case$/i.test(token)) {
+      const pairs: [Expression, Expression][] = [];
+      for (;;) {
+        cursor.skipwhitespace();
+        const whenStr = cursor.fork().read(cursor.walk((c) => c > 32));
+        if (/^else$/i.test(whenStr)) {
+          lhs = parseExpression(cursor);
+          continue;
+        }
+        if (/^end$/i.test(whenStr)) break;
+        else if (lhs) throw new Error("Expected END");
+        if (!/^when$/i.test(whenStr)) throw new Error("Expected WHEN");
+        const condition = parseExpression(cursor);
+        cursor.skipwhitespace();
+        const thenStr = cursor.fork().read(cursor.walk((c) => c > 32));
+        if (!/^then$/i.test(thenStr)) throw new Error("Expected THEN");
+        const then = parseExpression(cursor);
+        pairs.push([condition, then]);
+      }
+      if (!lhs) lhs = new Expression.Literal(null);
+      while (pairs.length) {
+        const [condition, then] = pairs.pop();
+        lhs = new Expression.Conditional(condition, then, lhs);
+      }
+    } else if (cursor.charCode === CHAR_OPEN_PAREN) {
+      cursor.step();
+      cursor.descend([CHAR_CLOSE_PAREN]);
+      cursor.skipwhitespace();
+      const args = [] as Expression[];
+      while (cursor.charCode) {
+        cursor.descend([CHAR_COMMA], false);
+        const e = parseExpression(cursor);
+        args.push(e);
+        cursor.ascend();
+        cursor.skipwhitespace();
+        if ((cursor.charCode as number) !== CHAR_COMMA) break;
+        cursor.step();
+      }
+      cursor.ascend();
+      cursor.step();
+      lhs = new Expression.FunctionCall(token, args);
+    } else {
+      const p = parsePath(cursor2);
+      lhs = new Expression.Parameter(p);
+      cursor.sync(cursor2);
+    }
+  }
+
+  for (;;) {
+    cursor.skipwhitespace();
+    const cursor2 = cursor.fork();
+    const op = findOperator(cursor2);
+    const p = PRECEDENCE[op];
+    if (p <= presedence) return lhs;
+    if (op === "IS NULL") {
+      lhs = new Expression.Unary("IS NULL", lhs);
+    } else if (op === "IS NOT NULL") {
+      lhs = new Expression.Unary("IS NOT NULL", lhs);
+    } else if (BINARY_OPERATORS.includes(op)) {
+      lhs = new Expression.Binary(op, lhs, parseExpression(cursor2, p));
+    } else if (op) throw new Error("Unrecognized operator: " + op);
+    else break;
+    cursor.sync(cursor2);
+  }
+  return lhs;
+}
+
+// Parse old-format alias value: unquoted, double-quoted, or single-quoted.
+// Returns the parsed string value. Cursor is advanced past the value.
+function parseOldAliasValue(cursor: Cursor): string {
+  cursor.walk((c) => c <= 32); // skip leading whitespace
+  const c = cursor.charCode;
+  if (c === CHAR_DOUBLE_QUOTE) {
+    // Double-quoted: use JSON.parse semantics (handles \", \n, \uXXXX etc.)
+    // Descend with no boundaries so commas/brackets inside quotes are not
+    // treated as terminators.
+    const start = cursor.pos;
+    cursor.step();
+    cursor.descend([]);
+    while (cursor.charCode) {
+      if (cursor.charCode === CHAR_BACKSLASH) {
+        cursor.step();
+        if (!cursor.charCode) break;
+      } else if (cursor.charCode === CHAR_DOUBLE_QUOTE) {
+        cursor.ascend();
+        cursor.step();
+        return JSON.parse(cursor.input.slice(start, cursor.pos)) as string;
+      }
+      cursor.step();
+    }
+    throw new Error("Unterminated string");
+  }
+  if (c === CHAR_SINGLE_QUOTE) {
+    // Single-quoted: strip quotes, no escape processing
+    cursor.step();
+    cursor.descend([]);
+    const start = cursor.pos;
+    while (cursor.charCode && cursor.charCode !== CHAR_SINGLE_QUOTE)
+      cursor.step();
+    if (!cursor.charCode) throw new Error("Unterminated string");
+    const value = cursor.input.slice(start, cursor.pos);
+    cursor.ascend();
+    cursor.step();
+    return value;
+  }
+  // Unquoted: read until , or ] (respecting boundary), trim result
+  const start = cursor.pos;
+  cursor.walk(() => true);
+  return cursor.input.slice(start, cursor.pos).trim();
+}
+
+// Parse old-format alias content: key:value,key:value,...
+// Cursor should be positioned at the start of bracket content
+// (after '[', with ']' set as boundary).
+// Returns an Expression: Binary("AND", ...) chain of Binary("=", param, literal).
+// Empty content returns Literal(true).
+function parseOldAlias(cursor: Cursor): Expression {
+  cursor.skipwhitespace();
+  if (!cursor.charCode) return new Expression.Literal(true);
+
+  let result: Expression | null = null;
+  for (;;) {
+    cursor.skipwhitespace();
+    // Parse key as a path (supports nested aliases via parsePath).
+    // Add colon and comma as boundaries so the path stops there.
+    cursor.descend([CHAR_COLON, CHAR_COMMA], false);
+    const keyPath = parsePath(cursor);
+    cursor.ascend();
+
+    // After ascend, check for the colon separator
+    cursor.skipwhitespace();
+    if ((cursor.charCode as number) !== CHAR_COLON)
+      throw new Error("Expected ':'");
+    cursor.step();
+
+    // Parse value (reads until boundary , or ])
+    cursor.descend([CHAR_COMMA], false);
+    const value = parseOldAliasValue(cursor);
+    cursor.ascend();
+
+    const pair = new Expression.Binary(
+      "=",
+      new Expression.Parameter(keyPath),
+      new Expression.Literal(value),
+    );
+    result = result ? new Expression.Binary("AND", result, pair) : pair;
+
+    cursor.skipwhitespace();
+    if ((cursor.charCode as number) !== CHAR_COMMA) break;
+    cursor.step();
+  }
+  return result;
+}
+
+export function parsePath(cur: Cursor): Path {
+  const segments: (string | Expression)[] = [];
+  let colon = 0;
+  const cur2 = cur.fork();
+  for (;;) {
+    let char = cur2.charCode;
+    let exp: Expression;
+
+    if (char === CHAR_OPEN_BRACKET) {
+      if (cur2.pos !== cur.pos) throw new Error("Invalid path");
+      cur2.step();
+      cur2.descend([CHAR_CLOSE_BRACKET]);
+      // Try parsing as expression first
+      const cur3 = cur2.fork();
+      let err: unknown;
+      try {
+        exp = parseExpression(cur3);
+      } catch (e) {
+        err = e;
+      }
+      // Fall back to old alias format if not a valid expression or produced
+      // a bare Parameter (old format like [a:1] mis-parsed as a path).
+      if (err || exp instanceof Expression.Parameter) {
+        exp = parseOldAlias(cur2);
+      } else {
+        cur2.sync(cur3);
+      }
+      cur2.ascend();
+      char = cur2.charCode;
+      if (char !== CHAR_CLOSE_BRACKET) throw new Error("Expected ']'");
+      char = cur2.step().charCode;
+    }
+
+    if (!PATH_CHARS.has(char)) {
+      if (cur2.pos <= cur.pos) throw new Error("Invalid path");
+
+      if (char === CHAR_COLON) {
+        if (colon) throw new Error("Multiple colons");
+        colon = segments.length + 1;
+      }
+
+      if (exp) segments.push(exp);
+      else segments.push(cur.read(cur2));
+
+      if (char !== CHAR_PERIOD && char !== CHAR_COLON) break;
+      cur2.step();
+      cur.sync(cur2);
+      continue;
+    }
+
+    if (exp) throw new Error("Invalid path");
+    cur2.step();
+  }
+  cur.sync(cur2);
+
+  if (colon) colon = segments.length - colon;
+  return new Path(segments, colon);
+}
+
+export function stringifyExpression(exp: Expression, level = 0): string {
+  function wrap(e: string, op: string): string {
+    if (PRECEDENCE[op] <= level) return `(${e})`;
+    else return e;
+  }
+
+  if (exp instanceof Expression.Literal) {
+    if (exp.value == null) return "NULL";
+    if (exp.value === true) return "TRUE";
+    if (exp.value === false) return "FALSE";
+    return JSON.stringify(exp.value);
+  } else if (exp instanceof Expression.Unary) {
+    if (exp.operator === "NOT") {
+      return wrap(
+        `NOT ${stringifyExpression(exp.operand, PRECEDENCE[exp.operator])}`,
+        "NOT",
+      );
+    } else if (exp.operator === "IS NULL" || exp.operator === "IS NOT NULL") {
+      return wrap(
+        `${stringifyExpression(exp.operand, PRECEDENCE[exp.operator])} ${exp.operator}`,
+        exp.operator,
+      );
+    }
+  } else if (exp instanceof Expression.Binary) {
+    const op = exp.operator;
+    if (!(op in PRECEDENCE)) throw new Error("Invalid operator");
+    return wrap(
+      `${stringifyExpression(exp.left, PRECEDENCE[op] - 1)} ${exp.operator} ${stringifyExpression(exp.right, PRECEDENCE[op])}`,
+      op,
+    );
+  } else if (exp instanceof Expression.Parameter) {
+    return exp.path.toString();
+  } else if (exp instanceof Expression.FunctionCall) {
+    return `${exp.name}(${exp.args.map((a) => stringifyExpression(a)).join(", ")})`;
+  } else if (exp instanceof Expression.Conditional) {
+    let str = `CASE WHEN ${stringifyExpression(exp.condition)} THEN ${stringifyExpression(exp.then)}`;
+    if (
+      exp.otherwise instanceof Expression.Literal &&
+      exp.otherwise.value == null
+    )
+      str += " END";
+    else if (exp.otherwise instanceof Expression.Conditional)
+      str += stringifyExpression(exp.otherwise).slice(4);
+    else str += ` ELSE ${stringifyExpression(exp.otherwise)} END`;
+    return str;
+  }
+
+  throw new Error("Invalid expression");
+}
+
+export function parseLikePattern(pat: string, esc: string): string[] {
+  const chars = pat.split("");
+
+  for (let i = 0; i < chars.length; ++i) {
+    const c = chars[i];
+    if (c === esc) {
+      chars[i] = chars[i + 1] || "";
+      chars[i + 1] = "";
+    } else if (c === "_") {
+      chars[i] = "\\_";
+    } else if (c === "%") {
+      chars[i] = "\\%";
+      while (chars[i + 1] === "%") chars[++i] = "";
+    }
+  }
+  return chars.filter((c) => c);
+}
+
+export function likePatternToRegExp(pat: string, esc = "", flags = ""): RegExp {
+  const convChars = {
+    "-": "\\-",
+    "/": "\\/",
+    "\\": "\\/",
+    "^": "\\^",
+    $: "\\$",
+    "*": "\\*",
+    "+": "\\+",
+    "?": "\\?",
+    ".": "\\.",
+    "(": "\\(",
+    ")": "\\)",
+    "|": "\\|",
+    "[": "\\[",
+    "]": "\\]",
+    "{": "\\{",
+    "}": "\\}",
+    "\\%": ".*",
+    "\\_": ".",
+  };
+  let chars = parseLikePattern(pat, esc);
+  if (!chars.length) return new RegExp("^$", flags);
+  chars = chars.map((c) => convChars[c] || c);
+  chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
+  const l = chars.length - 1;
+  chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
+  return new RegExp(chars.join(""), flags);
+}

lib/common/memoize.ts

@@ -1,22 +1,3 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 let cache1 = new Map();
 let cache2 = new Map();
 const keys = new WeakMap();
@@ -39,7 +20,7 @@ function getKey(obj): string {
   return k;
 }
 
-export default function memoize<T extends Function>(func: T): T {
+export default function memoize<T extends (...args: any[]) => any>(func: T): T {
   const funcKey = getKey(func);
   return ((...args) => {
     const key = JSON.stringify(args.map(getKey)) + funcKey;
@@ -47,9 +28,18 @@ export default function memoize<T extends Function>(func: T): T {
     if (cache1.has(key)) return cache1.get(key);
 
     let r;
-    if (cache2.has(key)) r = cache2.get(key);
-    else r = func(...args);
-    cache1.set(key, r);
+    if (cache2.has(key)) {
+      cache1.set(key, (r = cache2.get(key)));
+    } else {
+      cache1.set(key, (r = func(...args)));
+      // Evict rejected promises
+      if (r instanceof Promise) {
+        r.catch(() => {
+          cache1.delete(key);
+          cache2.delete(key);
+        });
+      }
+    }
     return r;
   }) as any;
 }

lib/common/path-set.ts

@@ -1,59 +1,32 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./path";
+import Path from "./path.ts";
 
 export default class PathSet {
-  private lengthIndex: Set<Path>[];
-  private fragmentIndex: Map<string, Set<Path>>[];
-  private stringIndex: Map<string, Path>;
+  private paramSegmentIndex: Map<string, Set<Path>>[] = [];
+  private attrSegmentIndex: Map<string, Set<Path>>[] = [];
+  private stringIndex: Map<string, Path> = new Map();
 
-  public constructor() {
-    this.lengthIndex = [];
-    this.fragmentIndex = [];
-    this.stringIndex = new Map();
-  }
+  public constructor() {}
 
-  public get depth(): number {
-    return this.lengthIndex.length;
-  }
-
-  public add(path: Path): Path {
+  public add(pathStr: string): Path {
+    let path: Path = this.get(pathStr);
+    if (path) return path;
+    path = Path.parse(pathStr);
     if (path.alias) throw new Error("PathSet does not support aliased paths");
-    const p = this.get(path);
-
-    if (p) return p;
 
     this.stringIndex.set(path.toString(), path);
 
-    while (this.lengthIndex.length <= path.length) {
-      this.lengthIndex.push(new Set());
-      // fragmentIndex is one less than lengthIndex
-      if (this.lengthIndex.length > 1) this.fragmentIndex.push(new Map());
-    }
+    while (this.paramSegmentIndex.length < path.paramLength)
+      this.paramSegmentIndex.push(new Map());
 
-    const lengthIndex = this.lengthIndex[path.length];
-    lengthIndex.add(path);
+    while (this.attrSegmentIndex.length < path.attrLength)
+      this.attrSegmentIndex.push(new Map());
 
     for (let i = 0; i < path.length; ++i) {
       const fragment = path.segments[i] as string;
-      const fragmentIndex = this.fragmentIndex[i];
+      const fragmentIndex =
+        i < path.paramLength
+          ? this.paramSegmentIndex[i]
+          : this.attrSegmentIndex[i - path.paramLength];
 
       let fragmentIndexSet = fragmentIndex.get(fragment);
       if (!fragmentIndexSet) {
@@ -67,64 +40,76 @@ export default class PathSet {
     return path;
   }
 
-  public get(path: Path): Path {
-    return this.stringIndex.get(path.toString()) || null;
+  public get(path: string): Path {
+    return this.stringIndex.get(path);
   }
 
-  public find(
+  public findCompat(
     path: Path,
-    superset: boolean = false,
-    subset: boolean = false,
-    depth: number = path.length
+    superset = false,
+    subset = false,
+    depth = path.length,
   ): Path[] {
+    if (path.attrLength)
+      throw new Error("findCompat() does not support attribute paths");
+
+    depth = Math.min(31, depth);
+    const paramMask = (1 << path.paramLength) - 1;
+    let mask = ((0b10 << depth) - 1) & ~paramMask;
+    if (superset) mask |= ~path.wildcard & paramMask;
+    if (subset) mask |= path.wildcard;
+    return this.find(path, mask, 1);
+  }
+
+  public find(path: Path, paramMask: number, attrMask: number): Path[] {
     if (path.alias) throw new Error("PathSet does not support aliased paths");
+    if (path.paramLength > this.paramSegmentIndex.length) return [];
+    if (path.attrLength > this.attrSegmentIndex.length) return [];
 
-    const len = path.length;
+    const emptySet: Set<Path> = new Set();
 
-    if (!superset && depth === len && (!subset || !path.wildcard)) {
-      const p = this.get(path);
-      return p ? [p] : [];
+    const indexes: [Set<Path>, Set<Path>][] = [];
+
+    const paramLengthMask = (1 << path.paramLength) - 1;
+    const attrLengthMask = (1 << path.attrLength) - 1;
+
+    const mask = (paramMask & paramLengthMask) | (attrMask << path.paramLength);
+
+    for (const [i, s] of path.segments.entries()) {
+      const b = 1 << i;
+      const m = mask & b;
+      const w = b & path.wildcard;
+      if (w && m) continue;
+      const idxSet =
+        i < path.paramLength
+          ? this.paramSegmentIndex[i]
+          : this.attrSegmentIndex[i - path.paramLength];
+      const idx1 = idxSet.get(s as string) || emptySet;
+      let idx2 = emptySet;
+      if (m) idx2 = idxSet.get("*") || emptySet;
+      indexes.push([idx1, idx2]);
     }
 
-    const lengthIndex = this.lengthIndex.slice(len, depth + 1);
-    if (!lengthIndex.length) return [];
+    indexes.sort((a, b) => a[0].size + a[1].size - (b[0].size + b[1].size));
 
-    let res;
-    for (let i = len - 1; i >= 0; --i) {
-      let fragmentIndexSet2;
-      const fragmentIndex = this.fragmentIndex[i];
+    let res: Path[];
 
-      if ((path.wildcard >> i) & 1) {
-        if (subset) continue;
-      } else if (superset) {
-        fragmentIndexSet2 = fragmentIndex.get("*");
-      }
+    if (!indexes.length) res = [...this.stringIndex.values()];
+    else res = [...indexes[0][0], ...indexes[0][1]];
 
-      const fragment = path.segments[i] as string;
-      const fragmentIndexSet1 = fragmentIndex.get(fragment);
+    const paramCover = ~paramLengthMask & paramMask;
+    const attrCover = ~attrLengthMask & attrMask;
 
-      if (!fragmentIndexSet1) {
-        if (!fragmentIndexSet2) return [];
-        if (!res) res = [...fragmentIndexSet2];
-        else res = res.filter(r => fragmentIndexSet2.has(r));
-      } else if (!fragmentIndexSet2) {
-        if (!res) res = [...fragmentIndexSet1];
-        else res = res.filter(r => fragmentIndexSet1.has(r));
-      } else {
-        if (!res) {
-          res = [...fragmentIndexSet1, ...fragmentIndexSet2];
-        } else {
-          res = res.filter(
-            r => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
-          );
-        }
-      }
-      if (!res.length) return res;
+    res = res.filter(
+      (p) =>
+        (1 << p.paramLength) & paramCover && (1 << p.attrLength) & attrCover,
+    );
+
+    for (let i = 1; i < indexes.length; ++i) {
+      const [idx1, idx2] = indexes[i];
+      res = res.filter((p) => idx1.has(p) || idx2.has(p));
     }
 
-    if (!res) res = [].concat(...lengthIndex.map(a => [...a]));
-    else res = res.filter(r => lengthIndex.some(a => a.has(r)));
-
     return res;
   }
 }

lib/common/path.ts

@@ -1,126 +1,31 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
+import Expression from "./expression.ts";
+import { Cursor, parsePath } from "./expression/parser.ts";
 
-type Segments = (string | Alias)[];
-type Alias = [Path, string][];
+type Segments = (string | Expression)[];
 
 let cache1 = new Map<string, Path>();
 let cache2 = new Map<string, Path>();
 
 export default class Path {
-  public readonly segments: Segments;
-  public readonly wildcard: number;
-  public readonly alias: number;
-  protected _string: string;
-  protected _stringIndex: number[];
+  declare public readonly segments: Segments;
+  declare public readonly colon: number;
+  declare public readonly wildcard: number;
+  declare public readonly alias: number;
+  declare protected _string: string;
+  declare protected _stringIndex: number[];
 
-  protected static parseAlias(
-    pattern: string,
-    index: number
-  ): { index: number; alias: Alias } {
-    const alias: Alias = [];
-    while (index < pattern.length && pattern[index] !== "]") {
-      const { index: idx, segments } = Path.parsePath(pattern, index);
-      let j = (index = idx + 1);
-      while (pattern[j] !== "]" && pattern[j] !== ",") {
-        if (pattern[j] === '"' && index === j) {
-          ++j;
-          while (pattern[j] !== '"' || pattern[j - 1] === "\\") {
-            if (++j >= pattern.length)
-              throw new Error("Invalid alias expression");
-          }
-        }
-        if (++j >= pattern.length) throw new Error("Invalid alias expression");
-      }
-
-      let value = pattern.slice(index, j).trim();
-      index = j;
-      if (value[0] === '"') {
-        try {
-          value = JSON.parse(value);
-        } catch (error) {
-          throw new Error("Invalid alias expression");
-        }
-      }
-
-      alias.push([new Path(segments), value]);
-      if (pattern[index] === ",") ++index;
-    }
-
-    // Ensure identical expressions have idential string representation
-    alias.sort((a, b) => {
-      if (a[0].toString() > b[0].toString()) return 1;
-      else if (a[0].toString() < b[0].toString()) return -1;
-      else if (a[1] > b[1]) return 1;
-      else if (a[1] < b[1]) return -1;
-      else return 0;
-    });
-
-    Object.freeze(alias);
-    return { index, alias };
-  }
-
-  protected static parsePath(
-    pattern: string,
-    index: number
-  ): { index: number; segments: Segments } {
-    const segments = [];
-    // Colon separator is needed for parseAlias
-    if (index < pattern.length && pattern[index] !== ":") {
-      for (;;) {
-        if (pattern[index] === "[") {
-          const { index: idx, alias } = Path.parseAlias(pattern, index + 1);
-          index = idx + 1;
-          segments.push(alias);
-        } else {
-          const j = index;
-          while (
-            index < pattern.length &&
-            pattern[index] !== ":" &&
-            pattern[index] !== "."
-          )
-            ++index;
-          const s = pattern.slice(j, index).trim();
-          segments.push(s);
-        }
-
-        if (index >= pattern.length || pattern[index] === ":") break;
-        else if (pattern[index] !== ".")
-          throw new Error("Invalid alias expression");
-        ++index;
-      }
-    }
+  constructor(segments: Segments, colon: number) {
+    if (!(colon <= segments.length)) throw new Error("Invalid path");
+    if (segments.length > 32) throw new Error("Path too long");
 
     Object.freeze(segments);
-    return { index, segments };
-  }
 
-  protected constructor(segments: Segments) {
     let alias = 0;
     let wildcard = 0;
     const arr = segments.map((s, i) => {
-      if (Array.isArray(s)) {
+      if (s instanceof Expression) {
         alias |= 1 << i;
-        const parts = s.map(
-          al => `${al[0].toString()}:${JSON.stringify(al[1])}`
-        );
-        return `[${parts.join(",")}]`;
+        return `[${s.toString()}]`;
       } else if (s === "*") {
         wildcard |= 1 << i;
       }
@@ -131,22 +36,27 @@ export default class Path {
     const stringIndex = arr.map((s, i) => (offset += s.length) + i);
 
     this.segments = segments;
+    this.colon = colon;
     this.wildcard = wildcard;
     this.alias = alias;
-    this._string = arr.join(".");
+    if (!colon) this._string = arr.join(".");
+    else
+      this._string =
+        arr.slice(0, -colon).join(".") + ":" + arr.slice(-colon).join(".");
     this._stringIndex = stringIndex;
   }
 
-  public static parse(str: string): Path {
-    let path = cache1.get(str);
+  public static parse(input: string): Path {
+    let path = cache1.get(input);
     if (!path) {
-      path = cache2.get(str);
+      path = cache2.get(input);
       if (!path) {
-        const { segments } = Path.parsePath(str, 0);
-        path = new Path(segments);
-        if (path.toString() !== str) cache1.set(path.toString(), path);
+        const cursor = new Cursor(input);
+        path = parsePath(cursor);
+        if (cursor.charCode) throw new Error("Unexpected character");
+        if (path.toString() !== input) cache1.set(path.toString(), path);
       }
-      cache1.set(str, path);
+      cache1.set(input, path);
     }
     return path;
   }
@@ -155,33 +65,43 @@ export default class Path {
     return this.segments.length;
   }
 
+  public get paramLength(): number {
+    return this.segments.length - this.colon;
+  }
+
+  public get attrLength(): number {
+    return this.colon;
+  }
+
   public toString(): string {
     return this._string;
   }
 
-  public slice(start: number = 0, end: number = this.segments.length): Path {
+  public slice(start = 0, end: number = this.segments.length): Path {
     if (start < 0) start = Math.max(0, this.segments.length + start);
     if (end < 0) end = Math.max(0, this.segments.length + end);
 
-    let str;
-    if (start >= end) {
-      str = "";
-    } else {
-      const i1 = start > 0 ? this._stringIndex[start - 1] + 1 : 0;
-      const i2 =
-        end <= this.segments.length
-          ? this._stringIndex[end - 1]
-          : this._string.length;
-      str = this._string.slice(i1, i2);
-    }
+    if (start >= end) return Path.root;
+
+    let i1 = start > 0 ? this._stringIndex[start - 1] + 1 : 0;
+    // Include the ":" when slicing exactly at the colon boundary
+    if (this.colon && start === this.segments.length - this.colon) --i1;
+    const i2 =
+      end <= this.segments.length
+        ? this._stringIndex[end - 1]
+        : this._string.length;
+    const str = this._string.slice(i1, i2);
 
     let path = cache1.get(str);
     if (!path) {
       path = cache2.get(str);
       if (!path) {
         const segments = this.segments.slice(start, end);
-        Object.freeze(segments);
-        path = new Path(segments);
+        const colon =
+          start <= this.segments.length - this.colon
+            ? Math.max(0, this.colon - this.segments.length + end)
+            : 0;
+        path = new Path(segments, colon);
       }
       cache1.set(str, path);
     }
@@ -193,15 +113,28 @@ export default class Path {
     if (!path2._string) return this;
     else if (!this._string) return path2;
 
-    const str = `${this._string}.${path2._string}`;
+    if (this.colon && path2.colon && path2.colon < path2.segments.length)
+      throw new Error("Invalid path");
+
+    const colon = this.colon ? this.colon + path2.segments.length : path2.colon;
+
+    let str;
+    if (this.colon && path2.colon === path2.segments.length) {
+      // Right is all-colon; strip its ":" prefix and join with "."
+      str = `${this._string}.${path2._string.slice(1)}`;
+    } else if (path2.colon === path2.segments.length) {
+      // Left has no colon; right is all-colon; concatenate directly
+      str = `${this._string}${path2._string}`;
+    } else {
+      str = `${this._string}.${path2._string}`;
+    }
 
     let path = cache1.get(str);
     if (!path) {
       path = cache2.get(str);
       if (!path) {
         const segments = this.segments.concat(path2.segments);
-        Object.freeze(segments);
-        path = new Path(segments);
+        path = new Path(segments, colon);
       }
       cache1.set(str, path);
     }
@@ -211,21 +144,30 @@ export default class Path {
 
   public stripAlias(): Path {
     if (!this.alias) return this;
-    const segments = this.segments.map(s => (Array.isArray(s) ? "*" : s));
-    const str = segments.join(".");
+    const segments = this.segments.map((s) =>
+      s instanceof Expression ? "*" : s,
+    );
+    let str: string;
+    if (!this.colon) str = segments.join(".");
+    else
+      str =
+        segments.slice(0, -this.colon).join(".") +
+        ":" +
+        segments.slice(-this.colon).join(".");
 
     let path = cache1.get(str);
     if (!path) {
       path = cache2.get(str);
       if (!path) {
-        Object.freeze(segments);
-        path = new Path(segments);
+        path = new Path(segments, this.colon);
       }
       cache1.set(str, path);
     }
 
     return path;
   }
+
+  static root = new Path([], 0);
 }
 
 const interval = setInterval(() => {

lib/common/sat-solver.ts

@@ -1,175 +0,0 @@
-/**
- * Naive DPLL solver for propositional logic adapted from
- * https://github.com/tammet/logictools
- *
- * Origianl code by Tanel Tammet released under the MIT license.
- *
- * Check whether a clause set is satisfiable or contradictory, using a naive
- * implementation of the dpll (Davis-Putnam-Logemann-Loveland) procedure
- * combining truth table search with the resolution-like unit propagation.
- *
- * The standard pure-literal rule is not implemented in this naive version,
- * since it is not really crucial for the algorithm.
- *
- * The code is very similar to the pure table solver, except we use a
- * unit-propagating, derived-varlist-returing satisfiableAt.
- */
-
-/* eslint-disable @typescript-eslint/no-use-before-define */
-
-function satisfiableAt(clauses, varVals, varnr, val): boolean {
-  if (varnr !== 0) varVals[varnr] = val;
-
-  const propRes = unitPropagate(clauses, varVals);
-  if (propRes === 1) {
-    varVals[varnr] = 0;
-    return true;
-  }
-
-  // if result false (-1) unitPropagate restores the varVals state itself
-  if (propRes === -1) {
-    varVals[varnr] = 0;
-    return false;
-  }
-
-  // find next unassigned var to split
-  let nextVar = 0;
-  for (let i = 1; i < varVals.length; i++) {
-    if (varVals[i] === 0) {
-      nextVar = i;
-      break;
-    }
-  }
-
-  if (nextVar === 0) throw new Error("Error in satisfiableAt");
-
-  if (
-    satisfiableAt(clauses, varVals, nextVar, 1) ||
-    satisfiableAt(clauses, varVals, nextVar, -1)
-  )
-    // not necessary to restore split var: search is halted anyway
-    return true;
-
-  // restore split var
-  varVals[nextVar] = 0;
-
-  // restore derived vars
-  for (let i = 0; i < propRes.length; i++) varVals[propRes[i]] = 0;
-
-  return false;
-}
-
-/**
- * Check whether clauses are satisfiable under the varVals assignment, without
- * search.
- *
- * Iterate through all clauses and check whether
- *
- *  - all literals in the clause are assigned false: undo derived units (set
- *    values in varVals to 0) and return false immediately
- *
- *  - a clause contains a literal assigned true: skip further actions on this
- *    clause
- *
- *  - a single literal in the clause is unassigned, all the rest are assigned
- *    false: derive this literal as a unit clause.
- *
- * In case at least one literal is derived during the process above, iterate
- * the whole process until no more new literals are derived.
- *
- * Returns:
- *
- *   1: all clauses were assigned true, hence the clause set is satisfiable.
- *   The whole search will terminate after that.
- *
- *   -1: clause set is contradictory. Derived units have been restored.
- *
- *   a list of derived variables:  the status of the clause set is
- *   undetermined. The values of variables on the list will be restored (set to
- *   0) during backtracking.
- */
-function unitPropagate(clauses, varVals): number[] | 1 | -1 {
-  // all vars with a value derived in the forthcoming main loop
-  const derivedVars = [];
-
-  // the main loop is run, deriving new unit clauses, until either:
-  // - the clause set is detected to be unsatisfiable
-  // - the clause set is detected to be satisfiable
-  // - there are no more derived unit clauses
-
-  for (;;) {
-    let allClausesFound = true; // set to false if we find an undetermined clause
-    const derivedQueue = []; // derived units (literals) during one iteration
-    for (let i = 0; i < clauses.length; i++) {
-      const clause = clauses[i];
-      let clauseVal = 0;
-      let unassignedCount = 0; // count unassigned vars in clause
-      let unassignedLit = 0; // 0 means none found for this clause
-
-      for (let j = 0; j < clause.length; j++) {
-        const lit = clause[j];
-        let nr, polarity;
-        if (lit < 0) {
-          nr = 0 - lit;
-          polarity = -1;
-        } else {
-          nr = lit;
-          polarity = 1;
-        }
-
-        if (varVals[nr] === polarity) {
-          clauseVal = 1;
-          break;
-        } else if (varVals[nr] === 0) {
-          unassignedCount++;
-          unassignedLit = lit;
-        }
-      }
-
-      if (clauseVal === 1) continue; // clause is subsumed
-
-      // clause is not subsumed by varVals
-      if (unassignedCount === 0) {
-        for (let j = 0; j < derivedVars.length; j++)
-          varVals[derivedVars[j]] = 0; // restore derived
-        return -1;
-      } else if (unassignedCount === 1) {
-        // unassignedLit is a derived literal
-        derivedQueue.push(unassignedLit);
-      }
-
-      if (unassignedCount !== 0) allClausesFound = false;
-    }
-
-    // if all clauses were subsumed or all the variables in a clause had a value
-    // no need to restore varVals: whole search finished
-    if (allClausesFound) return 1;
-
-    // in case there are any derived units present, store to varVals and
-    // iterate the main loop
-    if (derivedQueue.length === 0) break;
-
-    // some units were derived: iterate the main loop
-    for (let j = 0; j < derivedQueue.length; j++) {
-      const lit = derivedQueue[j];
-      let nr, polarity;
-      if (lit < 0) {
-        nr = 0 - lit;
-        polarity = -1;
-      } else {
-        nr = lit;
-        polarity = 1;
-      }
-      varVals[nr] = polarity;
-      derivedVars.push(nr);
-    }
-  }
-
-  return derivedVars;
-}
-
-export function naiveDpll(clauses, maxvarnr): boolean {
-  // variable values are 0 if not set, 1 if positive, -1 if negative
-  const varVals = new Int32Array(maxvarnr + 1);
-  return satisfiableAt(clauses, varVals, 0, 0);
-}

lib/common/yaml.ts

@@ -0,0 +1,218 @@
+const LINE_WIDTH = 80;
+const INDENTATION = "  ";
+
+const STRING_RESERVED = new Set([
+  "true",
+  "True",
+  "TRUE",
+  "false",
+  "False",
+  "FALSE",
+  "null",
+  "Null",
+  "NULL",
+]);
+
+function isPrintable(str: string): boolean {
+  return !/[^\t\n\x20-\x7e\x85\u{a0}-\u{d7ff}\u{e000}-\u{fffd}\u{10000}-\u{10ffff}]/u.test(
+    str,
+  );
+}
+
+function stringifyKey(str: string): string {
+  if (!str || !isPrintable(str)) return JSON.stringify(str);
+  if (/^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|[\n,[\]{}]|\s$/.test(str))
+    return JSON.stringify(str);
+  return str;
+}
+
+function foldString(str: string): string[] {
+  if (str.length <= LINE_WIDTH) return [str];
+  if (str.startsWith(" ")) return [str];
+  const lines: string[] = [];
+
+  let idx = 0;
+  let cand = 0;
+  for (let i = 1; i < str.length - 1; ++i) {
+    if (str[i] !== " ") continue;
+
+    if (str[i + 1] === " ") {
+      i += 2;
+      while (str[i] === " ") ++i;
+      continue;
+    }
+
+    if (i <= idx + LINE_WIDTH) {
+      cand = i;
+      continue;
+    }
+
+    const c = cand > idx ? cand : i;
+    lines.push(str.slice(idx, c));
+    idx = c + 1;
+    cand = i;
+  }
+
+  if (cand > idx && str.length > idx + LINE_WIDTH) {
+    lines.push(str.slice(idx, cand));
+    idx = cand + 1;
+  }
+
+  lines.push(str.slice(idx));
+
+  return lines;
+}
+
+function stringifyString(str: string, res: string[], prefix1, prefix2): void {
+  if (/^\s*$/.test(str) || STRING_RESERVED.has(str) || !isPrintable(str)) {
+    res.push(prefix1 + JSON.stringify(str));
+    return;
+  }
+
+  if (!prefix2) prefix2 = INDENTATION;
+
+  const lines = str.split("\n");
+  if (lines.length > 1) {
+    let idt = "";
+    let chmp = "-";
+    if ((lines.find((l) => l) || "").startsWith(" "))
+      idt = `${INDENTATION.length}`;
+
+    if (!lines[lines.length - 1]) {
+      lines.pop();
+      if (lines[lines.length - 1]) chmp = "";
+      else chmp = "+";
+    }
+
+    if (/^\s+$/.test(lines[lines.length - 1])) {
+      res.push(prefix1 + JSON.stringify(str));
+      return;
+    }
+
+    let isFolded = false;
+    const folded = lines.map((l) => {
+      const ls = foldString(l);
+      if (ls.length > 1) isFolded = true;
+      return ls;
+    });
+
+    if (!isFolded) {
+      res.push(
+        `${prefix1}|${idt}${chmp}`,
+        ...lines.map((l) => (l ? prefix2 + l : l)),
+      );
+      return;
+    }
+
+    res.push(`${prefix1}>${idt}${chmp}`);
+    res.push(...folded[0].map((f) => prefix2 + f));
+    for (let i = 1; i < folded.length; ++i) {
+      const prevLine = folded[i - 1][0];
+      if (prevLine && !folded[i - 1][0].startsWith(" ")) res.push("");
+      res.push(...folded[i].map((f) => prefix2 + f));
+    }
+    return;
+  }
+
+  if (
+    /^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|\s$/.test(str) ||
+    parseFloat(str) === +str
+  ) {
+    res.push(prefix1 + JSON.stringify(str));
+    return;
+  }
+
+  res.push(prefix1 + str);
+}
+
+function stringifyAny(
+  obj: unknown,
+  res: string[],
+  prefix1 = "",
+  prefix2 = "",
+): void {
+  if (obj == null) {
+    res.push(`${prefix1}null`);
+    return;
+  }
+  if (typeof obj === "number" || typeof obj === "boolean") {
+    res.push(`${prefix1}${JSON.stringify(obj)}`);
+    return;
+  }
+  if (obj instanceof Date) {
+    res.push(`${prefix1}${obj.toJSON()}`);
+    return;
+  }
+
+  if (typeof obj === "string") {
+    stringifyString(obj, res, prefix1, prefix2);
+    return;
+  }
+
+  if (Array.isArray(obj)) {
+    if (!obj.length) {
+      res.push(prefix1 + "[]");
+      return;
+    }
+
+    if (!prefix1 || prefix1.endsWith("- ")) {
+      stringifyAny(obj[0], res, prefix1 + "- ", prefix2 + INDENTATION);
+      prefix1 = prefix2 + "- ";
+      prefix2 = prefix2 + INDENTATION;
+      for (let i = 1; i < obj.length; ++i)
+        stringifyAny(obj[i], res, prefix1, prefix2);
+    } else {
+      res.push(prefix1);
+      prefix1 = prefix2 + "- ";
+      prefix2 = prefix2 + INDENTATION;
+      for (let i = 0; i < obj.length; ++i)
+        stringifyAny(obj[i], res, prefix1, prefix2);
+    }
+    return;
+  }
+
+  const entries = Object.entries(obj).filter((e) => e[1] !== undefined);
+
+  if (!entries.length) {
+    res.push(prefix1 + "{}");
+    return;
+  }
+
+  if (!prefix1 || prefix1.endsWith("- ")) {
+    stringifyAny(
+      entries[0][1],
+      res,
+      prefix1 + `${stringifyKey(entries[0][0])}: `,
+      prefix2 + INDENTATION,
+    );
+    prefix1 = prefix2;
+    prefix2 = prefix2 + INDENTATION;
+    for (let i = 1; i < entries.length; ++i) {
+      stringifyAny(
+        entries[i][1],
+        res,
+        prefix1 + `${stringifyKey(entries[i][0])}: `,
+        prefix2,
+      );
+    }
+  } else {
+    res.push(prefix1);
+    prefix1 = prefix2;
+    prefix2 = prefix2 + INDENTATION;
+    for (let i = 0; i < entries.length; ++i) {
+      stringifyAny(
+        entries[i][1],
+        res,
+        prefix1 + `${stringifyKey(entries[i][0])}: `,
+        prefix2,
+      );
+    }
+  }
+}
+
+export function stringify(obj: unknown): string {
+  if (obj === undefined) return undefined;
+  const lines: string[] = [];
+  stringifyAny(obj, lines);
+  return lines.join("\n") + "\n";
+}

lib/config.ts

@@ -1,24 +1,5 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { resolve } from "path";
-import { readFileSync, existsSync } from "fs";
+import { resolve } from "node:path";
+import { readFileSync, existsSync } from "node:fs";
 
 // Find project root directory
 export let ROOT_DIR = resolve(__dirname, "..");
@@ -32,27 +13,26 @@ while (!existsSync(`${ROOT_DIR}/package.json`)) {
 }
 
 // For compatibility with v1.1
-let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl;
+let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl, fsHostname;
 
 const options = {
   EXT_DIR: { type: "path", default: resolve(ROOT_DIR, "config/ext") },
   MONGODB_CONNECTION_URL: {
     type: "string",
-    default: "mongodb://127.0.0.1/genieacs"
+    default: "mongodb://127.0.0.1/genieacs",
   },
 
   CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
   CWMP_PORT: { type: "int", default: 7547 },
-  CWMP_INTERFACE: { type: "string", default: "0.0.0.0" },
+  CWMP_INTERFACE: { type: "string", default: "::" },
   CWMP_SSL_CERT: { type: "string", default: "" },
   CWMP_SSL_KEY: { type: "string", default: "" },
   CWMP_LOG_FILE: { type: "path", default: "" },
   CWMP_ACCESS_LOG_FILE: { type: "path", default: "" },
-  CWMP_KEEP_ALIVE_TIMEOUT: { type: "int", default: 0 },
 
   NBI_WORKER_PROCESSES: { type: "int", default: 0 },
   NBI_PORT: { type: "int", default: 7557 },
-  NBI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  NBI_INTERFACE: { type: "string", default: "::" },
   NBI_SSL_CERT: { type: "string", default: "" },
   NBI_SSL_KEY: { type: "string", default: "" },
   NBI_LOG_FILE: { type: "path", default: "" },
@@ -60,16 +40,16 @@ const options = {
 
   FS_WORKER_PROCESSES: { type: "int", default: 0 },
   FS_PORT: { type: "int", default: 7567 },
-  FS_INTERFACE: { type: "string", default: "0.0.0.0" },
+  FS_INTERFACE: { type: "string", default: "::" },
   FS_SSL_CERT: { type: "string", default: "" },
   FS_SSL_KEY: { type: "string", default: "" },
-  FS_HOSTNAME: { type: "string", default: "" },
+  FS_URL_PREFIX: { type: "string", default: "" },
   FS_LOG_FILE: { type: "path", default: "" },
   FS_ACCESS_LOG_FILE: { type: "path", default: "" },
 
   UI_WORKER_PROCESSES: { type: "int", default: 0 },
   UI_PORT: { type: "int", default: 3000 },
-  UI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  UI_INTERFACE: { type: "string", default: "::" },
   UI_SSL_CERT: { type: "string", default: "" },
   UI_SSL_KEY: { type: "string", default: "" },
   UI_LOG_FILE: { type: "path", default: "" },
@@ -77,6 +57,7 @@ const options = {
   UI_JWT_SECRET: { type: "string", default: "" },
 
   UDP_CONNECTION_REQUEST_PORT: { type: "int", default: 0 },
+  FORWARDED_HEADER: { type: "string", default: "" },
 
   DOWNLOAD_TIMEOUT: { type: "int", default: 3600 },
   EXT_TIMEOUT: { type: "int", default: 3000 },
@@ -99,16 +80,11 @@ const options = {
   CONNECTION_REQUEST_ALLOW_BASIC_AUTH: { type: "bool", default: false },
   MAX_COMMIT_ITERATIONS: { type: "int", default: 32 },
 
-  // XML configuration
-  XML_RECOVER: { type: "bool", default: false },
-  XML_IGNORE_ENC: { type: "bool", default: false },
-  XML_FORMAT: { type: "bool", default: false },
-  XML_NO_DECL: { type: "bool", default: false },
-  XML_NO_EMPTY: { type: "bool", default: false },
-  XML_LIBXMLJS: { type: "bool", default: false },
-
   // Should probably never be changed
-  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 }
+  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
+
+  XMPP_JID: { type: "string", default: "" },
+  XMPP_PASSWORD: { type: "string", default: "" },
 };
 
 const allConfig: { [name: string]: string | number } = {};
@@ -120,37 +96,20 @@ function setConfig(name, value, commandLineArgument = false): boolean {
   if (name === "CONFIG_DIR" || name === "config-dir")
     configDir = configDir || resolve(ROOT_DIR, value);
 
-  if (name === "CWMP_SSL" || name === "cwmp-ssl") {
-    cwmpSsl =
-      cwmpSsl ||
-      String(value)
-        .toLowerCase()
-        .trim();
-  }
+  if (name === "CWMP_SSL" || name === "cwmp-ssl")
+    cwmpSsl = cwmpSsl || String(value).toLowerCase().trim();
 
-  if (name === "NBI_SSL" || name === "nbi-ssl") {
-    nbiSsl =
-      nbiSsl ||
-      String(value)
-        .toLowerCase()
-        .trim();
-  }
+  if (name === "NBI_SSL" || name === "nbi-ssl")
+    nbiSsl = nbiSsl || String(value).toLowerCase().trim();
 
-  if (name === "FS_SSL" || name === "fs-ssl") {
-    fsSsl =
-      fsSsl ||
-      String(value)
-        .toLowerCase()
-        .trim();
-  }
+  if (name === "FS_SSL" || name === "fs-ssl")
+    fsSsl = fsSsl || String(value).toLowerCase().trim();
 
-  if (name === "UI_SSL" || name === "ui-ssl") {
-    uiSsl =
-      uiSsl ||
-      String(value)
-        .toLowerCase()
-        .trim();
-  }
+  if (name === "UI_SSL" || name === "ui-ssl")
+    uiSsl = uiSsl || String(value).toLowerCase().trim();
+
+  if (name === "FS_HOSTNAME" || name === "fs-hostname")
+    fsHostname = fsHostname || String(value).trim();
 
   // For compatibility with v1.0
   if (name === "PRESETS_CACHE_DURATION" || name === "presets-cache-duration")
@@ -168,12 +127,6 @@ function setConfig(name, value, commandLineArgument = false): boolean {
   )
     setConfig("GPV_BATCH_SIZE", value);
 
-  if (name === "XML_PARSE_IGNORE_ENC" || name === "xml-parse-ignore-enc")
-    setConfig("XML_IGNORE_ENC", value);
-
-  if (name === "XML_PARSE_RECOVER" || name === "xml-parse-recover")
-    setConfig("XML_RECOVER", value);
-
   if (name === "FS_IP" || name === "fs-ip") setConfig("FS_HOSTNAME", value);
 
   function cast(val, type): string | number | boolean {
@@ -181,11 +134,7 @@ function setConfig(name, value, commandLineArgument = false): boolean {
       case "int":
         return Number(val);
       case "bool":
-        return ["true", "1"].includes(
-          String(val)
-            .trim()
-            .toLowerCase()
-        );
+        return ["true", "1"].includes(String(val).trim().toLowerCase());
       case "string":
         return String(val);
       case "path":
@@ -275,13 +224,22 @@ if (["true", "1"].includes(uiSsl)) {
   setConfig("UI_SSL_KEY", `${d}/cwmp.key`);
 }
 
+if (fsHostname) {
+  const FS_PORT = allConfig["FS_PORT"] || 7567;
+  const FS_SSL = !!allConfig["FS_SSL_CERT"];
+  setConfig(
+    "FS_URL_PREFIX",
+    (FS_SSL ? "https" : "http") + `://${fsHostname}:${FS_PORT}/`,
+  );
+}
+
 // Defaults
 for (const [k, v] of Object.entries(options))
   if (v["default"] != null) setConfig(k, v["default"]);
 
 export function get(
   optionName: string,
-  deviceId?: string
+  deviceId?: string,
 ): string | number | boolean {
   if (!deviceId) return allConfig[optionName];
 
@@ -310,7 +268,7 @@ export function get(
   return null;
 }
 
-export function getDefault(optionName): string | number | boolean {
+export function getDefault(optionName: string): string | number | boolean {
   const option = options[optionName];
   if (!option) return null;
 

lib/connection-request.ts

@@ -1,93 +1,76 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as crypto from "crypto";
-import * as dgram from "dgram";
-import { parse } from "url";
-import * as http from "http";
-import { evaluateAsync } from "./common/expression";
-import { Expression } from "./types";
-import * as auth from "./auth";
-import * as extensions from "./extensions";
-import * as debug from "./debug";
+import * as crypto from "node:crypto";
+import * as dgram from "node:dgram";
+import * as http from "node:http";
+import Expression, { Value } from "./common/expression.ts";
+import * as auth from "./auth.ts";
+import * as extensions from "./extensions.ts";
+import * as debug from "./debug.ts";
+import XmppClient from "./xmpp-client.ts";
+import * as config from "../lib/config.ts";
+import { encodeEntities, parseAttrs, Element } from "./xml-parser.ts";
+import * as logger from "../lib/logger.ts";
 
 async function extractAuth(
   exp: Expression,
-  context: {},
-  now: number,
-  dflt: any
+  dflt: Value,
 ): Promise<[string, string, Expression]> {
-  let username, password;
-  const _exp = await evaluateAsync(
-    exp,
-    context,
-    now,
+  let username: string, password: string;
+  const _exp = await exp.evaluateAsync(
     async (e: Expression): Promise<Expression> => {
-      if (!username && Array.isArray(e) && e[0] === "FUNC") {
-        if (e[1] === "EXT") {
-          if (typeof e[2] !== "string" || typeof e[3] !== "string") return null;
+      if (e instanceof Expression.Parameter)
+        return new Expression.Literal(null);
+      if (e instanceof Expression.FunctionCall) {
+        if (e.name === "NOW") return new Expression.Literal(0);
+        if (!username) {
+          if (e.name === "EXT") {
+            if (!e.args.every((a) => a instanceof Expression.Literal))
+              return new Expression.Literal(null);
+            const args = e.args.map((a) => a.value.toString());
+            if (typeof args[0] !== "string" || typeof args[1] !== "string")
+              return new Expression.Literal(null);
 
-          for (let i = 4; i < e.length; i++)
-            if (Array.isArray(e[i])) return null;
-
-          const { fault, value } = await extensions.run(e.slice(2));
-          return fault ? null : value;
-        } else if (e[1] === "AUTH") {
-          if (!Array.isArray(e[2]) && !Array.isArray(e[3])) {
-            username = e[2];
-            password = e[3];
+            const { fault, value } = await extensions.run(args);
+            if (fault) return new Expression.Literal(null);
+            return new Expression.Literal(value);
+          } else if (e.name === "AUTH") {
+            if (e.args.every((a) => a instanceof Expression.Literal)) {
+              username = `${e.args[0].value ?? ""}`;
+              password = `${e.args[1].value ?? ""}`;
+            }
+            return new Expression.Literal(dflt);
           }
-          return dflt;
         }
       }
       return e;
-    }
+    },
   );
   return [username, password, _exp];
 }
 
 function httpGet(
+  url: URL,
   options: http.RequestOptions,
-  timeout,
   _debug: boolean,
-  deviceId: string
-): Promise<{ statusCode: number; headers: {} }> {
+  deviceId: string,
+): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
   return new Promise((resolve, reject) => {
     const req = http
-      .get(options, res => {
+      .get(url, options, (res) => {
         res.resume();
         resolve({ statusCode: res.statusCode, headers: res.headers });
         if (_debug) {
-          debug.outgoingHttpRequest(req, deviceId, options, null);
+          debug.outgoingHttpRequest(req, deviceId, "GET", url, null);
           debug.incomingHttpResponse(res, deviceId, null);
         }
       })
-      .on("error", err => {
-        req.abort();
-        reject(new Error("Device is offline"));
-        if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
+      .on("error", (err) => {
+        req.destroy();
+        reject(err);
+        if (_debug)
+          debug.outgoingHttpRequestError(req, deviceId, "GET", url, err);
       })
-      .on("socket", socket => {
-        socket.setTimeout(timeout);
-        socket.on("timeout", () => {
-          req.abort();
-        });
+      .on("timeout", () => {
+        req.destroy();
       });
   });
 }
@@ -95,23 +78,20 @@ function httpGet(
 export async function httpConnectionRequest(
   address: string,
   authExp: Expression,
-  context: {},
   allowBasicAuth: boolean,
   timeout: number,
   _debug: boolean,
-  deviceId: string
-): Promise<void> {
-  const now = Date.now();
-  const options: http.RequestOptions = parse(address);
-  if (options.protocol !== "http:")
-    throw new Error("Invalid connection request URL or protocol");
+  deviceId: string,
+): Promise<string> {
+  const url = new URL(address);
+  if (url.protocol !== "http:")
+    return "Invalid connection request URL or protocol";
 
-  options.agent = new http.Agent({
-    maxSockets: 1,
-    keepAlive: true
-  });
+  const options: http.RequestOptions = {
+    agent: new http.Agent({ maxSockets: 1, keepAlive: true, timeout: timeout }),
+  };
 
-  let authHeader: {};
+  let authHeader: Record<string, string>;
   let username: string;
   let password: string;
 
@@ -119,16 +99,15 @@ export async function httpConnectionRequest(
     let opts = options;
     if (authHeader) {
       if (authHeader["method"] === "Basic") {
-        if (!allowBasicAuth)
-          throw new Error("Basic HTTP authentication not allowed");
+        if (!allowBasicAuth) return "Basic HTTP authentication not allowed";
 
         opts = Object.assign(
           {
             headers: {
-              Authorization: auth.basic(username || "", password || "")
-            }
+              Authorization: auth.basic(username || "", password || ""),
+            },
           },
-          options
+          options,
         );
       } else if (authHeader["method"] === "Digest") {
         opts = Object.assign(
@@ -137,57 +116,70 @@ export async function httpConnectionRequest(
               Authorization: auth.solveDigest(
                 username,
                 password,
-                options.path,
+                url.pathname + url.search,
                 "GET",
                 null,
-                authHeader
-              )
-            }
+                authHeader,
+              ),
+            },
           },
-          options
+          options,
         );
       } else {
-        throw new Error("Unrecognized auth method");
+        return "Unrecognized auth method";
       }
     }
 
-    let res = await httpGet(opts, timeout, _debug, deviceId);
+    let res: { statusCode: number; headers: http.IncomingHttpHeaders };
+    try {
+      res = await httpGet(url, opts, _debug, deviceId);
+    } catch (err) {
+      // Workaround for some devices unexpectedly closing the connection
+      if (authHeader) {
+        try {
+          res = await httpGet(url, opts, _debug, deviceId);
+        } catch (err) {
+          return `Connection request error: ${err.message}`;
+        }
+      }
 
-    // Workaround for some devices unexpectedly closing the connection
-    if (res.statusCode === 0 && authHeader)
-      res = await httpGet(opts, timeout, _debug, deviceId);
-    if (res.statusCode === 0) throw new Error("Device is offline");
-    if (res.statusCode === 200 || res.statusCode === 204) return;
+      if (err["code"] === "ECONNRESET" || err["code"] === "ECONNREFUSED")
+        return "Device is offline";
+
+      return `Connection request error: ${err.message}`;
+    }
+
+    if (res.statusCode === 200 || res.statusCode === 204) return "";
+
+    // When a Connection Request is received for the Virtual CWMP Device and the Proxied
+    // Device is offline the CPE Proxier MUST respond with an HTTP 503 failure
+    if (res.statusCode === 503) return "Device is offline";
 
     if (res.statusCode === 401 && res.headers["www-authenticate"]) {
-      authHeader = auth.parseWwwAuthenticateHeader(
-        res.headers["www-authenticate"]
-      );
-      [username, password, authExp] = await extractAuth(
-        authExp,
-        context,
-        now,
-        false
-      );
+      try {
+        authHeader = auth.parseWwwAuthenticateHeader(
+          res.headers["www-authenticate"],
+        );
+      } catch {
+        return "Connection request error: Error parsing www-authenticate header";
+      }
+      [username, password, authExp] = await extractAuth(authExp, false);
     } else {
-      throw new Error(
-        `Unexpected response code from device: ${res.statusCode}`
-      );
+      return `Connection request error: Unexpected status code ${res.statusCode}`;
     }
   }
-  throw new Error("Incorrect connection request credentials");
+
+  return "Connection request error: Incorrect connection request credentials";
 }
 
 export async function udpConnectionRequest(
-  address: string,
+  host: string,
+  port: number,
   authExp: Expression,
-  context: {},
-  sourcePort: number = 0,
+  sourcePort = 0,
   _debug: boolean,
-  deviceId: string
+  deviceId: string,
 ): Promise<void> {
-  const [host, portStr] = address.split(":", 2);
-  const port = portStr ? parseInt(portStr) : 80;
   const now = Date.now();
 
   const client = dgram.createSocket({ type: "udp4", reuseAddr: true });
@@ -202,12 +194,7 @@ export async function udpConnectionRequest(
   let username: string;
   let password: string;
 
-  [username, password, authExp] = await extractAuth(
-    authExp,
-    context,
-    now,
-    null
-  );
+  [username, password, authExp] = await extractAuth(authExp, null);
 
   if (username == null) username = "";
   if (password == null) password = "";
@@ -219,12 +206,12 @@ export async function udpConnectionRequest(
       .createHmac("sha1", password)
       .update(`${ts}${id}${username}${cn}`)
       .digest("hex");
-    const uri = `http://${address}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
-    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${address}\r\n\r\n`;
+    const uri = `http://${host}:${port}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
+    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${host}:${port}\r\n\r\n`;
     const message = Buffer.from(msg);
 
     for (let i = 0; i < 3; ++i) {
-      await new Promise((resolve, reject) => {
+      await new Promise<void>((resolve, reject) => {
         client.send(message, 0, message.length, port, host, (err: Error) => {
           if (err) reject(err);
           else resolve();
@@ -233,12 +220,88 @@ export async function udpConnectionRequest(
       });
     }
 
-    [username, password, authExp] = await extractAuth(
-      authExp,
-      context,
-      now,
-      null
-    );
+    [username, password, authExp] = await extractAuth(authExp, null);
   }
   client.close();
 }
+
+const XMPP_JID = config.get("XMPP_JID") as string;
+const XMPP_PASSWORD = config.get("XMPP_PASSWORD") as string;
+const XMPP_RESOURCE = crypto.randomBytes(8).toString("hex");
+
+let xmppClient: XmppClient;
+
+function xmppClientOnError(err: Error): void {
+  xmppClient = null;
+  logger.error({
+    message: "XMPP exception",
+    exception: err,
+    pid: process.pid,
+  });
+}
+
+function xmppClientOnClose(): void {
+  xmppClient = null;
+}
+
+export async function xmppConnectionRequest(
+  jid: string,
+  authExp: Expression,
+  timeout: number,
+  _debug: boolean,
+  deviceId: string,
+): Promise<string> {
+  if (!xmppClient) {
+    const [host, username] = XMPP_JID.split("@").reverse();
+    xmppClient = await XmppClient.connect({
+      host,
+      username,
+      resource: XMPP_RESOURCE,
+      password: XMPP_PASSWORD,
+      timeout: 120000,
+    });
+    xmppClient.on("error", xmppClientOnError);
+    xmppClient.on("close", xmppClientOnClose);
+    xmppClient.unref();
+  }
+
+  let username: string;
+  let password: string;
+
+  [username, password, authExp] = await extractAuth(authExp, null);
+  while (username != null && password != null) {
+    const msg = `<connectionRequest xmlns="urn:broadband-forum-org:cwmp:xmppConnReq-1-0"><username>${encodeEntities(
+      username,
+    )}</username><password>${encodeEntities(
+      password,
+    )}</password></connectionRequest>`;
+    let res: Element, rawRes: string, rawReq: string;
+    try {
+      ({ res, rawRes, rawReq } = await xmppClient.sendIqStanza(
+        XMPP_JID,
+        jid,
+        "get",
+        msg,
+        timeout,
+      ));
+    } catch (err) {
+      return err.message;
+    }
+    if (_debug) {
+      debug.outgoingXmppStanza(deviceId, rawReq);
+      debug.incomingXmppStanza(deviceId, rawRes);
+    }
+    const attrs = parseAttrs(res.attrs);
+    const type = attrs.find((a) => a.name === "type");
+    if (type && type.value === "result") return "";
+    const error = res.children.find((c) => c.name === "error");
+    if (!error || !error.children[0])
+      return "Unexpected XMPP connection request response";
+    if (error.children[0].name === "service-unavailable")
+      return "Device is offline";
+    if (error.children[0].name !== "not-authorized")
+      return "Unexpected XMPP connection request response";
+    [username, password, authExp] = await extractAuth(authExp, null);
+  }
+  return "Incorrect connection request credentials";
+}

lib/cwmp/db.ts

@@ -1,121 +1,49 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { MongoClient, ObjectID, Collection } from "mongodb";
-import { get } from "./config";
-import { escapeRegExp } from "./common";
-import { parse } from "./common/expression-parser";
+import { ObjectId } from "mongodb";
+import { decodeTag, encodeTag, escapeRegExp } from "../util.ts";
 import {
   DeviceData,
   Attributes,
   SessionFault,
   Task,
   Operation,
-  Expression
-} from "./types";
-import Path from "./common/path";
+} from "../types.ts";
+import { collections } from "../db/db.ts";
+import { optimizeProjection } from "../db/util.ts";
+import * as MongoTypes from "../db/types.ts";
 
-export let tasksCollection: Collection,
-  devicesCollection: Collection,
-  presetsCollection: Collection,
-  objectsCollection: Collection,
-  provisionsCollection: Collection,
-  virtualParametersCollection: Collection,
-  faultsCollection: Collection,
-  filesCollection: Collection,
-  operationsCollection: Collection,
-  permissionsCollection: Collection,
-  usersCollection: Collection,
-  configCollection: Collection;
+const INVALID_PATH_SUFFIX = "__invalid";
 
-export let client: MongoClient;
-
-export async function connect(): Promise<void> {
-  client = await MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
-    useNewUrlParser: true
-  });
-
-  const db = client.db();
-
-  tasksCollection = db.collection("tasks");
-  await tasksCollection.createIndex({ device: 1, timestamp: 1 });
-
-  devicesCollection = db.collection("devices");
-  presetsCollection = db.collection("presets");
-  objectsCollection = db.collection("objects");
-  filesCollection = db.collection("fs.files");
-  provisionsCollection = db.collection("provisions");
-  virtualParametersCollection = db.collection("virtualParameters");
-  faultsCollection = db.collection("faults");
-  operationsCollection = db.collection("operations");
-  permissionsCollection = db.collection("permissions");
-  usersCollection = db.collection("users");
-  configCollection = db.collection("config");
-}
-
-export async function disconnect(): Promise<void> {
-  if (client) await client.close();
-}
-
-// Optimize projection by removing overlaps
-// This can modify the object
-function optimizeProjection(obj: {}): {} {
-  if (obj[""]) return { "": obj[""] };
-
-  const keys = Object.keys(obj).sort();
-  if (keys.length <= 1) return obj;
-
-  for (let i = 1; i < keys.length; ++i) {
-    const a = keys[i - 1];
-    const b = keys[i];
-    if (b.startsWith(a)) {
-      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
-        delete obj[b];
-        keys.splice(i--, 1);
-      }
-    }
-  }
-  return obj;
+function compareAccessLists(list1: string[], list2: string[]): boolean {
+  if (list1.length !== list2.length) return false;
+  for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
+  return true;
 }
 
 export async function fetchDevice(
   id: string,
-  timestamp: number
-): Promise<[Path, number, Attributes?][]> {
-  const res: [Path, number, Attributes?][] = [
+  timestamp: number,
+): Promise<[string, number, Attributes?][]> {
+  const res: [string, number, Attributes?][] = [
+    ["Events", timestamp, { object: [timestamp, 1], writable: [timestamp, 0] }],
     [
-      Path.parse("Events"),
+      "DeviceID",
       timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 0] }
+      { object: [timestamp, 1], writable: [timestamp, 0] },
     ],
-    [
-      Path.parse("DeviceID"),
-      timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 0] }
-    ]
   ];
 
-  const device = await devicesCollection.findOne({ _id: id });
+  const device = (await collections.devices.findOne({ _id: id })) as any;
   if (!device) return null;
 
-  function storeParams(obj, path: string, pathLength: number, ts): void {
+  function storeParams(
+    obj,
+    path: string,
+    pathLength: number,
+    ts: number,
+  ): void {
     if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
+    if (obj["_attributesTimestamp"])
+      obj["_attributesTimestamp"] = +obj["_attributesTimestamp"];
 
     const attrs: Attributes = {};
     let t = obj["_timestamp"] || 1;
@@ -123,7 +51,7 @@ export async function fetchDevice(
 
     if (obj["_value"] != null) {
       attrs.value = [obj["_timestamp"] || 1, [obj["_value"], obj["_type"]]];
-      if (obj["_type"] === "xsd:dateTime")
+      if (obj["_type"] === "xsd:dateTime" && obj["_value"] instanceof Date)
         attrs.value[1][0] = +attrs.value[1][0];
 
       obj["_object"] = false;
@@ -133,151 +61,176 @@ export async function fetchDevice(
 
     if (obj["_object"] != null) attrs.object = [t, obj["_object"] ? 1 : 0];
 
-    res.push([Path.parse(path.slice(0, -1)), t, attrs]);
+    if (obj["_notification"] != null) {
+      attrs.notification = [
+        obj["_attributesTimestamp"] || 1,
+        obj["_notification"],
+      ];
+    }
+
+    if (obj["_accessList"] != null)
+      attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
+
+    try {
+      res.push([path, t, attrs]);
+    } catch {
+      // The path parser is now more strict so we might be in a situation where
+      // the database contains invalid paths from before this change So here we
+      // encode the invalid characters.
+      const splits = path.split(".");
+      splits[splits.length - 1] =
+        encodeTag(splits[splits.length - 1]) + INVALID_PATH_SUFFIX;
+      path = splits.join(".");
+      res.push([path, t, attrs]);
+      return;
+    }
 
     for (const [k, v] of Object.entries(obj)) {
       if (!k.startsWith("_")) {
         obj["_object"] = true;
-        storeParams(v, path + k + ".", pathLength + 1, obj["_timestamp"]);
+        storeParams(v, `${path}.${k}`, pathLength + 1, obj["_timestamp"]);
       }
     }
 
-    if ((obj["_object"] || !pathLength) && obj["_timestamp"])
-      res.push([Path.parse(path + "*"), obj["_timestamp"]]);
+    if (obj["_object"] && obj["_timestamp"])
+      res.push([path + ".*", obj["_timestamp"]]);
   }
 
+  const ts: number = +device["_timestamp"] || 0;
+  if (ts) res.push(["*", ts]);
+
   for (const [k, v] of Object.entries(device)) {
     switch (k) {
       case "_lastInform":
         res.push([
-          Path.parse("Events.Inform"),
+          "Events.Inform",
           +v,
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]]
-          }
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
         ]);
         break;
       case "_lastBoot":
         res.push([
-          Path.parse("Events.1_BOOT"),
+          "Events.1_BOOT",
           +v,
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]]
-          }
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
         ]);
         break;
       case "_lastBootstrap":
         res.push([
-          Path.parse("Events.0_BOOTSTRAP"),
+          "Events.0_BOOTSTRAP",
           +v,
           {
             object: [+v, 0],
             writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]]
-          }
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
         ]);
         break;
       case "_registered":
         // Use current timestamp for registered event attribute timestamps
         res.push([
-          Path.parse("Events.Registered"),
+          "Events.Registered",
           timestamp,
           {
             object: [timestamp, 0],
             writable: [timestamp, 0],
-            value: [timestamp, [+v, "xsd:dateTime"]]
-          }
+            value: [timestamp, [+v, "xsd:dateTime"]],
+          },
         ]);
         break;
       case "_id":
         res.push([
-          Path.parse("DeviceID.ID"),
+          "DeviceID.ID",
           timestamp,
           {
             object: [timestamp, 0],
             writable: [timestamp, 0],
-            value: [timestamp, [v as string, "xsd:string"]]
-          }
+            value: [timestamp, [v as string, "xsd:string"]],
+          },
         ]);
         break;
       case "_tags":
         if ((v as string[]).length) {
           res.push([
-            Path.parse("Tags"),
+            "Tags",
             timestamp,
-            { object: [timestamp, 1], writable: [timestamp, 0] }
+            { object: [timestamp, 1], writable: [timestamp, 0] },
           ]);
         }
 
-        for (let t of v as string[]) {
-          t = t.replace(/[^a-zA-Z0-9-]+/g, "_");
+        for (const t of v as string[]) {
           res.push([
-            Path.parse("Tags." + t),
+            "Tags." + encodeTag(t),
             timestamp,
             {
               object: [timestamp, 0],
               writable: [timestamp, 1],
-              value: [timestamp, [true, "xsd:boolean"]]
-            }
+              value: [timestamp, [true, "xsd:boolean"]],
+            },
           ]);
         }
         break;
       case "_deviceId":
         if (v["_Manufacturer"] != null) {
           res.push([
-            Path.parse("DeviceID.Manufacturer"),
+            "DeviceID.Manufacturer",
             timestamp,
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_Manufacturer"], "xsd:string"]]
-            }
+              value: [timestamp, [v["_Manufacturer"], "xsd:string"]],
+            },
           ]);
         }
 
         if (v["_OUI"] != null) {
           res.push([
-            Path.parse("DeviceID.OUI"),
+            "DeviceID.OUI",
             timestamp,
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_OUI"], "xsd:string"]]
-            }
+              value: [timestamp, [v["_OUI"], "xsd:string"]],
+            },
           ]);
         }
 
         if (v["_ProductClass"] != null) {
           res.push([
-            Path.parse("DeviceID.ProductClass"),
+            "DeviceID.ProductClass",
             timestamp,
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_ProductClass"], "xsd:string"]]
-            }
+              value: [timestamp, [v["_ProductClass"], "xsd:string"]],
+            },
           ]);
         }
 
         if (v["_SerialNumber"] != null) {
           res.push([
-            Path.parse("DeviceID.SerialNumber"),
+            "DeviceID.SerialNumber",
             timestamp,
             {
               object: [timestamp, 0],
               writable: [timestamp, 0],
-              value: [timestamp, [v["_SerialNumber"], "xsd:string"]]
-            }
+              value: [timestamp, [v["_SerialNumber"], "xsd:string"]],
+            },
           ]);
         }
+        break;
+      default:
+        if (!k.startsWith("_")) storeParams(v, k, 1, ts);
     }
   }
-
-  storeParams(device, "", 0, 0);
   return res;
 }
 
@@ -285,7 +238,7 @@ export async function saveDevice(
   deviceId: string,
   deviceData: DeviceData,
   isNew: boolean,
-  sessionTimestamp: number
+  sessionTimestamp: number,
 ): Promise<void> {
   const update = { $set: {}, $unset: {}, $addToSet: {}, $pull: {} };
 
@@ -299,7 +252,7 @@ export async function saveDevice(
     )
       continue;
 
-    const parent = deviceData.paths.get(diff[0].slice(0, -1));
+    const parent = deviceData.paths.get(diff[0].slice(0, -1).toString());
 
     // Param timestamps may be greater than session timestamp to track revisions
     if (diff[2] > sessionTimestamp) diff[2] = sessionTimestamp;
@@ -329,6 +282,12 @@ export async function saveDevice(
     const object2 = ((diff[2] || {}).object || [])[1];
     const writable2 = ((diff[2] || {}).writable || [])[1];
     const writable1 = ((diff[1] || {}).writable || [])[1];
+    const attributesTimestamp1 = ((diff[1] || {}).notification || [])[0];
+    const attributesTimestamp2 = ((diff[2] || {}).notification || [])[0];
+    const notification1 = ((diff[1] || {}).notification || [])[1];
+    const notification2 = ((diff[2] || {}).notification || [])[1];
+    const accessList1 = ((diff[1] || {}).accessList || [])[1];
+    const accessList2 = ((diff[2] || {}).accessList || [])[1];
 
     switch (path.segments[0]) {
       case "Events":
@@ -392,21 +351,37 @@ export async function saveDevice(
           if (value2 != null) {
             if (!update["$addToSet"]["_tags"])
               update["$addToSet"]["_tags"] = { $each: [] };
-            update["$addToSet"]["_tags"]["$each"].push(path.segments[1]);
+            update["$addToSet"]["_tags"]["$each"].push(
+              decodeTag(path.segments[1] as string),
+            );
           } else {
             if (!update["$pull"]["_tags"]) {
               update["$pull"]["_tags"] = {
-                $in: []
+                $in: [],
               };
             }
-            update["$pull"]["_tags"]["$in"].push(path.segments[1]);
+            update["$pull"]["_tags"]["$in"].push(
+              decodeTag(path.segments[1] as string),
+            );
           }
         }
 
         break;
       default:
         if (!diff[2]) {
-          update["$unset"][path.toString()] = 1;
+          let pathStr = path.toString();
+          // Paths with that suffix are encoded and need to be decoded
+          if (pathStr.endsWith(INVALID_PATH_SUFFIX)) {
+            const splits = pathStr.split(".");
+            splits[splits.length - 1] = decodeTag(
+              splits[splits.length - 1].slice(
+                0,
+                0 - INVALID_PATH_SUFFIX.length,
+              ),
+            );
+            pathStr = splits.join(".");
+          }
+          update["$unset"][pathStr] = 1;
           continue;
         }
 
@@ -424,7 +399,7 @@ export async function saveDevice(
                     Number.isInteger(value2 as number)
                   ) {
                     update["$set"][path.toString() + "._value"] = new Date(
-                      value2 as number
+                      value2 as number,
                     );
                   } else {
                     update["$set"][path.toString() + "._value"] = value2;
@@ -436,13 +411,13 @@ export async function saveDevice(
 
                 if (valueTimestamp2 !== valueTimestamp1) {
                   update["$set"][path.toString() + "._timestamp"] = new Date(
-                    valueTimestamp2
+                    valueTimestamp2,
                   );
                 }
 
                 break;
               case "object":
-                if (!diff[1] || !diff[1].object || object2 !== object1) {
+                if (!diff[1]?.object || object2 !== object1) {
                   update["$set"][
                     path.length ? path.toString() + "._object" : "_object"
                   ] = !!object2;
@@ -450,11 +425,49 @@ export async function saveDevice(
 
                 break;
               case "writable":
-                if (!diff[1] || !diff[1].writable || writable2 !== writable1) {
+                if (!diff[1]?.writable || writable2 !== writable1) {
                   update["$set"][
                     path.length ? path.toString() + "._writable" : "_writable"
                   ] = !!writable2;
                 }
+
+                break;
+              case "notification":
+                if (
+                  !diff[1] ||
+                  !diff[1].notification ||
+                  notification2 !== notification1
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._notification"
+                      : "_notification"
+                  ] = notification2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][path.toString() + "._attributesTimestamp"] =
+                    new Date(attributesTimestamp2);
+                }
+
+                break;
+              case "accessList":
+                if (
+                  !diff[1] ||
+                  !diff[1].accessList ||
+                  !compareAccessLists(accessList2, accessList1)
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._accessList"
+                      : "_accessList"
+                  ] = accessList2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][path.toString() + "._attributesTimestamp"] =
+                    new Date(attributesTimestamp2);
+                }
             }
           }
         }
@@ -463,13 +476,19 @@ export async function saveDevice(
           for (const attrName of Object.keys(diff[1])) {
             if (
               diff[1][attrName][1] != null &&
-              (!diff[2] || !diff[2][attrName] || diff[2][attrName][1] == null)
+              diff[2]?.[attrName]?.[1] == null
             ) {
               const p = path.length ? path.toString() + "." : "";
               update["$unset"][`${p}_${attrName}`] = 1;
               if (attrName === "value") {
                 update["$unset"][p + "_type"] = 1;
                 update["$unset"][p + "_timestamp"] = 1;
+              } else if (attrName === "notification") {
+                if (accessList2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
+              } else if (attrName === "accessList") {
+                if (notification2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
               }
             }
           }
@@ -507,78 +526,128 @@ export async function saveDevice(
     delete update["$pull"];
   }
 
-  const result = await devicesCollection.updateOne({ _id: deviceId }, update, {
-    upsert: isNew
-  });
+  const result = await collections.devices.updateOne(
+    { _id: deviceId },
+    update,
+    {
+      upsert: isNew,
+    },
+  );
 
-  if (result.result.n !== 1)
+  if (!result.matchedCount && !result.upsertedCount)
     throw new Error(`Device ${deviceId} not found in database`);
 
   if (update2) {
-    await devicesCollection.updateOne({ _id: deviceId }, update2);
+    await collections.devices.updateOne({ _id: deviceId }, update2);
     return;
   }
 }
 
 export async function getFaults(
-  deviceId
+  deviceId: string,
 ): Promise<{ [channel: string]: SessionFault }> {
-  const res = await faultsCollection
+  const res = await collections.faults
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
     .toArray();
 
   const faults: { [channel: string]: SessionFault } = {};
   for (const r of res) {
     const channel = r._id.slice(deviceId.length + 1);
-    delete r._id;
-    delete r.channel;
-    delete r.device;
-    r.timestamp = +r.timestamp;
-    r.provisions = JSON.parse(r.provisions);
-    faults[channel] = r;
+    const fault: SessionFault = {
+      code: r.code,
+      message: r.message,
+      ...(r.detail && { detail: r.detail }),
+      timestamp: +r.timestamp,
+      provisions: JSON.parse(r.provisions),
+      retries: r.retries,
+      ...(r.expiry && { expiry: +r.expiry }),
+    };
+    faults[channel] = fault;
   }
 
   return faults;
 }
 
-export async function saveFault(deviceId, channel, fault): Promise<void> {
+export async function saveFault(
+  deviceId: string,
+  channel: string,
+  fault: SessionFault,
+): Promise<void> {
   const id = `${deviceId}:${channel}`;
-  fault = Object.assign({}, fault);
-  fault._id = id;
-  fault.device = deviceId;
-  fault.channel = channel;
-  fault.timestamp = new Date(fault.timestamp);
-  fault.provisions = JSON.stringify(fault.provisions);
-  await faultsCollection.replaceOne({ _id: id }, fault, { upsert: true });
+  const f: MongoTypes.Fault = {
+    _id: id,
+    device: deviceId,
+    channel: channel,
+    timestamp: new Date(fault.timestamp),
+    code: fault.code,
+    message: fault.message,
+    ...(fault.detail && { detail: fault.detail }),
+    retries: fault.retries,
+    ...(fault.expiry && { expiry: new Date(fault.expiry) }),
+    provisions: JSON.stringify(fault.provisions),
+  };
+  await collections.faults.replaceOne({ _id: id }, f, { upsert: true });
 }
 
-export async function deleteFault(deviceId, channel): Promise<void> {
-  await faultsCollection.deleteOne({ _id: `${deviceId}:${channel}` });
+export async function deleteFault(
+  deviceId: string,
+  channel: string,
+): Promise<void> {
+  await collections.faults.deleteOne({ _id: `${deviceId}:${channel}` });
 }
 
 export async function getDueTasks(
-  deviceId,
-  timestamp
+  deviceId: string,
+  timestamp: number,
 ): Promise<[Task[], number]> {
-  const cur = tasksCollection.find({ device: deviceId }).sort(["timestamp"]);
+  const cur = collections.tasks
+    .find({ device: deviceId })
+    .sort({ timestamp: 1 });
   const tasks = [] as Task[];
 
-  for await (const task of cur) {
-    if (task.timestamp) task.timestamp = +task.timestamp;
-    if (task.expiry) task.expiry = +task.expiry;
-    if (task.timestamp >= timestamp) return [tasks, +task.timestamp];
-    task._id = String(task._id);
+  for await (const t of cur) {
+    if (+t.timestamp >= timestamp) return [tasks, +t.timestamp];
+    const task: Task = {
+      _id: t._id.toString(),
+      name: t.name,
+      ...(t.timestamp && { timestamp: +t.timestamp }),
+      ...(t.expiry && { expiry: +t.expiry }),
+      ...(t.name === "getParameterValues" && {
+        parameterNames: t.parameterNames,
+      }),
+      ...(t.name === "setParameterValues" && {
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "refreshObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "download" && {
+        fileType: t.fileType,
+        fileName: t.fileName,
+        targetFileName: t.targetFileName,
+      }),
+      ...(t.name === "addObject" && {
+        objectName: t.objectName,
+        parameterValues: t.parameterValues,
+      }),
+      ...(t.name === "deleteObject" && {
+        objectName: t.objectName,
+      }),
+      ...(t.name === "provisions" && {
+        provisions: t.provisions,
+      }),
+    };
 
     tasks.push(task);
 
     // For API compatibility
-    if (task.name === "download" && task.file) {
+    if (task.name === "download" && t["file"]) {
       let q;
-      if (ObjectID.isValid(task.file))
-        q = { _id: { $in: [task.file, new ObjectID(task.file)] } };
-      else q = { _id: task.file };
+      if (ObjectId.isValid(t["file"]))
+        q = { _id: { $in: [t["file"], new ObjectId(t["file"])] } };
+      else q = { _id: t["file"] };
 
-      const res = await filesCollection.find(q).toArray();
+      const res = await collections.files.find(q).toArray();
 
       if (res[0]) {
         if (!task.fileType) task.fileType = res[0].metadata.fileType;
@@ -591,102 +660,68 @@ export async function getDueTasks(
   return [tasks, null];
 }
 
-export async function clearTasks(deviceId, taskIds): Promise<void> {
-  await tasksCollection.deleteMany({
-    _id: { $in: taskIds.map(id => new ObjectID(id)) }
+export async function clearTasks(
+  deviceId: string,
+  taskIds: string[],
+): Promise<void> {
+  await collections.tasks.deleteMany({
+    _id: { $in: taskIds.map((id) => new ObjectId(id)) },
   });
 }
 
 export async function getOperations(
-  deviceId
+  deviceId: string,
 ): Promise<{ [commandKey: string]: Operation }> {
-  const res = await operationsCollection
+  const res = await collections.operations
     .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
     .toArray();
 
   const operations: { [commandKey: string]: Operation } = {};
   for (const r of res) {
     const commandKey = r._id.slice(deviceId.length + 1);
-    delete r._id;
-    r.timestamp = +r.timestamp;
-    if (r.args) r.args = JSON.parse(r.args);
-    r.provisions = JSON.parse(r.provisions);
-    r.retries = JSON.parse(r.retries);
-    operations[commandKey] = r;
+    // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
+    if (typeof r.provisions !== "string") {
+      delete r._id;
+      operations[commandKey] = r as unknown as Operation;
+      continue;
+    }
+    const operation: Operation = {
+      name: r.name,
+      timestamp: +r.timestamp,
+      channels:
+        typeof r.channels === "string" ? JSON.parse(r.channels) : r.channels,
+      retries: JSON.parse(r.retries),
+      provisions: JSON.parse(r.provisions),
+      ...(r.args && { args: JSON.parse(r.args) }),
+    };
+    operations[commandKey] = operation;
   }
   return operations;
 }
 
 export async function saveOperation(
-  deviceId,
-  commandKey,
-  operation
+  deviceId: string,
+  commandKey: string,
+  operation: Operation,
 ): Promise<void> {
   const id = `${deviceId}:${commandKey}`;
-  operation = Object.assign({}, operation);
-  operation._id = id;
-  operation.timestamp = new Date(operation.timestamp);
-  operation.provisions = JSON.stringify(operation.provisions);
-  operation.retries = JSON.stringify(operation.retries);
-  operation.args = JSON.stringify(operation.args);
-  await operationsCollection.replaceOne({ _id: id }, operation, {
-    upsert: true
+  const o: MongoTypes.Operation = {
+    _id: id,
+    name: operation.name,
+    timestamp: new Date(operation.timestamp),
+    channels: JSON.stringify(operation.channels),
+    provisions: JSON.stringify(operation.provisions),
+    retries: JSON.stringify(operation.retries),
+    args: JSON.stringify(operation.args),
+  };
+  await collections.operations.replaceOne({ _id: id }, o, {
+    upsert: true,
   });
 }
 
-export async function deleteOperation(deviceId, commandKey): Promise<void> {
-  await operationsCollection.deleteOne({ _id: `${deviceId}:${commandKey}` });
-}
-
-export async function getPresets(): Promise<{}[]> {
-  return presetsCollection.find().toArray();
-}
-
-export async function getObjects(): Promise<{}[]> {
-  return objectsCollection.find().toArray();
-}
-
-export async function getProvisions(): Promise<{}[]> {
-  return provisionsCollection.find().toArray();
-}
-
-export async function getVirtualParameters(): Promise<{}[]> {
-  return virtualParametersCollection.find().toArray();
-}
-
-export function getFiles(): Promise<{}[]> {
-  return filesCollection.find().toArray();
-}
-
-export async function getConfig(): Promise<
-  { id: string; value: Expression }[]
-> {
-  const res = await configCollection.find().toArray();
-  return res.map(c => ({
-    id: c["_id"],
-    value: parse(c["value"])
-  }));
-}
-
-interface Permission {
-  role: string;
-  resource: string;
-  access: number;
-  filter: string;
-  validate: string;
-}
-
-export async function getPermissions(): Promise<Permission[]> {
-  return permissionsCollection.find().toArray();
-}
-
-interface User {
-  _id: string;
-  password: string;
-  salt: string;
-  roles: string;
-}
-
-export async function getUsers(): Promise<User[]> {
-  return usersCollection.find().toArray();
+export async function deleteOperation(
+  deviceId: string,
+  commandKey: string,
+): Promise<void> {
+  await collections.operations.deleteOne({ _id: `${deviceId}:${commandKey}` });
 }

lib/cwmp/local-cache.ts

@@ -0,0 +1,373 @@
+import * as vm from "node:vm";
+import * as crypto from "node:crypto";
+import { collections } from "../db/db.ts";
+import { convertOldPrecondition } from "../db/util.ts";
+import * as logger from "../logger.ts";
+import * as scheduling from "../scheduling.ts";
+import Expression, { Value } from "../common/expression.ts";
+import {
+  Preset,
+  Provisions,
+  VirtualParameters,
+  Files,
+  Config,
+} from "../types.ts";
+import { LocalCache } from "../local-cache.ts";
+
+interface Snapshot {
+  presets: Preset[];
+  provisions: Provisions;
+  virtualParameters: VirtualParameters;
+  files: Files;
+  config: Config;
+}
+
+function flattenObject<T extends Record<keyof T, unknown>>(
+  src: T,
+  prefix = "",
+  dst = {} as T,
+): T {
+  for (const k of Object.keys(src)) {
+    const v = src[k];
+    if (typeof v === "object" && !Array.isArray(v))
+      flattenObject(v as T, `${prefix}${k}.`, dst);
+    else dst[`${prefix}${k}`] = v;
+  }
+  return dst;
+}
+
+async function fetchPresets(): Promise<[string, Preset[]]> {
+  const res = await collections.presets.find().toArray();
+  let objects = await collections.objects.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  objects.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto
+    .createHash("md5")
+    .update(JSON.stringify(res))
+    .update(JSON.stringify(objects))
+    .digest("hex");
+
+  objects = objects.map((obj) => {
+    // Flatten object
+    obj = flattenObject(obj);
+
+    // If no keys are defined, consider all parameters as keys to keep the
+    // same behavior from v1.0
+    if (!obj["_keys"]?.length)
+      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
+
+    return obj;
+  });
+
+  res.sort((a, b) => {
+    if (a["weight"] === b["weight"])
+      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
+    else return a["weight"] - b["weight"];
+  });
+
+  const presets = [] as Preset[];
+  for (const preset of res) {
+    let schedule: { md5: string; duration: number; schedule: any } = null;
+    if (preset["schedule"]) {
+      const parts = preset["schedule"].trim().split(/\s+/);
+      schedule = {
+        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
+        duration: null,
+        schedule: null,
+      };
+
+      try {
+        schedule.duration = +parts.shift() * 1000;
+        schedule.schedule = scheduling.parseCron(parts.join(" "));
+      } catch {
+        logger.warn({
+          message: "Invalid preset schedule",
+          preset: preset["_id"],
+          schedule: preset["schedule"],
+        });
+        schedule.schedule = false;
+      }
+    }
+
+    const events = preset["events"] || {};
+    let precondition: Expression = new Expression.Literal(true);
+    if (preset["precondition"]) {
+      try {
+        precondition = Expression.parse(preset["precondition"]);
+      } catch {
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"]),
+        );
+      }
+
+      // Simplify expression
+      precondition = precondition.evaluate((e) => e);
+    }
+
+    const _provisions: Preset["provisions"] = [];
+
+    // Generate provisions from the old configuration format
+    for (const c of preset.configurations) {
+      switch (c.type) {
+        case "age":
+          _provisions.push([
+            "refresh",
+            new Expression.Literal(c.name),
+            new Expression.Literal(+c.age),
+          ]);
+          break;
+
+        case "value":
+          _provisions.push([
+            "value",
+            new Expression.Literal(c.name),
+            new Expression.Literal(c.value),
+          ]);
+          break;
+
+        case "add_tag":
+          _provisions.push([
+            "tag",
+            new Expression.Literal(c.tag),
+            new Expression.Literal(true),
+          ]);
+          break;
+
+        case "delete_tag":
+          _provisions.push([
+            "tag",
+            new Expression.Literal(c.tag),
+            new Expression.Literal(false),
+          ]);
+          break;
+
+        case "provision":
+          _provisions.push([
+            c.name,
+            ...(c.args || []).map((a) => Expression.parse(a)),
+          ]);
+          break;
+
+        case "add_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push([
+                "instances",
+                new Expression.Literal(p),
+                new Expression.Literal(1),
+              ]);
+
+              for (const k in obj) {
+                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
+                  _provisions.push([
+                    "value",
+                    new Expression.Literal(`${p}.${k}`),
+                    new Expression.Literal(obj[k]),
+                  ]);
+              }
+            }
+          }
+
+          break;
+
+        case "delete_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push([
+                "instances",
+                new Expression.Literal(p),
+                new Expression.Literal(0),
+              ]);
+            }
+          }
+
+          break;
+
+        default: {
+          const exhaustiveCheck: never = c;
+          throw new Error(
+            `Unknown configuration type ${exhaustiveCheck["type"]}`,
+          );
+        }
+      }
+    }
+
+    presets.push({
+      name: preset["_id"],
+      channel: (preset["channel"] as string) || "default",
+      schedule: schedule,
+      events: events,
+      precondition: precondition,
+      provisions: _provisions,
+    });
+  }
+
+  return [h, presets];
+}
+
+async function fetchProvisions(): Promise<[string, Provisions]> {
+  const res = await collections.provisions.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+
+  const provisions = {};
+  for (const r of res) {
+    provisions[r._id] = {};
+    provisions[r._id].md5 = crypto
+      .createHash("md5")
+      .update(r.script)
+      .digest("hex");
+    provisions[r._id].script = new vm.Script(
+      `"use strict";(function(){\n${r.script}\n})();`,
+      { filename: r._id, lineOffset: -1 },
+    );
+  }
+
+  return [h, provisions];
+}
+
+async function fetchVirtualParameters(): Promise<[string, VirtualParameters]> {
+  const res = await collections.virtualParameters.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+
+  const virtualParameters = {};
+  for (const r of res) {
+    virtualParameters[r._id] = {};
+    virtualParameters[r._id].md5 = crypto
+      .createHash("md5")
+      .update(r.script)
+      .digest("hex");
+    virtualParameters[r._id].script = new vm.Script(
+      `"use strict";(function(){\n${r.script}\n})();`,
+      { filename: r._id, lineOffset: -1 },
+    );
+  }
+
+  return [h, virtualParameters];
+}
+
+async function fetchFiles(): Promise<[string, Files]> {
+  const res = await collections.files.find().toArray();
+  res.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(res)).digest("hex");
+  const files = {};
+
+  for (const r of res) {
+    const id = r.filename || r._id.toString();
+    files[id] = {};
+    files[id].length = r.length;
+  }
+
+  return [h, files];
+}
+
+async function fetchConfig(): Promise<[string, Config]> {
+  const conf = await collections.config.find().toArray();
+  conf.sort((a, b) => (a._id > b._id ? 1 : -1));
+  const h = crypto.createHash("md5").update(JSON.stringify(conf)).digest("hex");
+
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    _config[c._id] = Expression.parse(c.value).evaluate((e) => e);
+  }
+
+  return [h, _config];
+}
+
+const localCache = new LocalCache<Snapshot>("cwmp-local-cache-hash", refresh);
+
+async function refresh(): Promise<[string, Snapshot]> {
+  const res = await Promise.all([
+    fetchPresets(),
+    fetchProvisions(),
+    fetchVirtualParameters(),
+    fetchFiles(),
+    fetchConfig(),
+  ]);
+
+  const h = crypto.createHash("md5");
+  for (const r of res) h.update(r[0]);
+
+  const snapshot = {
+    presets: res[0][1],
+    provisions: res[1][1],
+    virtualParameters: res[2][1],
+    files: res[3][1],
+    config: res[4][1],
+  };
+
+  return [h.digest("hex"), snapshot];
+}
+
+export async function getRevision(): Promise<string> {
+  return await localCache.getRevision();
+}
+
+export function getPresets(revision: string): Preset[] {
+  return localCache.get(revision).presets;
+}
+
+export function getProvisions(revision: string): Provisions {
+  return localCache.get(revision).provisions;
+}
+
+export function getVirtualParameters(revision: string): VirtualParameters {
+  return localCache.get(revision).virtualParameters;
+}
+
+export function getFiles(revision: string): Files {
+  return localCache.get(revision).files;
+}
+
+export function getConfig(
+  revision: string,
+  key: string,
+  dflt: string,
+  fn: (e: Expression) => Expression.Literal,
+): string;
+export function getConfig(
+  revision: string,
+  key: string,
+  dflt: number,
+  fn: (e: Expression) => Expression.Literal,
+): number;
+export function getConfig(
+  revision: string,
+  key: string,
+  dflt: boolean,
+  fn: (e: Expression) => Expression.Literal,
+): boolean;
+export function getConfig(
+  revision: string,
+  key: string,
+  dflt: Value,
+  fn: (e: Expression) => Expression.Literal,
+): Value {
+  const snapshot = localCache.get(revision);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const e = snapshot.config[key];
+  if (!e) return dflt;
+  const v = e.evaluate(fn).value;
+  if (typeof v !== typeof dflt) return dflt;
+  return v;
+}
+
+export function getConfigExpression(
+  snapshotKey: string,
+  key: string,
+): Expression {
+  const snapshot = localCache.get(snapshotKey);
+  return snapshot.config[key];
+}

lib/db/db.ts

@@ -0,0 +1,57 @@
+import { MongoClient, Collection, GridFSBucket } from "mongodb";
+import { get } from "../config.ts";
+import * as MongoTypes from "./types.ts";
+
+export let filesBucket: GridFSBucket;
+
+export const collections = {
+  devices: null as Collection<MongoTypes.Device>,
+  presets: null as Collection<MongoTypes.Preset>,
+  objects: null as Collection<MongoTypes.Object>,
+  provisions: null as Collection<MongoTypes.Provision>,
+  virtualParameters: null as Collection<MongoTypes.VirtualParameter>,
+  faults: null as Collection<MongoTypes.Fault>,
+  tasks: null as Collection<MongoTypes.Task>,
+  files: null as Collection<MongoTypes.File>,
+  operations: null as Collection<MongoTypes.Operation>,
+  permissions: null as Collection<MongoTypes.Permission>,
+  users: null as Collection<MongoTypes.User>,
+  config: null as Collection<MongoTypes.Config>,
+  cache: null as Collection<MongoTypes.Cache>,
+  locks: null as Collection<MongoTypes.Lock>,
+};
+
+let clientPromise: Promise<MongoClient>;
+
+export async function connect(): Promise<void> {
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"));
+
+  const client = await clientPromise;
+  const db = client.db();
+
+  collections.tasks = db.collection("tasks");
+  collections.devices = db.collection("devices");
+  collections.presets = db.collection("presets");
+  collections.objects = db.collection("objects");
+  collections.files = db.collection("fs.files");
+  collections.provisions = db.collection("provisions");
+  collections.virtualParameters = db.collection("virtualParameters");
+  collections.faults = db.collection("faults");
+  collections.operations = db.collection("operations");
+  collections.permissions = db.collection("permissions");
+  collections.users = db.collection("users");
+  collections.config = db.collection("config");
+  collections.cache = db.collection("cache");
+  collections.locks = db.collection("locks");
+  filesBucket = new GridFSBucket(db);
+
+  await Promise.all([
+    collections.tasks.createIndex({ device: 1, timestamp: 1 }),
+    collections.cache.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }),
+    collections.locks.createIndex({ expire: 1 }, { expireAfterSeconds: 0 }),
+  ]);
+}
+
+export async function disconnect(): Promise<void> {
+  if (clientPromise != null) await (await clientPromise).close();
+}

lib/db/synth.ts

@@ -0,0 +1,660 @@
+import { Filter } from "mongodb";
+import { EJSON } from "bson";
+import { complement } from "espresso-iisojs";
+import { parseLikePattern } from "../common/expression/parser.ts";
+import Expression from "../common/expression.ts";
+import { decodeTag } from "../util.ts";
+import {
+  SynthContextBase,
+  likeDisjoint,
+  likeImplies,
+  Clause,
+} from "../common/expression/synth.ts";
+import normalize from "../common/expression/normalize.ts";
+
+type Minterm = number[];
+
+function getParam(exp: Expression, collection: string): string {
+  if (!(exp instanceof Expression.Parameter))
+    throw new Error("Left-hand operand must be a parameter");
+
+  const p = exp.path.toString();
+  if (collection === "devices") {
+    if (p === "DeviceID.ID") return "_id";
+    else if (p === "DeviceID") return "_deviceId";
+    else if (p.startsWith("DeviceID.")) return "_deviceId._" + p.slice(9);
+    else if (p === "Events.Inform") return "_lastInform";
+    else if (p === "Events.Registered") return "_registered";
+    else if (p === "Events.0_BOOTSTRAP") return "_lastBootstrap";
+    else if (p === "Events.1_BOOT") return "_lastBoot";
+    else if (!p.endsWith("._value") && !p.startsWith("Tags."))
+      return `${p}._value`;
+  }
+
+  return p;
+}
+
+function getTypes(parameter: string, collection: string): string[] {
+  if (collection === "devices") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "_lastInform") return ["date"];
+    if (parameter === "_registered") return ["date"];
+    if (parameter === "_lastBootstrap") return ["date"];
+    if (parameter === "_lastBoot") return ["date"];
+    if (parameter.startsWith("_deviceId.")) return ["string"];
+    if (parameter === "Reboot._value") return ["date"];
+    if (parameter === "FactoryReset._value") return ["date"];
+    if (parameter.endsWith("_timestamp")) return ["date"];
+    if (parameter.startsWith("Downloads.")) {
+      if (parameter.endsWith("Download._value")) return ["date"];
+      if (parameter.endsWith("Time._value")) return ["date"];
+      if (parameter.endsWith("Name._value")) return ["string"];
+      if (parameter.endsWith("Type._value")) return ["string"];
+    }
+    if (parameter.endsWith("_value"))
+      return ["bool", "number", "date", "string"];
+  } else if (collection === "tasks") {
+    if (parameter === "_id") return ["oid"];
+    if (parameter === "timestamp") return ["date"];
+    if (parameter === "expiry") return ["date"];
+    if (parameter === "name") return ["string"];
+    if (parameter === "device") return ["string"];
+  } else if (collection === "faults") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "timestamp") return ["date"];
+    if (parameter === "expiry") return ["date"];
+    if (parameter === "code") return ["string"];
+    if (parameter === "retries") return ["number"];
+    if (parameter === "channel") return ["string"];
+    if (parameter === "device") return ["string"];
+    if (parameter === "message") return ["string"];
+  } else if (collection === "users") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "roles") return ["string"];
+    else if (parameter === "password")
+      throw new Error("Cannot query restricted parameters");
+    else if (parameter === "salt")
+      throw new Error("Cannot query restricted parameters");
+  } else if (collection === "config") {
+    if (parameter === "_id") return ["string"];
+    else if (parameter === "value") return ["string"];
+  } else if (collection === "files") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "metadata.fileType") return ["string"];
+    if (parameter === "metadata.oui") return ["string"];
+    if (parameter === "metadata.productClass") return ["string"];
+    if (parameter === "metadata.version") return ["string"];
+  } else if (collection === "permissions") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "role") return ["string"];
+    if (parameter === "resource") return ["devices"];
+    if (parameter === "access") return ["number"];
+    if (parameter === "filter") return ["string"];
+    if (parameter === "validate") return ["string"];
+  } else if (collection === "presets") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "weight") return ["number"];
+    if (parameter === "channel") return ["string"];
+    if (parameter === "precondition") return ["string"];
+    if (parameter.startsWith("events.")) return ["bool"];
+  } else if (collection === "provisions") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "script") return ["string"];
+  } else if (collection === "virtualParameters") {
+    if (parameter === "_id") return ["string"];
+    if (parameter === "script") return ["string"];
+  }
+
+  if (parameter === "_id") return ["oid", "string"];
+  return ["bool", "number", "date", "string"];
+}
+
+function roundOid(oid: string, roundUp: boolean): string {
+  const match = (oid.match(/^[0-9a-f]*/)?.[0] ?? "").slice(0, 24);
+  let num = BigInt("0x0" + match);
+  let lastChar = 0;
+  if (oid.length > match.length) lastChar += oid.charCodeAt(match.length);
+  if (match.length < 24) lastChar -= 48;
+  if (lastChar > 0 && roundUp) num++;
+  num <<= BigInt(4 * (24 - match.length));
+  if (lastChar < 0 && !roundUp) --num;
+  const str = num.toString(16);
+  if (str.length > 24 || str.startsWith("-")) return null;
+  return str.padStart(24, "0");
+}
+
+function groupBy<T, K>(
+  input: T[],
+  callback: (item: T) => K,
+): Iterable<[K, T[]]> {
+  const groups = new Map<K, T[]>();
+  for (const item of input) {
+    const key = callback(item);
+    let arr = groups.get(key);
+    if (!arr) groups.set(key, (arr = []));
+    arr.push(item);
+  }
+
+  return groups.entries();
+}
+
+abstract class MongoClause {
+  abstract readonly parameter: string;
+  abstract toQuery(truthy: boolean): Filter<unknown>;
+  toString(): string {
+    return JSON.stringify(this.toQuery(true));
+  }
+}
+
+class MongoClauseArray extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly value: string,
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    if (truthy) return { [this.parameter]: { $eq: this.value } };
+    else return { [this.parameter]: { $ne: this.value } };
+  }
+}
+
+class MongoClauseCompare<T> extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly op: "$eq" | "$gt" | "$lt" | "$gte" | "$lte",
+    public readonly value: T,
+    public readonly type: "" | "bool" | "number" | "string" | "date" | "oid",
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    let v: any = this.value;
+    if (this.type === "date" && typeof v === "number")
+      v = { $date: new Date(v).toISOString() };
+    if (this.type === "oid" && typeof v === "string") v = { $oid: v };
+
+    if (truthy) return { [this.parameter]: { [this.op]: v } };
+    else if (this.op === "$eq") return { [this.parameter]: { $ne: v } };
+    else return { [this.parameter]: { $not: { [this.op]: v } } };
+  }
+}
+
+class MongoClauseType extends MongoClause {
+  constructor(
+    public readonly parameter: string,
+    public readonly type: string,
+  ) {
+    super();
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    if (truthy) return { [this.parameter]: { $type: this.type } };
+    else return { [this.parameter]: { $not: { $type: this.type } } };
+  }
+}
+
+class MongoClauseLike extends MongoClause {
+  readonly pattern: string[];
+  constructor(
+    public readonly parameter: string,
+    pat: string,
+    esc: string,
+    public readonly caseSensitive: boolean,
+  ) {
+    super();
+    this.pattern = parseLikePattern(pat, esc);
+  }
+
+  toQuery(truthy: boolean): Filter<unknown> {
+    const convChars = {
+      "-": "\\-",
+      "/": "\\/",
+      "\\": "\\/",
+      "^": "\\^",
+      $: "\\$",
+      "*": "\\*",
+      "+": "\\+",
+      "?": "\\?",
+      ".": "\\.",
+      "(": "\\(",
+      ")": "\\)",
+      "|": "\\|",
+      "[": "\\[",
+      "]": "\\]",
+      "{": "\\{",
+      "}": "\\}",
+      "\\%": ".*",
+      "\\_": ".",
+    };
+
+    const chars = this.pattern.map((c) => convChars[c] ?? c);
+    chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
+    const l = chars.length - 1;
+    chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
+    const pattern = chars.join("");
+    const options = this.caseSensitive ? "s" : "is";
+    if (truthy) {
+      return { [this.parameter]: { $regularExpression: { options, pattern } } };
+    } else {
+      return {
+        [this.parameter]: {
+          $not: { $regularExpression: { options, pattern } },
+        },
+      };
+    }
+  }
+}
+
+class MongoSynthContext extends SynthContextBase<Clause, MongoClause> {
+  constructor(private readonly collection: string) {
+    super();
+  }
+
+  getMinterms(clause: Clause, res: number): number[][] {
+    res = res & 0b111;
+    if (res & (res - 1)) return complement(this.getMinterms(clause, ~res));
+
+    const exp = clause.expression();
+    if (res === 0b001) {
+      const minterms: number[][] = [];
+      for (const dep of clause.getNullables()) {
+        const e = dep.operand.expression();
+        const param = getParam(e, this.collection);
+        if (param.startsWith("Tags.") && this.collection === "devices") {
+          const t = decodeTag(param.slice(5));
+          const c = new MongoClauseArray("_tags", t);
+          minterms.push([this.getVar(c) << 1]);
+          continue;
+        }
+        const c = new MongoClauseCompare(param, "$eq", null, "");
+        minterms.push([(this.getVar(c) << 1) ^ 1]);
+      }
+      return minterms;
+    }
+
+    if (!(exp instanceof Expression.Binary))
+      throw new Error("Invalid query expression");
+
+    const truthy = res === 0b100;
+
+    if ([">", "<", "="].includes(exp.operator)) {
+      const param = getParam(exp.left, this.collection);
+
+      if (!(exp.right instanceof Expression.Literal))
+        throw new Error(`Right-hand operand must be a literal value`);
+
+      let rhs = exp.right.value;
+      if (typeof rhs === "boolean") rhs = +rhs;
+
+      if (
+        param.startsWith("Tags.") &&
+        this.collection === "devices" &&
+        exp.operator === "="
+      ) {
+        const t = decodeTag(param.slice(5));
+        const c = new MongoClauseArray("_tags", t);
+        if (typeof rhs === "string") rhs = 2;
+        if (exp.operator === "=") {
+          if ((rhs === 1) !== truthy) return [];
+        } else if (exp.operator === ">") {
+          if ((truthy && rhs >= 1) || (!truthy && rhs < 1)) return [];
+        } else if (exp.operator === "<") {
+          if ((truthy && rhs <= 1) || (!truthy && rhs > 1)) return [];
+        }
+        return [[(this.getVar(c) << 1) ^ 1]];
+      }
+
+      let op: "$gt" | "$lt" | "$eq" | "$gte" | "$lte";
+      if (exp.operator === "=") op = "$eq";
+      else if (exp.operator === ">") op = truthy ? "$gt" : "$lte";
+      else if (exp.operator === "<") op = truthy ? "$lt" : "$gte";
+
+      const possibleTypes = new Set(getTypes(param, this.collection));
+      const clauses: MongoClause[] = [];
+
+      if (typeof rhs === "number") {
+        if (possibleTypes.has("number"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "number"));
+
+        if (possibleTypes.has("date"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "date"));
+        if (possibleTypes.has("bool") && (rhs === 0 || rhs === 1))
+          clauses.push(new MongoClauseCompare(param, op, !!rhs, "bool"));
+      } else if (typeof rhs === "string") {
+        if (possibleTypes.has("string"))
+          clauses.push(new MongoClauseCompare(param, op, rhs, "string"));
+
+        if (possibleTypes.has("oid")) {
+          const oid = roundOid(rhs, op === "$lt" || op === "$lte");
+          if (oid && (op !== "$eq" || oid === rhs))
+            clauses.push(new MongoClauseCompare(param, op, oid, "oid"));
+        }
+      }
+
+      if (op === "$eq" && !truthy) {
+        clauses.push(new MongoClauseCompare(param, "$eq", null, ""));
+        return [clauses.map((c) => this.getVar(c) << 1)];
+      }
+
+      // In the following clauses we could use $type operator, but we want the
+      // final query to use comparison operators to check for type because the
+      // $type operator in MongoDB doesn't use indexes
+      if (typeof rhs === "number") {
+        if (possibleTypes.has("bool")) {
+          if (
+            (rhs > 1 && (op === "$lt" || op === "$lte")) ||
+            (rhs < 0 && (op === "$gt" || op === "$gte"))
+          )
+            clauses.push(new MongoClauseCompare(param, "$gte", false, "bool"));
+        }
+
+        if (op === "$gt" || op === "$gte") {
+          if (possibleTypes.has("string"))
+            clauses.push(new MongoClauseCompare(param, "$gte", "", "string"));
+
+          if (possibleTypes.has("oid")) {
+            clauses.push(
+              new MongoClauseCompare(
+                param,
+                "$gte",
+                "000000000000000000000000",
+                "oid",
+              ),
+            );
+          }
+        }
+      } else if (typeof rhs === "string") {
+        if (op === "$lt" || op === "$lte") {
+          if (possibleTypes.has("bool"))
+            clauses.push(new MongoClauseCompare(param, "$gte", false, "bool"));
+
+          if (possibleTypes.has("number")) {
+            clauses.push(new MongoClauseCompare(param, "$gte", 0, "number"));
+            clauses.push(new MongoClauseCompare(param, "$lt", 0, "number"));
+          }
+          if (possibleTypes.has("date")) {
+            clauses.push(new MongoClauseCompare(param, "$gte", 0, "date"));
+            clauses.push(new MongoClauseCompare(param, "$lt", 0, "date"));
+          }
+        }
+      }
+
+      return clauses.map((c) => [(this.getVar(c) << 1) ^ 1]);
+    } else if (exp.operator === "LIKE") {
+      if (
+        !(
+          exp.right instanceof Expression.Literal &&
+          typeof exp.right.value === "string"
+        )
+      )
+        throw new Error("Right-hand operand of 'LIKE' must be a string");
+      const pat = exp.right.value;
+
+      let p = exp.left;
+      let caseSensitive = true;
+      if (
+        p instanceof Expression.FunctionCall &&
+        ["UPPER", "LOWER"].includes(p.name)
+      ) {
+        if (p.name === "UPPER" && pat !== pat.toUpperCase())
+          return truthy ? [] : [[]];
+        if (p.name === "LOWER" && pat !== pat.toLowerCase())
+          return truthy ? [] : [[]];
+        caseSensitive = false;
+        p = p.args[0];
+      }
+
+      const param = getParam(p, this.collection);
+      const c = new MongoClauseLike(param, pat, null, caseSensitive);
+      if (truthy) return [[(this.getVar(c) << 1) ^ 1]];
+      const typeClause = new MongoClauseCompare(param, "$gte", "", "string");
+      const r = [[this.getVar(c) << 1, (this.getVar(typeClause) << 1) ^ 1]];
+      return r;
+    } else {
+      throw new Error("Invalid query expression");
+    }
+  }
+
+  getDcSet(minterms: Minterm[]): number[][] {
+    const dcSet: number[][] = [];
+
+    const vars = new Set(minterms.flat().map((l) => l >> 1));
+    const clauses = Array.from(vars).map((v) => this.getClause(v));
+
+    for (const [parameter, clauses2] of groupBy(clauses, (c) => c.parameter)) {
+      const comparisons = clauses2.filter(
+        (c) => c instanceof MongoClauseCompare && c.value !== null,
+      ) as MongoClauseCompare<unknown>[];
+
+      for (const [type, clauses3] of groupBy(comparisons, (c) => c.type)) {
+        const isType = this.getVar(new MongoClauseType(parameter, type));
+        const values = new Set(clauses3.map((c) => c.value));
+        const valuesSorted = Array.from(values).sort((a, b) =>
+          a > b ? 1 : -1,
+        );
+        for (const [i, v] of valuesSorted.entries()) {
+          const eq = this.getVar(
+            new MongoClauseCompare(parameter, "$eq", v, type),
+          );
+          const gt = this.getVar(
+            new MongoClauseCompare(parameter, "$gt", v, type),
+          );
+          const lt = this.getVar(
+            new MongoClauseCompare(parameter, "$lt", v, type),
+          );
+          const gte = this.getVar(
+            new MongoClauseCompare(parameter, "$gte", v, type),
+          );
+          const lte = this.getVar(
+            new MongoClauseCompare(parameter, "$lte", v, type),
+          );
+
+          if (type === "bool") {
+            if (v === false) dcSet.push([(lt << 1) ^ 1]);
+            else if (v === true) dcSet.push([(gt << 1) ^ 1]);
+          } else if (type === "string") {
+            if (v === "") dcSet.push([(lt << 1) ^ 1]);
+          } else if (type === "oid") {
+            if (v < "000000000000000000000000") dcSet.push([(lte << 1) ^ 1]);
+            else if (v === "000000000000000000000000")
+              dcSet.push([(lt << 1) ^ 1]);
+            else if (v > "ffffffffffffffffffffffff")
+              dcSet.push([(gte << 1) ^ 1]);
+            else if (v === "ffffffffffffffffffffffff")
+              dcSet.push([(gt << 1) ^ 1]);
+          }
+          dcSet.push([(lt << 1) ^ 1, (gte << 1) ^ 1]);
+          dcSet.push([(gt << 1) ^ 1, (gte << 1) ^ 0]);
+          dcSet.push([(eq << 1) ^ 0, (lt << 1) ^ 0, (lte << 1) ^ 1]);
+          dcSet.push([(eq << 1) ^ 1, (gte << 1) ^ 0]);
+          dcSet.push([(eq << 1) ^ 1, (gt << 1) ^ 1]);
+
+          if (i === 0) {
+            dcSet.push([(isType << 1) ^ 0, (gte << 1) ^ 1]);
+            dcSet.push([(isType << 1) ^ 0, (lt << 1) ^ 1]);
+          } else {
+            const gt2 = this.getVar(
+              new MongoClauseCompare(
+                parameter,
+                "$gt",
+                valuesSorted[i - 1],
+                type,
+              ),
+            );
+            const lte2 = this.getVar(
+              new MongoClauseCompare(
+                parameter,
+                "$lte",
+                valuesSorted[i - 1],
+                type,
+              ),
+            );
+            dcSet.push([(gt2 << 1) ^ 0, (gte << 1) ^ 1]);
+            dcSet.push([(gt2 << 1) ^ 0, (lte2 << 1) ^ 0, (lt << 1) ^ 1]);
+          }
+
+          if (i === valuesSorted.length - 1)
+            dcSet.push([(isType << 1) ^ 1, (gt << 1) ^ 0, (lte << 1) ^ 0]);
+        }
+      }
+
+      const likes = clauses2.filter(
+        (c) => c instanceof MongoClauseLike,
+      ) as MongoClauseLike[];
+      if (likes.length) {
+        const isType = this.getVar(new MongoClauseType(parameter, "string"));
+        for (let i1 = 0; i1 < likes.length; ++i1) {
+          const l1 = likes[i1];
+          let p1 = l1.pattern;
+          dcSet.push([(this.getVar(l1) << 1) ^ 1, (isType << 1) ^ 0]);
+          for (let i2 = i1 + 1; i2 < likes.length; ++i2) {
+            const l2 = likes[i2];
+            let p2 = l2.pattern;
+            if (!l1.caseSensitive || !l2.caseSensitive) {
+              p1 = p1.map((c) => c.toLowerCase());
+              p2 = p2.map((c) => c.toLowerCase());
+            }
+            if (likeDisjoint(p1, p2)) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 1,
+                (this.getVar(l2) << 1) ^ 1,
+              ]);
+            } else if (
+              (!l1.caseSensitive || l2.caseSensitive) &&
+              likeImplies(p1, p2)
+            ) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 0,
+                (this.getVar(l2) << 1) ^ 1,
+              ]);
+            } else if (
+              (!l2.caseSensitive || l1.caseSensitive) &&
+              likeImplies(p2, p1)
+            ) {
+              dcSet.push([
+                (this.getVar(l1) << 1) ^ 1,
+                (this.getVar(l2) << 1) ^ 0,
+              ]);
+            }
+          }
+        }
+      }
+
+      const isNull = this.getVar(
+        new MongoClauseCompare(parameter, "$eq", null, ""),
+      );
+
+      const types = getTypes(parameter, this.collection).map((t) =>
+        this.getVar(new MongoClauseType(parameter, t)),
+      );
+
+      dcSet.push([(isNull << 1) ^ 0, ...types.map((t) => (t << 1) ^ 0)]);
+      for (let i = 0; i < types.length; ++i) {
+        const t1 = types[i];
+        dcSet.push([(t1 << 1) ^ 1, (isNull << 1) ^ 1]);
+        for (let j = i + 1; j < types.length; ++j) {
+          const t2 = types[j];
+          dcSet.push([(t1 << 1) ^ 1, (t2 << 1) ^ 1]);
+        }
+      }
+
+      if (parameter === "_id") dcSet.push([(isNull << 1) ^ 1]);
+    }
+    return dcSet;
+  }
+
+  canRaise(i: number, s: Set<number>): boolean {
+    if (!(i & 1)) return true;
+    const c = this.getClause(i >> 1);
+    if (c instanceof MongoClauseCompare) {
+      for (const j of s) {
+        if (j === i) continue;
+        const c2 = this.getClause(j >> 1);
+        if (c2.parameter !== c.parameter) continue;
+        if (!(j & 1)) return false;
+        if (c2 instanceof MongoClauseType) return false;
+      }
+    }
+    return true;
+  }
+
+  toQuery(minterms: Minterm[]): Filter<unknown> {
+    const or = [];
+
+    const ejsonOps = ["$oid", "$date", "$regularExpression"];
+
+    for (const minterm of minterms) {
+      const query = {};
+      loop: for (const clause of minterm) {
+        if (!minterm.length) return {};
+        const negate = !!(clause & 1);
+        const c = this.getClause(clause >> 1);
+        const q = c.toQuery(negate);
+        if (Object.keys(q).length !== 1)
+          throw new Error("Invalid query expression");
+        const [param, value] = Object.entries(q)[0];
+        const dests = [query, ...(query["$and"] ?? [])];
+        for (const dest of dests) {
+          if (!(param in dest)) {
+            dest[param] = value;
+            continue loop;
+          }
+
+          let src = value;
+          let dst = dest[param];
+          if (Object.getPrototypeOf(src).constructor !== Object) continue;
+          if (Object.getPrototypeOf(dst).constructor !== Object) continue;
+          let srcKeys = Object.keys(src);
+          let dstKeys = Object.keys(dst);
+
+          if ([...srcKeys, ...dstKeys].every((k) => k === "$not")) {
+            src = src["$not"];
+            dst = dst["$not"];
+            if (Object.getPrototypeOf(src).constructor !== Object) continue;
+            if (Object.getPrototypeOf(dst).constructor !== Object) continue;
+            srcKeys = Object.keys(src);
+            dstKeys = Object.keys(dst);
+          }
+
+          if (srcKeys.some((k) => dstKeys.includes(k))) continue;
+
+          // Don't mix regular operators with EJSON special operators
+          if (srcKeys.some((k) => ejsonOps.includes(k))) continue;
+          if (dstKeys.some((k) => ejsonOps.includes(k))) continue;
+
+          Object.assign(dst, src);
+          continue loop;
+        }
+
+        query["$and"] ??= [];
+        query["$and"].push({ [param]: value });
+      }
+      or.push(query);
+    }
+
+    if (or.length === 1) return or[0];
+    return { $or: or };
+  }
+}
+
+export function toMongoQuery(
+  exp: Expression,
+  resource: string,
+): Filter<unknown> | false {
+  exp = normalize(exp);
+  const clause = Clause.fromExpression(normalize(exp));
+  const context = new MongoSynthContext(resource);
+  const minterms = clause.getMinterms(context, 0b100);
+  const minimized = context.minimize(minterms);
+  if (!minimized.length) return false;
+  return EJSON.deserialize(context.toQuery(minimized));
+}
+
+export function validQuery(exp: Expression, resource: string): void {
+  const clause = Clause.fromExpression(normalize(exp));
+  const context = new MongoSynthContext(resource);
+  clause.getMinterms(context, 0b100);
+}

lib/db/types.ts

@@ -0,0 +1,184 @@
+import { ObjectId } from "mongodb";
+import { FaultStruct } from "../types.ts";
+import { Value } from "../common/expression.ts";
+
+export interface Fault {
+  _id: string;
+  device: string;
+  channel: string;
+  timestamp: Date;
+  provisions: string;
+  retries: number;
+  code: string;
+  message: string;
+  detail?:
+    | FaultStruct
+    | {
+        name: string;
+        message: string;
+        stack?: string;
+      };
+  expiry?: Date;
+}
+
+interface TaskBase {
+  _id: ObjectId;
+  timestamp?: Date;
+  expiry?: Date;
+  name: string;
+  device: string;
+}
+
+interface TaskGetParameterValues extends TaskBase {
+  name: "getParameterValues";
+  parameterNames: string[];
+}
+
+interface TaskSetParameterValues extends TaskBase {
+  name: "setParameterValues";
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskRefreshObject extends TaskBase {
+  name: "refreshObject";
+  objectName: string;
+}
+
+interface TaskReboot extends TaskBase {
+  name: "reboot";
+}
+
+interface TaskFactoryReset extends TaskBase {
+  name: "factoryReset";
+}
+
+interface TaskDownload extends TaskBase {
+  name: "download";
+  fileType: string;
+  fileName: string;
+  targetFileName?: string;
+}
+
+interface TaskAddObject extends TaskBase {
+  name: "addObject";
+  objectName: string;
+  parameterValues: [string, string | number | boolean, string?][];
+}
+
+interface TaskDeleteObject extends TaskBase {
+  name: "deleteObject";
+  objectName: string;
+}
+
+interface TaskProvisions extends TaskBase {
+  name: "provisions";
+  provisions?: [string, ...Value[]][];
+}
+
+export type Task =
+  | TaskGetParameterValues
+  | TaskSetParameterValues
+  | TaskRefreshObject
+  | TaskReboot
+  | TaskFactoryReset
+  | TaskDownload
+  | TaskAddObject
+  | TaskDeleteObject
+  | TaskProvisions;
+
+export interface Operation {
+  _id: string;
+  name: string;
+  timestamp: Date;
+  channels: string;
+  retries: string;
+  provisions: string;
+  args: string;
+}
+
+export interface Config {
+  _id: string;
+  value: string;
+}
+
+export interface Cache {
+  _id: string;
+  value: string;
+  timestamp: Date;
+  expire: Date;
+}
+
+export interface Device {
+  _id: string;
+  _lastInform: Date;
+  _registered: Date;
+  _tags?: string[];
+  _timestamp?: Date;
+}
+
+type Configuration =
+  | { type: "age"; name: string; age: number }
+  | { type: "value"; name: string; value: boolean | number | string }
+  | { type: "add_tag"; tag: string }
+  | { type: "delete_tag"; tag: string }
+  | { type: "add_object"; name: string; object: string }
+  | { type: "delete_object"; name: string; object: string }
+  | { type: "provision"; name: string; args?: string[] };
+
+export interface Preset {
+  _id: string;
+  weight: number;
+  channel: string;
+  events: Record<string, boolean>;
+  configurations: Configuration[];
+}
+
+export interface Object {
+  _id: string;
+}
+
+export interface Provision {
+  _id: string;
+  script: string;
+}
+
+export interface VirtualParameter {
+  _id: string;
+  script: string;
+}
+
+export interface File {
+  _id: string;
+  length: number;
+  filename: string;
+  uploadDate: Date;
+  metadata?: {
+    fileType?: string;
+    oui?: string;
+    productClass?: string;
+    version?: string;
+  };
+}
+
+export interface Permission {
+  _id: string;
+  role: string;
+  resource: string;
+  access: 1 | 2 | 3;
+  filter?: string;
+  validate?: string;
+}
+
+export interface User {
+  _id: string;
+  password: string;
+  roles: string;
+  salt: string;
+}
+
+export interface Lock {
+  _id: string;
+  value: string;
+  timestamp: Date;
+  expire: Date;
+}

lib/db/util.ts

@@ -0,0 +1,183 @@
+import Expression, { Value } from "../common/expression.ts";
+import Path from "../common/path.ts";
+import { encodeTag } from "../util.ts";
+
+// Optimize projection by removing overlaps
+// This can modify the object
+export function optimizeProjection(obj: { [path: string]: 1 }): {
+  [path: string]: 1;
+} {
+  if (obj[""]) return { "": obj[""] };
+
+  const keys = Object.keys(obj).sort();
+  if (keys.length <= 1) return obj;
+
+  for (let i = 1; i < keys.length; ++i) {
+    const a = keys[i - 1];
+    const b = keys[i];
+    if (b.startsWith(a)) {
+      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
+        delete obj[b];
+        keys.splice(i--, 1);
+      }
+    }
+  }
+  return obj;
+}
+
+export function convertOldPrecondition(q: Record<string, unknown>): Expression {
+  function recursive(_query): Expression {
+    let res: Expression = new Expression.Literal(true);
+    for (const [k, v] of Object.entries(_query)) {
+      if (k[0] === "$") {
+        if (k === "$and") {
+          for (const vv of Object.values(v))
+            res = Expression.and(res, recursive(vv));
+        } else if (k === "$or") {
+          let or: Expression = new Expression.Literal(false);
+          for (const vv of Object.values(v))
+            or = Expression.or(or, recursive(vv));
+          res = Expression.and(res, or);
+        } else {
+          throw new Error(`Operator ${k} not supported`);
+        }
+      } else if (k === "_tags") {
+        if (typeof v === "object") {
+          if (Array.isArray(v)) throw new Error(`Invalid type`);
+          for (const [op, val] of Object.entries(v)) {
+            if (op === "$ne") {
+              if (typeof v["$ne"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              res = Expression.and(
+                res,
+                new Expression.Unary(
+                  "IS NULL",
+                  new Expression.Parameter(
+                    Path.parse(`Tags.${encodeTag(val)}`),
+                  ),
+                ),
+              );
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              res = Expression.and(
+                res,
+                new Expression.Unary(
+                  "IS NOT NULL",
+                  new Expression.Parameter(
+                    Path.parse(`Tags.${encodeTag(val)}`),
+                  ),
+                ),
+              );
+            } else {
+              throw new Error(`Invalid tag query`);
+            }
+          }
+        } else {
+          res = Expression.and(
+            res,
+            new Expression.Unary(
+              "IS NOT NULL",
+              new Expression.Parameter(
+                Path.parse(`Tags.${encodeTag(v as string)}`),
+              ),
+            ),
+          );
+        }
+      } else if (k.startsWith("Tags.")) {
+        let exists: boolean;
+        if (typeof v === "boolean") exists = v;
+        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
+        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
+        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
+        else throw new Error(`Invalid tag query`);
+
+        res = Expression.and(
+          res,
+          new Expression.Unary(
+            exists ? "IS NOT NULL" : "IS NULL",
+            new Expression.Parameter(Path.parse(k)),
+          ),
+        );
+      } else if (typeof v === "object") {
+        if (Array.isArray(v)) throw new Error(`Invalid type`);
+        for (const [kk, vv] of Object.entries(v)) {
+          if (kk === "$eq") {
+            res = Expression.and(
+              res,
+              new Expression.Binary(
+                "=",
+                new Expression.Parameter(Path.parse(k)),
+                new Expression.Literal(vv),
+              ),
+            );
+          } else if (kk === "$ne") {
+            const p = new Expression.Parameter(Path.parse(k));
+            res = Expression.and(
+              res,
+              Expression.or(
+                new Expression.Binary("<>", p, new Expression.Literal(vv)),
+                new Expression.Unary("IS NULL", p),
+              ),
+            );
+          } else if (kk === "$lt") {
+            res = Expression.and(
+              res,
+              new Expression.Binary(
+                "<",
+                new Expression.Parameter(Path.parse(k)),
+                new Expression.Literal(vv),
+              ),
+            );
+          } else if (kk === "$lte") {
+            res = Expression.and(
+              res,
+              new Expression.Binary(
+                "<=",
+                new Expression.Parameter(Path.parse(k)),
+                new Expression.Literal(vv),
+              ),
+            );
+          } else if (kk === "$gt") {
+            res = Expression.and(
+              res,
+              new Expression.Binary(
+                ">",
+                new Expression.Parameter(Path.parse(k)),
+                new Expression.Literal(vv),
+              ),
+            );
+          } else if (kk === "$gte") {
+            res = Expression.and(
+              res,
+              new Expression.Binary(
+                ">=",
+                new Expression.Parameter(Path.parse(k)),
+                new Expression.Literal(vv),
+              ),
+            );
+          } else {
+            throw new Error(`Operator ${kk} not supported`);
+          }
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
+        }
+      } else {
+        res = Expression.and(
+          res,
+          new Expression.Binary(
+            "=",
+            new Expression.Parameter(Path.parse(k)),
+            new Expression.Literal(v as Value),
+          ),
+        );
+      }
+    }
+    return res;
+  }
+
+  // empty filter
+  if (!Object.keys(q).length) return new Expression.Literal(true);
+
+  return recursive(q);
+}

lib/debug.ts

@@ -1,32 +1,9 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import {
-  IncomingMessage,
-  ServerResponse,
-  ClientRequest,
-  RequestOptions
-} from "http";
-import { Socket } from "net";
-import { appendFileSync } from "fs";
-import * as yaml from "yaml";
-import * as config from "./config";
+import { IncomingMessage, ServerResponse, ClientRequest } from "node:http";
+import { Socket } from "node:net";
+import { appendFileSync } from "node:fs";
+import { stringify } from "./common/yaml.ts";
+import * as config from "./config.ts";
+import { getSocketEndpoints } from "./server.ts";
 
 const DEBUG_FILE = "" + config.get("DEBUG_FILE");
 const DEBUG_FORMAT = "" + config.get("DEBUG_FORMAT");
@@ -45,26 +22,27 @@ function getConnectionTimestamp(connection: Socket): Date {
 export function incomingHttpRequest(
   httpRequest: IncomingMessage,
   deviceId: string,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
+  const socketEndpoints = getSocketEndpoints(con);
   const msg = {
     event: "incoming HTTP request",
     timestamp: now,
-    remoteAddress: con.remoteAddress,
+    remoteAddress: socketEndpoints.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
-    localPort: con.localPort,
+    localPort: socketEndpoints.localPort,
     method: httpRequest.method,
     url: httpRequest.url,
     headers: httpRequest.headers,
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -73,24 +51,25 @@ export function incomingHttpRequest(
 export function outgoingHttpResponse(
   httpResponse: ServerResponse,
   deviceId: string,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
+  const socketEndpoints = getSocketEndpoints(con);
   const msg = {
     event: "outgoing HTTP response",
     timestamp: now,
-    remoteAddress: con.remoteAddress,
+    remoteAddress: socketEndpoints.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
     statusCode: httpResponse.statusCode,
     headers: httpResponse.getHeaders(),
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -99,27 +78,28 @@ export function outgoingHttpResponse(
 export function outgoingHttpRequest(
   httpRequest: ClientRequest,
   deviceId: string,
-  options: RequestOptions,
-  body: string
+  method: "GET" | "PUT" | "POST" | "DELETE",
+  url: URL,
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpRequest.connection;
+  const con = httpRequest.socket;
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
     connection: getConnectionTimestamp(con),
-    remotePort: options.port,
-    method: options.method || "GET",
-    url: options.path,
+    remotePort: url.port,
+    method: method,
+    url: url.pathname + url.search,
     headers: httpRequest.getHeaders(),
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -128,26 +108,27 @@ export function outgoingHttpRequest(
 export function outgoingHttpRequestError(
   httpRequest: ClientRequest,
   deviceId: string,
-  options: RequestOptions,
-  err: Error
+  method: "GET" | "PUT" | "POST" | "DELETE",
+  url: URL,
+  err: Error,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
   const msg = {
     event: "outgoing HTTP request",
     timestamp: now,
-    remoteAddress: options.hostname,
+    remoteAddress: url.hostname,
     deviceId: deviceId,
     connection: null,
-    remotePort: options.port,
-    method: options.method,
-    url: options.path,
+    remotePort: url.port,
+    method: method,
+    url: url.pathname + url.search,
     headers: httpRequest.getHeaders(),
-    error: err.message
+    error: err.message,
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -156,24 +137,24 @@ export function outgoingHttpRequestError(
 export function incomingHttpResponse(
   httpResponse: IncomingMessage,
   deviceId: string,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
-  const con = httpResponse.connection;
+  const con = httpResponse.socket;
   const msg = {
     event: "incoming HTTP response",
     timestamp: now,
     remoteAddress: con.remoteAddress,
     deviceId: deviceId,
-    connection: getConnectionTimestamp(httpResponse.connection),
+    connection: getConnectionTimestamp(httpResponse.socket),
     statusCode: httpResponse.statusCode,
     headers: httpResponse.headers,
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);
@@ -183,7 +164,7 @@ export function outgoingUdpMessage(
   remoteAddress: string,
   deviceId: string,
   remotePort: number,
-  body: string
+  body: string,
 ): void {
   if (!DEBUG_FILE) return;
   const now = new Date();
@@ -193,11 +174,62 @@ export function outgoingUdpMessage(
     remoteAddress: remoteAddress,
     deviceId: deviceId,
     remotePort: remotePort,
-    body: body
+    body: body,
   };
 
   if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + yaml.stringify(msg));
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function clientError(remoteAddress: string, err: Error): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "client error",
+    timestamp: now,
+    remoteAddress: remoteAddress,
+    error: err.message,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function incomingXmppStanza(deviceId: string, body: string): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "incoming XMPP stanza",
+    timestamp: now,
+    deviceId: deviceId,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
   else if (DEBUG_FORMAT === "json")
     appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
   else throw new Error(`Unrecognized DEBUG_FORMAT option`);

lib/default-provisions.ts

@@ -1,110 +1,192 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
-import * as config from "./config";
-import * as device from "./device";
-import * as scheduling from "./scheduling";
+import Path from "./common/path.ts";
+import * as config from "./config.ts";
+import * as device from "./device.ts";
+import * as scheduling from "./scheduling.ts";
+import { SessionContext, Declaration } from "./types.ts";
 
 const MAX_DEPTH = +config.get("MAX_DEPTH");
 
-export function refresh(sessionContext, provision, declarations): boolean {
-  const segments = Path.parse(provision[1]).segments.slice();
-  const l = segments.length;
-  segments.length = MAX_DEPTH;
-  segments.fill("*", l);
-  const path = Path.parse(segments.join("."));
-  const every = 1000 * (provision[2] || 1);
+export function refresh(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (
+    (provision.length !== 2 || typeof provision[1] !== "string") &&
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number") &&
+    (provision.length < 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number" ||
+      typeof provision[3] !== "boolean")
+  )
+    throw new Error("Invalid arguments");
+
+  const every = 1000 * ((provision[2] as number) || 1);
   const offset = scheduling.variance(sessionContext.deviceId, every);
   const t = scheduling.interval(sessionContext.timestamp, every, offset);
 
-  for (let i = l; i < path.length; ++i) {
+  let attrGet;
+  let refreshChildren;
+  if (provision[3] == null) {
+    refreshChildren = true;
+    attrGet = { object: 1, writable: 1, value: t };
+  } else {
+    attrGet = {};
+    refreshChildren = !!provision[3];
+    for (const a of provision.slice(4)) attrGet[a as string] = t;
+  }
+
+  let path = Path.parse(provision[1]);
+  let l = path.length;
+  if (refreshChildren) {
+    const segments = path.segments.slice();
+    l = segments.length;
+    segments.length = MAX_DEPTH;
+    segments.fill("*", l);
+    path = Path.parse(segments.join("."));
+  }
+
+  for (let i = l; i <= path.length; ++i) {
     declarations.push({
       path: path.slice(0, i),
       pathGet: t,
       pathSet: null,
-      attrGet: { object: 1, writable: 1, value: t },
+      attrGet: attrGet,
       attrSet: null,
-      defer: true
+      defer: true,
     });
   }
 
   return true;
 }
 
-export function value(sessionContext, provision, declarations): boolean {
+export function value(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (
+    provision.length < 3 ||
+    provision.length > 4 ||
+    typeof provision[1] !== "string"
+  )
+    throw new Error("Invalid arguments");
+
+  let attr: string, val: any;
+
+  if (provision.length === 3) {
+    attr = "value";
+    val = provision[2];
+  } else {
+    attr = (provision[2] as string) || "";
+    val = provision[3];
+  }
+
+  if (attr === "accessList") {
+    val = (val || "")
+      .split(",")
+      .map((s) => s.trim())
+      .filter((s) => !!s);
+  } else if (attr === "value") {
+    val = [val];
+  }
+
   declarations.push({
     path: Path.parse(provision[1]),
     pathGet: 1,
     pathSet: null,
-    attrGet: { value: 1 },
-    attrSet: { value: [provision[2]] },
-    defer: true
+    attrGet: { [attr]: 1 },
+    attrSet: { [attr]: val },
+    defer: true,
   });
 
   return true;
 }
 
-export function tag(sessionContext, provision, declarations): boolean {
+export function tag(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (
+    provision.length !== 3 ||
+    typeof provision[1] !== "string" ||
+    typeof provision[2] !== "boolean"
+  )
+    throw new Error("Invalid arguments");
+
   declarations.push({
     path: Path.parse(`Tags.${provision[1]}`),
     pathGet: 1,
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [provision[2]] },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
-export function reboot(sessionContext, provision, declarations): boolean {
+export function reboot(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
   declarations.push({
     path: Path.parse("Reboot"),
     pathGet: 1,
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
-export function reset(sessionContext, provision, declarations): boolean {
+export function reset(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
   declarations.push({
     path: Path.parse("FactoryReset"),
     pathGet: 1,
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
-export function download(sessionContext, provision, declarations): boolean {
+export function download(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+): boolean {
+  if (
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string") &&
+    (provision.length !== 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string" ||
+      typeof provision[3] !== "string")
+  )
+    throw new Error("Invalid arguments");
+
   const alias = [
     `FileType:${JSON.stringify(provision[1] || "")}`,
     `FileName:${JSON.stringify(provision[2] || "")}`,
-    `TargetFileName:${JSON.stringify(provision[3] || "")}`
+    `TargetFileName:${JSON.stringify(provision[3] || "")}`,
   ].join(",");
 
   declarations.push({
@@ -113,7 +195,7 @@ export function download(sessionContext, provision, declarations): boolean {
     pathSet: 1,
     attrGet: null,
     attrSet: null,
-    defer: true
+    defer: true,
   });
 
   declarations.push({
@@ -122,22 +204,25 @@ export function download(sessionContext, provision, declarations): boolean {
     pathSet: null,
     attrGet: { value: 1 },
     attrSet: { value: [sessionContext.timestamp] },
-    defer: true
+    defer: true,
   });
 
   return true;
 }
 
 export function instances(
-  sessionContext,
-  provision,
-  declarations,
-  startRevision,
-  endRevision
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+  startRevision: number,
+  endRevision: number,
 ): boolean {
+  if (provision.length !== 3 || typeof provision[1] !== "string")
+    throw new Error("Invalid arguments");
+
   let count = Number(provision[2]);
 
-  if (Number.isNaN(count)) return true;
+  if (Number.isNaN(count)) throw new Error("Invalid arguments");
 
   const path = Path.parse(provision[1]);
 
@@ -148,7 +233,7 @@ export function instances(
       pathSet: null,
       attrGet: null,
       attrSet: null,
-      defer: true
+      defer: true,
     });
 
     if (endRevision === startRevision) return false;
@@ -156,7 +241,7 @@ export function instances(
     const unpacked = device.unpack(
       sessionContext.deviceData,
       path,
-      startRevision + 1
+      startRevision + 1,
     );
     count = Math.max(0, unpacked.length + count);
   }
@@ -167,7 +252,7 @@ export function instances(
     pathSet: count,
     attrGet: null,
     attrSet: null,
-    defer: true
+    defer: true,
   });
 
   return true;

lib/device.ts

@@ -1,29 +1,19 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
-import { DeviceData, Declaration, Clear } from "./types";
+import Expression, { Value } from "./common/expression.ts";
+import Path from "./common/path.ts";
+import {
+  DeviceData,
+  Declaration,
+  Clear,
+  AttributeTimestamps,
+  Attributes,
+} from "./types.ts";
 
 const CHANGE_FLAGS = {
   object: 2,
   writable: 4,
-  value: 8
+  value: 8,
+  notification: 16,
+  accessList: 32,
 };
 
 function parseBool(v): boolean {
@@ -35,8 +25,8 @@ function parseBool(v): boolean {
 }
 
 export function sanitizeParameterValue(
-  parameterValue: [string | number | boolean, string]
-): [string | number | boolean, string?] {
+  parameterValue: [string | number | boolean, string],
+): [string | number | boolean, string] {
   if (parameterValue[0] != null) {
     switch (parameterValue[1]) {
       case "xsd:boolean":
@@ -46,13 +36,6 @@ export function sanitizeParameterValue(
           else parameterValue[0] = b;
         }
         break;
-      case "xsd:string":
-      case "xsd:base64":
-      case "xsd:hexBinary":
-        if (typeof parameterValue[0] !== "string")
-          parameterValue[0] = "" + parameterValue[0];
-
-        break;
       case "xsd:int":
       case "xsd:unsignedInt":
         if (typeof parameterValue[0] !== "number") {
@@ -75,7 +58,7 @@ export function sanitizeParameterValue(
         }
         break;
       default:
-        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
+        parameterValue[0] = "" + parameterValue[0];
         break;
     }
   }
@@ -86,7 +69,7 @@ export function sanitizeParameterValue(
 export function getAliasDeclarations(
   path: Path,
   timestamp: number,
-  attrGet = null
+  attrGet = null,
 ): Declaration[] {
   const stripped = path.stripAlias();
   let decs: Declaration[] = [
@@ -96,19 +79,19 @@ export function getAliasDeclarations(
       pathSet: null,
       attrGet: attrGet,
       attrSet: null,
-      defer: true
-    }
+      defer: true,
+    },
   ];
 
   if (path.alias) {
     for (const [i, alias] of path.segments.entries()) {
-      if (Array.isArray(alias)) {
+      if (alias instanceof Expression) {
         const parent = stripped.slice(0, i + 1);
-        for (const [p] of alias as [Path, string][]) {
+        for (const [p] of expressionToAlias(alias)) {
           decs = decs.concat(
             getAliasDeclarations(parent.concat(p), timestamp, {
-              value: timestamp
-            })
+              value: timestamp,
+            }),
           );
         }
       }
@@ -118,24 +101,42 @@ export function getAliasDeclarations(
   return decs;
 }
 
+export function expressionToAlias(exp: Expression): [Path, Value][] {
+  if (exp instanceof Expression.Literal && exp.value === true) return [];
+  if (exp instanceof Expression.Binary) {
+    if (exp.operator === "AND")
+      return [...expressionToAlias(exp.left), ...expressionToAlias(exp.right)];
+    else if (exp.operator === "=") {
+      if (
+        exp.left instanceof Expression.Parameter &&
+        exp.right instanceof Expression.Literal
+      )
+        return [[exp.left.path, exp.right.value]];
+    }
+  }
+  throw new Error("Invalid alias expression");
+}
+
 export function unpack(
   deviceData: DeviceData,
   path: Path,
-  revision?: number
+  revision?: number,
 ): Path[] {
   let allMatches = [] as Path[];
   if (!path.alias) {
-    for (const p of deviceData.paths.find(path, false, true))
+    for (const p of deviceData.paths.findCompat(path, false, true))
       if (deviceData.attributes.has(p, revision)) allMatches.push(p);
   } else {
     const wildcardPath = path.stripAlias();
 
-    for (const p of deviceData.paths.find(wildcardPath, false, true))
+    for (const p of deviceData.paths.findCompat(wildcardPath, false, true))
       if (deviceData.attributes.has(p, revision)) allMatches.push(p);
 
     for (let i = path.length - 1; i >= 0; --i) {
       if (path.alias & (1 << i)) {
-        for (const [param, val] of path.segments[i] as [Path, string][]) {
+        for (const [param, val] of expressionToAlias(
+          path.segments[i] as Expression,
+        )) {
           const p = wildcardPath.slice(0, i + 1).concat(param);
           const unpacked = unpack(deviceData, p, revision);
           const filtered: Path[] = [];
@@ -189,29 +190,29 @@ export function unpack(
 }
 
 export function clear(
-  deviceData,
-  path,
-  timestamp,
-  attributes,
-  changeFlags = 0
+  deviceData: DeviceData,
+  path: Path,
+  timestamp: number,
+  attributes: AttributeTimestamps,
+  changeFlags = 0,
 ): void {
   const changeTrackers = {};
 
   timestamp = timestamp || 0;
 
   let descendantsTimestamp = timestamp;
-  if (attributes && attributes.object) {
+  if (attributes?.object) {
     if (attributes.object > descendantsTimestamp)
       descendantsTimestamp = attributes.object;
     if (!(attributes.object <= attributes.value))
       attributes.value = attributes.object;
   }
 
-  for (const p of deviceData.paths.find(
+  for (const p of deviceData.paths.findCompat(
     path,
     true,
     true,
-    descendantsTimestamp ? 99 : path.length
+    descendantsTimestamp ? 99 : path.length,
   )) {
     const tracker = deviceData.trackers.get(p);
     for (const k in tracker) changeTrackers[k] |= tracker[k];
@@ -270,12 +271,12 @@ function compareEquality(a, b): boolean {
 
 export function set(
   deviceData: DeviceData,
-  path: Path,
-  timestamp,
-  attributes,
-  toClear?: Clear[]
+  pathStr: string,
+  timestamp: number,
+  attributes: Attributes,
+  toClear?: Clear[],
 ): Clear[] {
-  path = deviceData.paths.add(path);
+  const path = deviceData.paths.add(pathStr);
 
   const currentTimestamp = deviceData.timestamps.get(path);
 
@@ -300,7 +301,7 @@ export function set(
       attributes.object[1] &&
       attributes.object[0] >= (attributes.value ? attributes.value[0] : 0)
     )
-      attributes.value = [attributes.object[0]];
+      attributes.value = [attributes.object[0], null];
 
     const newAttributes = Object.assign({}, currentAttributes, attributes);
 
@@ -314,7 +315,7 @@ export function set(
         else if (
           !compareEquality(
             attributes[attrName][1],
-            currentAttributes[attrName][1]
+            currentAttributes[attrName][1],
           )
         )
           changeFlags |= CHANGE_FLAGS[attrName];
@@ -330,10 +331,10 @@ export function set(
       if (path.length > 1) {
         toClear = set(
           deviceData,
-          path.slice(0, path.length - 1),
+          path.slice(0, path.length - 1).toString(),
           timestamp,
           { object: [timestamp, 1] },
-          toClear
+          toClear,
         );
       }
     }
@@ -344,7 +345,12 @@ export function set(
       deviceData.attributes.delete(path);
       changeFlags |= 1;
     } else if (path.wildcard) {
-      for (const p of deviceData.paths.find(path, false, true, path.length)) {
+      for (const p of deviceData.paths.findCompat(
+        path,
+        false,
+        true,
+        path.length,
+      )) {
         if (timestamp > deviceData.timestamps.get(p)) {
           toClear = toClear || [];
           toClear.push([p, timestamp]);
@@ -361,7 +367,12 @@ export function set(
       toClear = toClear || [];
       toClear.push([path, 0, { object: attributes.object[0] }, changeFlags]);
     } else {
-      for (const p of deviceData.paths.find(path, true, false, path.length)) {
+      for (const p of deviceData.paths.findCompat(
+        path,
+        true,
+        false,
+        path.length,
+      )) {
         const tracker = deviceData.trackers.get(p);
         for (const k in tracker)
           if (tracker[k] & changeFlags) deviceData.changes.add(k);
@@ -374,11 +385,11 @@ export function set(
 
 export function track(
   deviceData: DeviceData,
-  path: Path,
+  pathStr: string,
   marker: string,
-  attributes?
+  attributes?: string[],
 ): void {
-  path = deviceData.paths.add(path);
+  const path = deviceData.paths.add(pathStr);
   let f = 1;
 
   if (attributes)
@@ -393,7 +404,10 @@ export function track(
   cur[marker] |= f;
 }
 
-export function clearTrackers(deviceData: DeviceData, tracker): void {
+export function clearTrackers(
+  deviceData: DeviceData,
+  tracker: string | string[],
+): void {
   if (Array.isArray(tracker)) {
     for (const v of deviceData.trackers.values())
       for (const t of tracker) delete v[t];

lib/extensions.ts

@@ -1,62 +1,39 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { spawn, ChildProcess } from "child_process";
-import * as crypto from "crypto";
-import * as config from "./config";
-import { Fault } from "./types";
+import { spawn, ChildProcess } from "node:child_process";
+import * as crypto from "node:crypto";
+import readline from "node:readline";
+import * as config from "./config.ts";
+import { Fault } from "./types.ts";
+import { ROOT_DIR } from "./config.ts";
+import * as logger from "./logger.ts";
 
 const TIMEOUT = +config.get("EXT_TIMEOUT");
 
 const processes: { [script: string]: ChildProcess } = {};
 const jobs = new Map();
 
-function messageHandler(message): void {
-  const func = jobs.get(message[0]);
-  if (func) {
-    jobs.delete(message[0]);
-    func({ fault: message[1], value: message[2] });
-  }
-}
-
-export function run(args): Promise<{ fault: Fault; value: any }> {
-  return new Promise(resolve => {
+export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
+  return new Promise((resolve) => {
     const scriptName = args[0];
 
     const id = crypto.randomBytes(8).toString("hex");
     jobs.set(id, resolve);
 
     if (!processes[scriptName]) {
-      const p = spawn("bin/genieacs-ext", [scriptName], {
-        stdio: ["inherit", "inherit", "inherit", "ipc"]
+      const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
+        stdio: ["ignore", "pipe", "pipe", "ipc"],
       });
       processes[scriptName] = p;
 
-      p.on("error", err => {
+      p.on("error", (err) => {
         if (processes[scriptName] === p) {
           if (jobs.delete(id)) {
             resolve({
               fault: { code: err.name, message: err.message },
-              value: null
+              value: null,
             });
           }
 
+          // eslint-disable-next-line @typescript-eslint/no-floating-promises
           kill(processes[scriptName]);
           delete processes[scriptName];
         }
@@ -66,14 +43,33 @@ export function run(args): Promise<{ fault: Fault; value: any }> {
         if (processes[scriptName] === p) delete processes[scriptName];
       });
 
-      p.on("message", messageHandler);
+      p.on("message", (message) => {
+        const func = jobs.get(message[0]);
+        if (func) {
+          jobs.delete(message[0]);
+          // Wait for any disconnect even to fire
+          setTimeout(() => {
+            func({ fault: message[1], value: message[2] });
+          });
+        }
+      });
+
+      const rlstdout = readline.createInterface(p.stdout);
+      rlstdout.on("line", (line) => {
+        logger.info({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
+      });
+
+      const rlstderr = readline.createInterface(p.stderr);
+      rlstderr.on("line", (line) => {
+        logger.warn({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
+      });
     }
 
     setTimeout(() => {
       if (jobs.delete(id)) {
         resolve({
           fault: { code: "timeout", message: "Extension timed out" },
-          value: null
+          value: null,
         });
       }
     }, TIMEOUT);
@@ -85,7 +81,7 @@ export function run(args): Promise<{ fault: Fault; value: any }> {
 }
 
 function kill(process: ChildProcess): Promise<void> {
-  return new Promise(resolve => {
+  return new Promise((resolve) => {
     const timeToKill = Date.now() + 5000;
 
     process.kill();
@@ -108,6 +104,6 @@ export async function killAll(): Promise<void> {
     Object.entries(processes).map(([k, p]) => {
       delete processes[k];
       return kill(p);
-    })
+    }),
   );
 }

lib/forwarded.ts

@@ -0,0 +1,160 @@
+import { IncomingMessage } from "node:http";
+import { TLSSocket } from "node:tls";
+import { parseCIDR, parse, IPv6, IPv4 } from "ipaddr.js";
+import * as config from "./config.ts";
+import { getSocketEndpoints } from "./server.ts";
+
+interface RequestOrigin {
+  localAddress: string;
+  localPort: number;
+  remoteAddress: string;
+  remotePort: number;
+  host: string;
+  encrypted: boolean;
+}
+
+const FORWARDED_HEADER = "" + config.get("FORWARDED_HEADER");
+const cache = new WeakMap<IncomingMessage, RequestOrigin>();
+const cidrs: [IPv4 | IPv6, number][] = [];
+
+for (const str of FORWARDED_HEADER.split(",").map((s) => s.trim())) {
+  try {
+    cidrs.push(parseCIDR(str));
+  } catch {
+    // Not a valid CIDR format, try parsing as IP
+    try {
+      const ip = parse(str);
+      cidrs.push([ip, ip.toByteArray().length * 8]);
+    } catch {
+      // Not a valid IP either, ignore
+    }
+  }
+}
+
+function parseForwardedHeader(str: string): { [name: string]: string } {
+  str = str.toLowerCase();
+  const res: { [name: string]: string } = {};
+  let keyIdx = 0;
+  let valueIdx = -1;
+  let key: string;
+  for (let i = 0; i < str.length; ++i) {
+    const char = str.charCodeAt(i);
+    if (char === 61 /* = */) {
+      if (keyIdx >= 0) {
+        key = str.slice(keyIdx, i).trim();
+        keyIdx = -1;
+        valueIdx = i + 1;
+      }
+    } else if (char === 59 /* ; */) {
+      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
+      valueIdx = -1;
+      keyIdx = i + 1;
+    } else if (char === 44 /* , */) {
+      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
+      return res;
+    } else if (char === 34 /* " */) {
+      if (valueIdx >= 0) {
+        const quoteIdx = i;
+        if (!str.slice(valueIdx, quoteIdx).trim()) {
+          for (i = i + 1; i < str.length; ++i) {
+            const c = str.charCodeAt(i);
+            if (c === 92 /* \ */) ++i;
+            if (c === 34 /* " */) {
+              res[key] = JSON.parse(str.slice(quoteIdx, i + 1).trim());
+              valueIdx = -1;
+              keyIdx = i + 1;
+              break;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  if (valueIdx >= 0) res[key] = str.slice(valueIdx).trim();
+
+  return res;
+}
+
+export function getRequestOrigin(request: IncomingMessage): RequestOrigin {
+  let origin = cache.get(request);
+  if (!origin) {
+    const soc = request.socket;
+    const socketEndpoints = getSocketEndpoints(soc);
+    origin = {
+      localAddress: socketEndpoints.localAddress,
+      localPort: socketEndpoints.localPort,
+      remoteAddress: socketEndpoints.remoteAddress,
+      remotePort: socketEndpoints.remotePort,
+      host: request.headers["host"],
+      encrypted: !!(request.socket as TLSSocket).encrypted,
+    };
+
+    const header = request.headers["forwarded"];
+    if (header) {
+      const ip = parse(socketEndpoints.remoteAddress);
+      if (
+        cidrs.some((cidr) => ip.kind() === cidr[0].kind() && ip.match(cidr))
+      ) {
+        const parsed = parseForwardedHeader(header);
+
+        if (parsed["proto"] === "https") {
+          origin.encrypted = true;
+          origin.localPort = 443;
+        } else if (parsed["proto"] === "http") {
+          origin.encrypted = false;
+          origin.localPort = 80;
+        }
+
+        if (parsed["host"]) {
+          origin.host = parsed["host"];
+          const [, port] = parsed["host"].split(":", 2);
+          origin.localPort = +port || origin.localPort;
+        }
+
+        if (parsed["for"]) {
+          if (parsed["for"].startsWith("[")) {
+            const i = parsed["for"].lastIndexOf("]");
+            if (i >= 0) {
+              origin.remoteAddress = parsed["for"].slice(1, i);
+              origin.remotePort =
+                parseInt(parsed["for"].slice(i + 2)) || origin.remotePort;
+            }
+          } else {
+            const i = parsed["for"].lastIndexOf(":");
+            if (i >= 0) {
+              origin.remoteAddress = parsed["for"].slice(0, i);
+              origin.remotePort =
+                parseInt(parsed["for"].slice(i + 1)) || origin.remotePort;
+            } else {
+              origin.remoteAddress = parsed["for"];
+            }
+          }
+        }
+
+        if (parsed["by"]) {
+          if (parsed["by"].startsWith("[")) {
+            const i = parsed["by"].lastIndexOf("]");
+            if (i >= 0) {
+              origin.localAddress = parsed["by"].slice(1, i);
+              origin.localPort =
+                parseInt(parsed["by"].slice(i + 2)) || origin.localPort;
+            }
+          } else {
+            const i = parsed["by"].lastIndexOf(":");
+            if (i >= 0) {
+              origin.localAddress = parsed["by"].slice(0, i);
+              origin.localPort =
+                parseInt(parsed["by"].slice(i + 1)) || origin.localPort;
+            } else {
+              origin.localAddress = parsed["by"];
+            }
+          }
+        }
+      }
+    }
+
+    cache.set(request, origin);
+  }
+  return origin;
+}

lib/fs.ts

@@ -1,63 +1,203 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
+import * as url from "node:url";
+import { IncomingMessage, ServerResponse } from "node:http";
+import { PassThrough, pipeline, Readable } from "node:stream";
+import { createHash } from "node:crypto";
+import { filesBucket, collections } from "./db/db.ts";
+import * as logger from "./logger.ts";
+import { getRequestOrigin } from "./forwarded.ts";
+import memoize from "./common/memoize.ts";
 
-import * as url from "url";
-import * as querystring from "querystring";
-import { GridFSBucket } from "mongodb";
-import * as db from "./db";
-import * as logger from "./logger";
+const getFile = memoize(
+  async (
+    etag: string,
+    size: number,
+    filename: string,
+  ): Promise<Iterable<Buffer>> => {
+    const chunks: Buffer[] = [];
+    // Using for-await over the download stream can throw ERR_STREAM_PREMATURE_CLOSE
+    // for very small files. Possibly a bug in MongoDB driver or Nodejs itself.
+    // Using a PassThrough stream to avoid this.
+    const downloadStream = pipeline(
+      filesBucket.openDownloadStreamByName(filename),
+      new PassThrough(),
+      (err) => {
+        if (err) throw err;
+      },
+    );
+    for await (const chunk of downloadStream) chunks.push(chunk);
+    // Node 12-14 don't throw error when stream is closed prematurely.
+    // However, we don't need to check for that since we're checking file size.
+    if (size !== chunks.reduce((a, b) => a + b.length, 0))
+      throw new Error("File size mismatch");
+    return chunks;
+  },
+);
 
-export function listener(request, response): void {
-  const urlParts = url.parse(request.url, true);
-  if (request.method === "GET") {
-    const filename = querystring.unescape(urlParts.pathname.substring(1));
+async function* partialContent(
+  chunks: Iterable<Buffer>,
+  start: number,
+  end: number,
+): AsyncIterable<Buffer> {
+  let bytesToSkip = start;
+  let bytesToRead = end - start;
 
-    const log = {
-      message: "Fetch file",
-      filename: filename,
-      remoteAddress: request.connection.remoteAddress
-    };
-
-    db.filesCollection.findOne({ _id: filename }, (err, file) => {
-      if (err) throw err;
-
-      if (!file) {
-        response.writeHead(404);
-        response.end();
-        log.message += " not found";
-        logger.accessError(log);
-        return;
-      }
-
-      response.writeHead(200, {
-        "Content-Type": file.contentType || "application/octet-stream",
-        "Content-Length": file.length
-      });
-
-      const bucket = new GridFSBucket(db.client.db());
-      const downloadStream = bucket.openDownloadStreamByName(filename);
-      downloadStream.pipe(response);
-
-      logger.accessInfo(log);
-    });
-  } else {
-    response.writeHead(405, { Allow: "GET" });
-    response.end("405 Method Not Allowed");
+  for (let chunk of chunks) {
+    if (bytesToRead <= 0) return;
+    if (bytesToSkip >= chunk.length) {
+      bytesToSkip -= chunk.length;
+      continue;
+    }
+    chunk = chunk.subarray(bytesToSkip, bytesToSkip + bytesToRead);
+    bytesToRead -= chunk.length;
+    bytesToSkip = 0;
+    yield chunk;
   }
 }
+
+function generateETag(file: {
+  _id: string;
+  uploadDate: Date;
+  length: number;
+}): string {
+  const hash = createHash("md5");
+  hash.update(`${file._id}-${file.uploadDate.getTime()}-${file.length}`);
+  return hash.digest("hex");
+}
+
+function matchEtag(etag: string, header: string): boolean {
+  for (let t of header.split(",")) {
+    t = t.trim();
+    if (t.startsWith("W/")) t = t.substring(2);
+    try {
+      t = JSON.parse(t);
+    } catch {
+      // Ignore
+    }
+    if (t === "*") return true;
+    if (etag === t) return true;
+  }
+  return false;
+}
+
+export async function listener(
+  request: IncomingMessage,
+  response: ServerResponse,
+): Promise<void> {
+  if (request.method !== "GET" && request.method !== "HEAD") {
+    response.writeHead(405, { Allow: "GET, HEAD" });
+    response.end("405 Method Not Allowed");
+    return;
+  }
+
+  const urlParts = url.parse(request.url, true);
+  const filename = decodeURIComponent(urlParts.pathname.substring(1));
+
+  const log = {
+    message: "Fetch file",
+    filename: filename,
+    remoteAddress: getRequestOrigin(request).remoteAddress,
+    method: request.method,
+  };
+
+  const file = await collections.files.findOne({ _id: filename });
+
+  if (!file) {
+    response.writeHead(404);
+    response.end();
+    log.message += " not found";
+    logger.accessError(log);
+    return;
+  }
+
+  logger.accessInfo(log);
+
+  const etag = generateETag(file);
+  const lastModified = file["uploadDate"];
+  lastModified.setMilliseconds(0);
+
+  let status = 200;
+  let start = 0;
+  let end = file.length;
+
+  if (request.headers["if-match"])
+    if (!matchEtag(etag, request.headers["if-match"])) status = 412;
+
+  if (request.headers["if-unmodified-since"]) {
+    const d = new Date(request.headers["if-unmodified-since"]);
+    if (lastModified > d) status = 412;
+  }
+
+  if (request.headers["if-none-match"]) {
+    if (matchEtag(etag, request.headers["if-none-match"])) status = 304;
+  } else if (request.headers["if-modified-since"]) {
+    const d = new Date(request.headers["if-modified-since"]);
+    if (lastModified <= d) status = 304;
+  }
+
+  if (request.headers.range && status === 200) {
+    const match = request.headers.range.match(/^bytes=(\d*)-(\d*)$/);
+    status = 416;
+    if (match && (match[1] || match[2])) {
+      if (match[2]) end = parseInt(match[2]) + 1;
+      if (match[1]) start = parseInt(match[1]);
+      else start = file.length - parseInt(match[2]);
+      if (start < end && end <= file.length) status = 206;
+    }
+
+    if (request.headers["if-range"]) {
+      const h = request.headers["if-range"] as string;
+      const d = new Date(h);
+      if (!matchEtag(etag, h) && !(lastModified <= d)) {
+        status = 200;
+        start = 0;
+        end = file.length;
+      }
+    }
+  }
+
+  if (status === 412) {
+    response.writeHead(412);
+    response.end();
+    return;
+  }
+
+  if (status === 304) {
+    response.writeHead(304, {
+      ETag: etag,
+      "Last-Modified": lastModified.toUTCString(),
+    });
+    response.end();
+    return;
+  }
+
+  if (status === 416) {
+    response.writeHead(416, {
+      "Content-Range": `bytes */${file.length}`,
+      "Content-Length": "0",
+    });
+    response.end();
+    return;
+  }
+
+  response.writeHead(status, {
+    "Content-Type": "application/octet-stream",
+    "Content-Length": end - start,
+    "Accept-Ranges": "bytes",
+    ETag: etag,
+    "Last-Modified": lastModified.toUTCString(),
+    ...(status === 206 && {
+      "Content-Range": `bytes ${start}-${end - 1}/${file.length}`,
+    }),
+  });
+
+  if (request.method === "HEAD") {
+    response.end();
+    return;
+  }
+
+  const chunks = await getFile(etag, file.length, filename);
+
+  pipeline(Readable.from(partialContent(chunks, start, end)), response, () => {
+    // Ignore errors resulting from client disconnecting
+  });
+}

lib/gpn-heuristic.ts

@@ -1,21 +1,4 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
+import Path from "./common/path.ts";
 
 const WILDCARD_MULTIPLIER = 2;
 const UNDISCOVERED_DEPTH = 7;
@@ -27,9 +10,12 @@ const UNDISCOVERED_DEPTH = 7;
 // flags is an int where its bits mark the segments in the pattern that
 // need refreshing. Leading 0s indicate that the pattern up to that
 // point has been discovered.
-export function estimateGpnCount(gpnPatterns, depth = 0): number {
-  const children = {};
-  const wildcardChildren = [];
+export function estimateGpnCount(
+  gpnPatterns: [Path, number][],
+  depth = 0,
+): number {
+  const children: { [segment: string]: [Path, number][] } = {};
+  const wildcardChildren: [Path, number][] = [];
   let wildcardDiscovered = false;
   let gpnCount = 0;
 
@@ -37,7 +23,7 @@ export function estimateGpnCount(gpnPatterns, depth = 0): number {
     const path = pattern[0];
     const flags = pattern[1] >> depth;
 
-    const k = path[depth];
+    const k = path.segments[depth] as string;
 
     if (!k) {
       if (flags & 1) gpnCount = 1;

lib/init.ts

@@ -1,26 +1,17 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as localCache from "./local-cache";
-import { generateSalt, hashPassword } from "./auth";
-import * as db from "./ui/db";
-import { del } from "./cache";
+import { getRevision, getUiConfig, getUsers } from "./ui/local-cache.ts";
+import { generateSalt, hashPassword } from "./auth.ts";
+import { collections } from "./db/db.ts";
+import {
+  putConfig,
+  putPermission,
+  putPreset,
+  putProvision,
+  putUser,
+} from "./ui/db.ts";
+import { del } from "./cache.ts";
+import BOOTSTRAP_SCRIPT from "../seed/bootstrap.js" with { type: "text" };
+import DEFAULT_SCRIPT from "../seed/default.js" with { type: "text" };
+import INFORM_SCRIPT from "../seed/inform.js" with { type: "text" };
 
 interface Status {
   users: boolean;
@@ -31,75 +22,34 @@ interface Status {
   overview: boolean;
 }
 
-const BOOTSTRAP_SCRIPT = `
-const now = Date.now();
-
-// Clear cached data model to force a refresh
-clear("Device", now);
-clear("InternetGatewayDevice", now);
-`.trim();
-
-const DEFAULT_SCRIPT = `
-const hourly = Date.now(3600000);
-
-// Refresh basic parameters hourly
-declare("InternetGatewayDevice.DeviceInfo.HardwareVersion", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.DeviceInfo.SoftwareVersion", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID", {path: hourly, value: hourly});
-// Don't refresh password field periodically because CPEs always report blank passowrds for security reasons
-declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase", {path: hourly, value: 1});
-declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress", {path: hourly, value: hourly});
-`.trim();
-
-const INFORM_SCRIPT = `
-// Device ID as user name
-const username = declare("DeviceID.ID", {value: 1}).value[0]
-
-// Password will be fixed a given device because Math.random() is seeded with devcie ID by default.
-const password = "" + Math.random();
-
-const informInterval = 300;
-
-// Refresh values daily
-const daily = Date.now(86400000);
-
-// Unique inform offset per device for better load distribution
-const informTime = daily % 86400000;
-
-declare("InternetGatewayDevice.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
-declare("InternetGatewayDevice.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
-declare("InternetGatewayDevice.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
-declare("InternetGatewayDevice.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
-declare("InternetGatewayDevice.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
-
-declare("Device.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
-declare("Device.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
-declare("Device.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
-declare("Device.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
-declare("Device.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
-`.trim();
-
 export async function getStatus(): Promise<Status> {
-  const configSnapshot = await localCache.getCurrentSnapshot();
-  const users = localCache.getUsers(configSnapshot);
-  const presets = localCache.getPresets(configSnapshot);
-  const ui = localCache.getUiConfig(configSnapshot);
+  const [configSnapshot, presetCount] = await Promise.all([
+    getRevision(),
+    collections.presets.countDocuments(),
+  ]);
+  const users = getUsers(configSnapshot);
+  const ui = getUiConfig(configSnapshot);
 
-  return {
+  const status = {
     users: !Object.keys(users).length,
-    presets: !presets.length,
-    filters: !Object.keys(ui["filters"]).length,
-    device: !Object.keys(ui["device"]).length,
-    index: !Object.keys(ui["index"]).length,
-    overview: !Object.keys(ui["overview"]).length
+    presets: !presetCount,
+    filters: true,
+    device: true,
+    index: true,
+    overview: true,
   };
+
+  for (const k of Object.keys(ui)) {
+    if (k.startsWith("filters.")) status.filters = false;
+    if (k.startsWith("device.")) status.device = false;
+    if (k.startsWith("index.")) status.index = false;
+    if (k.startsWith("overview.")) status.overview = false;
+  }
+
+  return status;
 }
 
-export async function seed(options): Promise<void> {
+export async function seed(options: Record<string, boolean>): Promise<void> {
   const resources = {};
   const proms = [];
 
@@ -117,12 +67,12 @@ export async function seed(options): Promise<void> {
         role: "admin",
         resource: "virtualParameters",
         access: 3,
-        validate: "true"
-      }
+        validate: "true",
+      },
     ];
 
     resources["users"] = [
-      { username: "admin", password: "admin", roles: ["admin"] }
+      { username: "admin", password: "admin", roles: ["admin"] },
     ];
   }
 
@@ -135,8 +85,7 @@ export async function seed(options): Promise<void> {
       { _id: "ui.filters.1.parameter", value: "DeviceID.ProductClass" },
       { _id: "ui.filters.1.type", value: "'string'" },
       { _id: "ui.filters.2.label", value: "'Tag'" },
-      { _id: "ui.filters.2.parameter", value: "tag" },
-      { _id: "ui.filters.2.type", value: "'string'" }
+      { _id: "ui.filters.2.type", value: "'tag'" },
     ]);
   }
 
@@ -150,118 +99,118 @@ export async function seed(options): Promise<void> {
       { _id: "ui.device.2.parameters.0.label", value: "'Last inform'" },
       {
         _id: "ui.device.2.parameters.0.components.0.type",
-        value: "'parameter'"
+        value: "'parameter'",
       },
       { _id: "ui.device.2.parameters.0.components.1.chart", value: "'online'" },
       {
         _id: "ui.device.2.parameters.0.components.1.type",
-        value: "'overview-dot'"
+        value: "'overview-dot'",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.type",
-        value: "'summon-button'"
+        value: "'summon-button'",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.0",
-        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.1",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.2",
         value:
-          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress"
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.3",
         value:
-          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress"
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.4",
-        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID"
+        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.5",
         value:
-          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase"
+          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.6",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName"
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.7",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress"
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress",
       },
       {
         _id: "ui.device.2.parameters.0.components.2.parameters.8",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress"
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress",
       },
       {
         _id: "ui.device.2.parameters.0.parameter",
-        value: "DATE_STRING(Events.Inform)"
+        value: "DATE_STRING(Events.Inform)",
       },
       { _id: "ui.device.2.parameters.1.label", value: "'Serial number'" },
       {
         _id: "ui.device.2.parameters.1.parameter",
-        value: "DeviceID.SerialNumber"
+        value: "DeviceID.SerialNumber",
       },
       { _id: "ui.device.2.parameters.2.label", value: "'Product class'" },
       {
         _id: "ui.device.2.parameters.2.parameter",
-        value: "DeviceID.ProductClass"
+        value: "DeviceID.ProductClass",
       },
       { _id: "ui.device.2.parameters.3.label", value: "'OUI'" },
       {
         _id: "ui.device.2.parameters.3.parameter",
-        value: "DeviceID.OUI"
+        value: "DeviceID.OUI",
       },
       { _id: "ui.device.2.parameters.4.label", value: "'Manufacturer'" },
       {
         _id: "ui.device.2.parameters.4.parameter",
-        value: "DeviceID.Manufacturer"
+        value: "DeviceID.Manufacturer",
       },
       { _id: "ui.device.2.parameters.5.label", value: "'Hardware version'" },
       {
         _id: "ui.device.2.parameters.5.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
       },
       { _id: "ui.device.2.parameters.6.label", value: "'Software version'" },
       {
         _id: "ui.device.2.parameters.6.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
       },
       { _id: "ui.device.2.parameters.7.label", value: "'MAC'" },
       {
         _id: "ui.device.2.parameters.7.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress"
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress",
       },
       { _id: "ui.device.2.parameters.8.label", value: "'IP'" },
       {
         _id: "ui.device.2.parameters.8.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
       },
       { _id: "ui.device.2.parameters.9.label", value: "'WLAN SSID'" },
       {
         _id: "ui.device.2.parameters.9.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
       },
       { _id: "ui.device.2.parameters.10.label", value: "'WLAN passphrase'" },
       {
         _id: "ui.device.2.parameters.10.parameter",
         value:
-          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase"
+          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
       },
       { _id: "ui.device.3.type", value: "'parameter-table'" },
-      { _id: "ui.device.3.label", value: "'WLAN hosts'" },
+      { _id: "ui.device.3.label", value: "'LAN hosts'" },
       {
         _id: "ui.device.3.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host"
+        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host",
       },
       { _id: "ui.device.3.childParameters.0.label", value: "'Host name'" },
       { _id: "ui.device.3.childParameters.0.parameter", value: "HostName" },
@@ -279,12 +228,12 @@ export async function seed(options): Promise<void> {
       { _id: "ui.device.5.element", value: "'div.container-full-width'" },
       {
         _id: "ui.device.5.components.0.components.0",
-        value: "'All parameters'"
+        value: "'All parameters'",
       },
       { _id: "ui.device.5.components.0.element", value: "'h3'" },
       { _id: "ui.device.5.components.0.type", value: "'container'" },
       { _id: "ui.device.5.components.1.type", value: "'all-parameters'" },
-      { _id: "ui.device.6.type", value: "'device-actions'" }
+      { _id: "ui.device.6.type", value: "'device-actions'" },
     ]);
   }
 
@@ -299,18 +248,18 @@ export async function seed(options): Promise<void> {
       { _id: "ui.index.2.label", value: "'Software version'" },
       {
         _id: "ui.index.2.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion"
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
       },
       { _id: "ui.index.3.label", value: "'IP'" },
       {
         _id: "ui.index.3.parameter",
         value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress"
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
       },
       { _id: "ui.index.4.label", value: "'SSID'" },
       {
         _id: "ui.index.4.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID"
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
       },
       { _id: "ui.index.5.type", value: "'container'" },
       { _id: "ui.index.5.label", value: "'Last inform'" },
@@ -323,7 +272,7 @@ export async function seed(options): Promise<void> {
       { _id: "ui.index.6.label", value: "'Tags'" },
       { _id: "ui.index.6.parameter", value: "Tags" },
       { _id: "ui.index.6.unsortable", value: "true" },
-      { _id: "ui.index.6.writable", value: "false" }
+      { _id: "ui.index.6.writable", value: "false" },
     ]);
   }
 
@@ -332,43 +281,44 @@ export async function seed(options): Promise<void> {
       { _id: "ui.overview.charts.online.label", value: "'Online status'" },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.color",
-        value: "'#31a354'"
+        value: "'#31a354'",
       },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.filter",
-        value: "Events.Inform > NOW() - 5 * 60 * 1000"
+        value: "Events.Inform > NOW() - 5 * 60 * 1000",
       },
       {
         _id: "ui.overview.charts.online.slices.1_onlineNow.label",
-        value: "'Online now'"
+        value: "'Online now'",
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.color",
-        value: "'#a1d99b'"
+        value: "'#a1d99b'",
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.filter",
         value:
-          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)"
+          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)",
       },
       {
         _id: "ui.overview.charts.online.slices.2_past24.label",
-        value: "'Past 24 hours'"
+        value: "'Past 24 hours'",
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.color",
-        value: "'#e5f5e0'"
+        value: "'#e5f5e0'",
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.filter",
-        value: "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)"
+        value:
+          "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)",
       },
       {
         _id: "ui.overview.charts.online.slices.3_others.label",
-        value: "'Others'"
+        value: "'Others'",
       },
       { _id: "ui.overview.groups.online.label", value: "''" },
-      { _id: "ui.overview.groups.online.charts.0", value: "'online'" }
+      { _id: "ui.overview.groups.online.charts.0", value: "'online'" },
     ]);
   }
 
@@ -378,23 +328,24 @@ export async function seed(options): Promise<void> {
         _id: "bootstrap",
         weight: 0,
         channel: "bootstrap",
-        provision: "bootstrap"
+        events: "0 BOOTSTRAP",
+        provision: "bootstrap",
       },
       { _id: "default", weight: 0, channel: "default", provision: "default" },
-      { _id: "inform", weight: 0, channel: "inform", provision: "inform" }
+      { _id: "inform", weight: 0, channel: "inform", provision: "inform" },
     ];
 
     resources["provisions"] = [
       { _id: "bootstrap", script: BOOTSTRAP_SCRIPT },
       { _id: "default", script: DEFAULT_SCRIPT },
-      { _id: "inform", script: INFORM_SCRIPT }
+      { _id: "inform", script: INFORM_SCRIPT },
     ];
   }
 
   if (resources["permissions"]) {
     for (const p of resources["permissions"]) {
       p["_id"] = `${p["role"]}:${p["resource"]}:${p["access"]}`;
-      proms.push(db.putPermission(p["_id"], p));
+      proms.push(putPermission(p["_id"], p));
     }
   }
 
@@ -405,21 +356,21 @@ export async function seed(options): Promise<void> {
       u["roles"] = (u["roles"] || []).join(",");
       u["_id"] = u["username"];
       delete u["username"];
-      proms.push(db.putUser(u["_id"], u));
+      proms.push(putUser(u["_id"], u));
     }
   }
 
   if (resources["provisions"]) {
     for (const p of resources["provisions"])
-      proms.push(db.putProvision(p["_id"], p));
+      proms.push(putProvision(p["_id"], p));
   }
 
   if (resources["presets"])
-    for (const p of resources["presets"]) proms.push(db.putPreset(p["_id"], p));
+    for (const p of resources["presets"]) proms.push(putPreset(p["_id"], p));
 
   if (resources["config"])
-    for (const c of resources["config"]) proms.push(db.putConfig(c["_id"], c));
+    for (const c of resources["config"]) proms.push(putConfig(c["_id"], c));
 
   await proms;
-  return del("presets_hash");
+  await Promise.all([del("ui-local-cache-hash"), del("cwmp-local-cache-hash")]);
 }

lib/instance-set.ts

@@ -1,28 +1,9 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 interface Instance {
   [name: string]: string;
 }
 
 export default class InstanceSet {
-  private set: Set<Instance>;
+  declare private set: Set<Instance>;
 
   public constructor() {
     this.set = new Set();

lib/local-cache.ts

@@ -1,524 +1,73 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
+import { setTimeoutPromise } from "./util.ts";
+import { get, set } from "./cache.ts";
+import { acquireLock, releaseLock } from "./lock.ts";
 
-import * as vm from "vm";
-import * as crypto from "crypto";
-import * as config from "./config";
-import * as db from "./db";
-import * as cache from "./cache";
-import { mongoQueryToFilter } from "./mongodb-functions";
-import * as logger from "./logger";
-import * as scheduling from "./scheduling";
-import * as expression from "./common/expression";
-import { parse } from "./common/expression-parser";
-import { Preset, Expression } from "./types";
+const REFRESH = 5000;
+const EVICT_TIMEOUT = 120000;
 
-interface Permissions {
-  [role: string]: {
-    [access: number]: {
-      [resource: string]: {
-        access: number;
-        filter: Expression;
-        validate?: Expression;
-      };
-    };
-  };
-}
+export class LocalCache<T> {
+  private nextRefresh = 1;
+  private currentRevision: string = null;
+  private snapshots: Map<string, T> = new Map();
 
-const REFRESH = 3000;
-const EVICT_TIMEOUT = 60000;
+  constructor(
+    private cacheKey: string,
+    private callback: () => Promise<[string, T]>,
+  ) {}
 
-const snapshots = new Map();
-let currentSnapshot = null;
-let nextRefresh = 1;
-
-function computeHash(snapshot): string {
-  // MD5 hash for presets, provisions, virtual parameters for detecting changes
-  const h = crypto.createHash("md5");
-  for (const p of snapshot.presets) {
-    h.update(JSON.stringify(p.name));
-    h.update(JSON.stringify(p.channel));
-    h.update(JSON.stringify(p.schedule));
-    h.update(JSON.stringify(p.events));
-    h.update(JSON.stringify(p.precondition));
-    h.update(JSON.stringify(p.provisions));
+  async getRevision(): Promise<string> {
+    if (Date.now() > this.nextRefresh) await this.refresh();
+    return this.currentRevision;
   }
 
-  let keys;
-
-  keys = Object.keys(snapshot.provisions).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(snapshot.provisions[k].md5);
-
-  keys = Object.keys(snapshot.virtualParameters).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(snapshot.virtualParameters[k].md5);
-
-  keys = Object.keys(snapshot.config).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.config[k]));
-
-  keys = Object.keys(snapshot.files).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.files[k]));
-
-  keys = Object.keys(snapshot.users).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.users[k]));
-
-  const roles = Object.keys(snapshot.permissions).sort();
-  h.update(JSON.stringify(roles));
-  for (const r of roles) {
-    const levels = Object.keys(snapshot.permissions[r]).sort();
-    h.update(JSON.stringify(levels));
-    for (const l of levels) {
-      keys = Object.keys(snapshot.permissions[r][l]).sort();
-      h.update(JSON.stringify(keys));
-      for (const k of keys)
-        h.update(JSON.stringify(snapshot.permissions[r][l][k]));
-    }
+  hasRevision(revision: string): boolean {
+    return this.snapshots.has(revision);
   }
 
-  return h.digest("hex");
-}
-
-function flattenObject(src, prefix = "", dst = {}): {} {
-  for (const k of Object.keys(src)) {
-    const v = src[k];
-    if (typeof v === "object" && !Array.isArray(v))
-      flattenObject(v, `${prefix}${k}.`, dst);
-    else dst[`${prefix}${k}`] = v;
+  get(revision: string): T {
+    const snapshot = this.snapshots.get(revision);
+    if (!snapshot) throw new Error("Cache snapshot does not exist");
+    return snapshot;
   }
-  return dst;
-}
 
-async function fetchPresets(): Promise<Preset[]> {
-  const res = await db.getPresets();
-  let objects = await db.getObjects();
-
-  objects = objects.map(obj => {
-    // Flatten object
-    obj = flattenObject(obj);
-
-    // If no keys are defined, consider all parameters as keys to keep the
-    // same behavior from v1.0
-    if (!obj["_keys"] || !obj["_keys"].length)
-      obj["_keys"] = Object.keys(obj).filter(k => !k.startsWith("_"));
-
-    return obj;
-  });
-
-  res.sort((a, b) => {
-    if (a["weight"] === b["weight"])
-      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
-    else return a["weight"] - b["weight"];
-  });
-
-  const presets = [] as Preset[];
-  for (const preset of res) {
-    let schedule: { md5: string; duration: number; schedule: any } = null;
-    if (preset["schedule"]) {
-      const parts = preset["schedule"].trim().split(/\s+/);
-      schedule = {
-        md5: crypto
-          .createHash("md5")
-          .update(preset["schedule"])
-          .digest("hex"),
-        duration: null,
-        schedule: null
-      };
-
-      try {
-        schedule.duration = +parts.shift() * 1000;
-        schedule.schedule = scheduling.parseCron(parts.join(" "));
-      } catch (err) {
-        logger.warn({
-          message: "Invalid preset schedule",
-          preset: preset["_id"],
-          schedule: preset["schedule"]
-        });
-        schedule.schedule = false;
-      }
+  async refresh(): Promise<void> {
+    if (!this.nextRefresh) {
+      await setTimeoutPromise(20);
+      await this.refresh();
+      return;
     }
 
-    const events = preset["events"] || {};
-    let precondition = true as Expression;
-    if (preset["precondition"]) {
-      try {
-        precondition = parse(preset["precondition"]);
-      } catch (error) {
-        precondition = mongoQueryToFilter(JSON.parse(preset["precondition"]));
-      }
+    const now = Date.now();
+    if (now < this.nextRefresh) return;
+    this.nextRefresh = 0;
 
-      // Simplify expression
-      precondition = expression.evaluate(precondition);
+    const dbHash = await get(this.cacheKey);
+
+    if (this.currentRevision && dbHash === this.currentRevision) {
+      this.nextRefresh = now + (REFRESH - (now % REFRESH));
+      return;
     }
 
-    const _provisions = preset["provisions"] || [];
+    const lockToken = await acquireLock(this.cacheKey, 5000);
 
-    // Generate provisions from the old configuration format
-    for (const c of preset["configurations"]) {
-      switch (c.type) {
-        case "age":
-          _provisions.push(["refresh", c.name, c.age]);
-          break;
+    const [hash, snapshot] = await this.callback();
 
-        case "value":
-          _provisions.push(["value", c.name, c.value]);
-          break;
-
-        case "add_tag":
-          _provisions.push(["tag", c.tag, true]);
-          break;
-
-        case "delete_tag":
-          _provisions.push(["tag", c.tag, false]);
-          break;
-
-        case "provision":
-          _provisions.push([c.name].concat(c.args || []));
-          break;
-
-        case "add_object":
-          for (const obj of objects) {
-            if (obj["_id"] === c.object) {
-              const alias = obj["_keys"]
-                .map(k => `${k}:${JSON.stringify(obj[k])}`)
-                .join(",");
-              const p = `${c.name}.[${alias}]`;
-              _provisions.push(["instances", p, 1]);
-
-              for (const k in obj) {
-                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
-                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
-              }
-            }
-          }
-
-          break;
-
-        case "delete_object":
-          for (const obj of objects) {
-            if (obj["_id"] === c.object) {
-              const alias = obj["_keys"]
-                .map(k => `${k}:${JSON.stringify(obj[k])}`)
-                .join(",");
-              const p = `${c.name}.[${alias}]`;
-              _provisions.push(["instances", p, 0]);
-            }
-          }
-
-          break;
-
-        default:
-          throw new Error(`Unknown configuration type ${c.type}`);
-      }
+    if (this.currentRevision) {
+      const r = this.currentRevision;
+      const s = this.snapshots.get(r);
+      setTimeout(() => {
+        if (this.snapshots.get(r) === s) this.snapshots.delete(r);
+      }, EVICT_TIMEOUT).unref();
     }
 
-    presets.push({
-      name: preset["_id"],
-      channel: (preset["channel"] as string) || "default",
-      schedule: schedule,
-      events: events,
-      precondition: precondition,
-      provisions: _provisions
-    });
-  }
+    this.currentRevision = hash;
+    this.snapshots.set(hash, snapshot);
 
-  return presets;
-}
-
-async function fetchProvisions(): Promise<{}> {
-  const res = await db.getProvisions();
-
-  const provisions = {};
-  for (const r of res) {
-    provisions[r["_id"]] = {};
-    provisions[r["_id"]].md5 = crypto
-      .createHash("md5")
-      .update(r["script"])
-      .digest("hex");
-    provisions[r["_id"]].script = new vm.Script(
-      `"use strict";(function(){\n${r["script"]}\n})();`,
-      { filename: r["_id"], lineOffset: -1, timeout: 50 }
-    );
-  }
-
-  return provisions;
-}
-
-async function fetchVirtualParameters(): Promise<{}> {
-  const res = await db.getVirtualParameters();
-
-  const virtualParameters = {};
-  for (const r of res) {
-    virtualParameters[r["_id"]] = {};
-    virtualParameters[r["_id"]].md5 = crypto
-      .createHash("md5")
-      .update(r["script"])
-      .digest("hex");
-    virtualParameters[r["_id"]].script = new vm.Script(
-      `"use strict";(function(){\n${r["script"]}\n})();`,
-      { filename: r["_id"], lineOffset: -1, timeout: 50 }
-    );
-  }
-
-  return virtualParameters;
-}
-
-async function fetchPermissions(): Promise<Permissions> {
-  const perms = await db.getPermissions();
-  const permissions: Permissions = {};
-
-  for (const p of perms) {
-    if (!permissions[p.role]) permissions[p.role] = {};
-    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
-
-    permissions[p.role][p.access][p.resource] = {
-      access: p.access,
-      filter: parse(p.filter || "true")
-    };
-    if (p.validate)
-      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
-  }
-
-  return permissions;
-}
-
-async function fetchFiles(): Promise<{}> {
-  const res = await db.getFiles();
-  const files = {};
-
-  for (const r of res) {
-    const id = r["filename"] || r["_id"].toString();
-    files[id] = {};
-    files[id].length = r["length"];
-    files[id].md5 = r["md5"];
-    files[id].contentType = r["contentType"];
-  }
-
-  return files;
-}
-
-async function fetchUsers(): Promise<{}> {
-  const _users = await db.getUsers();
-  const users = {};
-
-  for (const user of _users) {
-    users[user._id] = {
-      password: user.password,
-      salt: user.salt,
-      roles: user.roles.split(",").map(s => s.trim())
-    };
-  }
-
-  return users;
-}
-
-async function fetchConfig(): Promise<[{}, {}]> {
-  const conf = await db.getConfig();
-
-  conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
-
-  const ui = {
-    filters: {},
-    device: {},
-    index: {},
-    overview: {}
-  };
-  const _config = {};
-
-  for (const c of conf) {
-    // Evaluate expressions to simplify them
-    const val = expression.evaluate(c.value);
-    _config[c.id] = val;
-    if (c.id.startsWith("ui.")) {
-      const keys = c.id.split(".");
-      // remove the first key(ui)
-      keys.shift();
-      let ref = ui;
-      while (keys.length > 1) {
-        const k = keys.shift();
-        if (typeof ref[k] !== "object") ref[k] = {};
-        ref = ref[k];
-      }
-      ref[keys[0]] = val;
+    if (lockToken) {
+      if (hash !== dbHash) await set(this.cacheKey, hash, 300);
+      await releaseLock(this.cacheKey, lockToken);
     }
+
+    this.nextRefresh = now + (REFRESH - (now % REFRESH));
   }
-
-  return [_config, ui];
-}
-
-async function refresh(): Promise<void> {
-  if (!nextRefresh) {
-    await new Promise(resolve => setTimeout(resolve, 20));
-    await refresh();
-    return;
-  }
-
-  nextRefresh = 0;
-  const now = Date.now();
-
-  const dbHash = await cache.get("presets_hash");
-
-  if (currentSnapshot && dbHash === currentSnapshot) {
-    nextRefresh = now + (REFRESH - (now % REFRESH));
-    return;
-  }
-
-  const unlockOrExtend = await cache.lock("presets_hash_lock", 3);
-
-  const res = await Promise.all([
-    fetchPresets(),
-    fetchProvisions(),
-    fetchVirtualParameters(),
-    fetchFiles(),
-    fetchPermissions(),
-    fetchUsers(),
-    fetchConfig()
-  ]);
-
-  const snapshot = {
-    presets: res[0],
-    provisions: res[1],
-    virtualParameters: res[2],
-    files: res[3],
-    permissions: res[4],
-    users: res[5],
-    config: res[6][0],
-    ui: res[6][1]
-  };
-
-  if (currentSnapshot) {
-    const h = currentSnapshot;
-    const s = snapshots.get(h);
-    setTimeout(() => {
-      if (snapshots.get(h) === s) snapshots.delete(h);
-    }, EVICT_TIMEOUT).unref();
-  }
-
-  currentSnapshot = computeHash(snapshot);
-  snapshots.set(currentSnapshot, snapshot);
-  await cache.set("presets_hash", currentSnapshot, 300);
-  await unlockOrExtend(0);
-
-  nextRefresh = now + (REFRESH - (now % REFRESH));
-}
-
-export async function getCurrentSnapshot(): Promise<string> {
-  if (Date.now() > nextRefresh) await refresh();
-  return currentSnapshot;
-}
-
-export function hasSnapshot(hash): boolean {
-  return snapshots.has(hash);
-}
-
-export function getPresets(snapshotKey): Preset[] {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.presets;
-}
-
-export function getProvisions(
-  snapshotKey
-): { [name: string]: { md5: string; script: vm.Script } } {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.provisions;
-}
-
-export function getVirtualParameters(
-  snapshotKey
-): { [name: string]: { md5: string; script: vm.Script } } {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.virtualParameters;
-}
-
-export function getFiles(
-  snapshotKey
-): { [name: string]: { length: number; md5: string; contentType: string } } {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.files;
-}
-
-export function getConfig(
-  snapshotKey,
-  key,
-  context?,
-  now?
-): string | number | boolean | null {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  const oldOpts = {
-    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
-    "cwmp.debug": "DEBUG",
-    "cwmp.retryDelay": "RETRY_DELAY",
-    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
-    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
-    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
-    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
-    "cwmp.cookiesPath": "COOKIES_PATH",
-    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
-    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
-    "cwmp.connectionRequestAllowBasicAuth":
-      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
-    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
-    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
-    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT"
-  };
-
-  if (!(key in snapshot.config)) {
-    if (key in oldOpts)
-      return config.get(oldOpts[key], context ? context.id : null);
-    else return null;
-  }
-  return expression.evaluate(snapshot.config[key], context, now || Date.now());
-}
-
-export function getConfigExpression(snapshotKey, key): Expression {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.config[key];
-}
-
-export function getUsers(snapshotKey): {} {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.users;
-}
-
-export function getPermissions(snapshotKey): {} {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.permissions;
-}
-
-export function getUiConfig(snapshotKey): {} {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.ui;
 }

lib/lock.ts

@@ -0,0 +1,50 @@
+import { collections } from "./db/db.ts";
+
+const CLOCK_SKEW_TOLERANCE = 30000;
+
+export async function acquireLock(
+  lockName: string,
+  ttl: number,
+  timeout = 0,
+  token = Math.random().toString(36).slice(2),
+): Promise<string> {
+  try {
+    const now = Date.now();
+    const r = await collections.locks.findOneAndUpdate(
+      { _id: lockName, value: token },
+      {
+        $set: {
+          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
+        },
+        $currentDate: { timestamp: true },
+      },
+      { upsert: true, returnDocument: "after" },
+    );
+    if (Math.abs(r.value.timestamp.getTime() - now) > CLOCK_SKEW_TOLERANCE)
+      throw new Error("Database clock skew too great");
+  } catch (err) {
+    if (err.code !== 11000) throw err;
+    if (!(timeout > 0)) return null;
+    const w = 50 + Math.random() * 50;
+    await new Promise((resolve) => setTimeout(resolve, w));
+    return acquireLock(lockName, ttl, timeout - w, token);
+  }
+
+  return token;
+}
+
+export async function releaseLock(
+  lockName: string,
+  token: string,
+): Promise<void> {
+  const res = await collections.locks.deleteOne({
+    _id: lockName,
+    value: token,
+  });
+  if (res.deletedCount !== 1) throw new Error("Lock expired");
+}
+
+export async function getToken(lockName: string): Promise<string> {
+  const res = await collections.locks.findOne({ _id: lockName });
+  return res?.value;
+}

lib/logger.ts

@@ -1,26 +1,15 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as fs from "fs";
-import * as os from "os";
-
-import * as config from "./config";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as config from "./config.ts";
+import { getRequestOrigin } from "./forwarded.ts";
+import {
+  SessionContext,
+  AcsRequest,
+  CpeRequest,
+  CpeFault,
+  InformRequest,
+  Fault,
+} from "./types.ts";
 
 const REOPEN_EVERY = 60000;
 
@@ -66,7 +55,7 @@ function reopen(): void {
       if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
         logStream.end();
         logStream = fs.createWriteStream(null, {
-          fd: fs.openSync(LOG_FILE, "a")
+          fd: fs.openSync(LOG_FILE, "a"),
         });
         logStat = fs.fstatSync(logStream.fd);
       }
@@ -90,7 +79,7 @@ function reopen(): void {
       ) {
         accessLogStream.end();
         accessLogStream = fs.createWriteStream(null, {
-          fd: fs.openSync(ACCESS_LOG_FILE, "a")
+          fd: fs.openSync(ACCESS_LOG_FILE, "a"),
         });
         accessLogStat = fs.fstatSync(accessLogStream.fd);
       }
@@ -104,7 +93,7 @@ function reopen(): void {
     setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
 }
 
-export function init(service, version): void {
+export function init(service: string, version: string): void {
   defaultMeta.hostname = os.hostname();
   defaultMeta.pid = process.pid;
   defaultMeta.name = `genieacs-${service}`;
@@ -120,7 +109,7 @@ export function init(service, version): void {
 
   if (ACCESS_LOG_FILE) {
     accessLogStream = fs.createWriteStream(null, {
-      fd: fs.openSync(ACCESS_LOG_FILE, "a")
+      fd: fs.openSync(ACCESS_LOG_FILE, "a"),
     });
     accessLogStat = fs.fstatSync(accessLogStream.fd);
   }
@@ -146,61 +135,75 @@ export function close(): void {
   logStream.end();
 }
 
-export function flatten(details): {} {
+export function flatten(
+  details: Record<string, unknown>,
+): Record<string, unknown> {
   if (details.sessionContext) {
-    details.deviceId = details.sessionContext.deviceId;
-    details.remoteAddress =
-      details.sessionContext.httpRequest.connection.remoteAddress;
+    const sessionContext = details.sessionContext as SessionContext;
+    details.deviceId = sessionContext.deviceId;
+    details.remoteAddress = getRequestOrigin(
+      sessionContext.httpRequest,
+    ).remoteAddress;
     delete details.sessionContext;
   }
 
   if (details.exception) {
-    details.exceptionName = details.exception.name;
-    details.exceptionMessage = details.exception.message;
-    details.exceptionStack = details.exception.stack;
+    const err = details.exception as Error;
+    details.exceptionName = err.name;
+    details.exceptionMessage = err.message;
+    details.exceptionStack = err.stack;
     delete details.exception;
   }
 
   if (details.task) {
-    details.taskId = details.task._id;
+    details.taskId = details.task["_id"];
     delete details.task;
   }
 
   if (details.rpc) {
-    if (details.rpc.acsRequest) {
-      details.acsRequestId = details.rpc.id;
-      details.acsRequestName = details.rpc.acsRequest.name;
-      if (details.rpc.acsRequest.commandKey)
-        details.acsRequestCommandKey = details.rpc.acsRequest.commandKey;
-    } else if (details.rpc.cpeRequest) {
-      details.cpeRequestId = details.rpc.id;
-      if (details.rpc.cpeRequest.name === "Inform") {
-        details.informEvent = details.rpc.cpeRequest.event.join(",");
-        details.informRetryCount = details.rpc.cpeRequest.retryCount;
+    const rpc = details.rpc as {
+      id: string;
+      acsRequest?: AcsRequest;
+      cpeRequest?: CpeRequest;
+      cpeFault?: CpeFault;
+    };
+    if (rpc.acsRequest) {
+      details.acsRequestId = rpc.id;
+      details.acsRequestName = rpc.acsRequest.name;
+      if (rpc.acsRequest["commandKey"])
+        details.acsRequestCommandKey = rpc.acsRequest["commandKey"];
+    } else if (rpc.cpeRequest) {
+      details.cpeRequestId = rpc.id;
+      if (rpc.cpeRequest.name === "Inform") {
+        details.informEvent = (rpc.cpeRequest as InformRequest).event.join(",");
+        details.informRetryCount = (rpc.cpeRequest as InformRequest).retryCount;
       } else {
-        details.cpeRequestName = details.rpc.cpeRequest.name;
-        if (details.rpc.cpeRequest.commandKey)
-          details.cpeRequestCommandKey = details.rpc.cpeRequest.commandKey;
+        details.cpeRequestName = rpc.cpeRequest.name;
+        if (rpc.cpeRequest["commandKey"])
+          details.cpeRequestCommandKey = rpc.cpeRequest["commandKey"];
       }
-    } else if (details.rpc.cpeFault) {
-      details.acsRequestId = details.rpc.id;
-      details.cpeFaultCode = details.rpc.cpeFault.detail.faultCode;
-      details.cpeFaultString = details.rpc.cpeFault.detail.faultString;
+    } else if (rpc.cpeFault) {
+      details.acsRequestId = rpc.id;
+      details.cpeFaultCode = rpc.cpeFault.detail.faultCode;
+      details.cpeFaultString = rpc.cpeFault.detail.faultString;
     }
     delete details.rpc;
   }
 
   if (details.fault) {
-    details.faultCode = details.fault.code;
-    details.faultMessage = details.fault.message;
+    const fault = details.fault as Fault;
+    details.faultCode = fault.code;
+    details.faultMessage = fault.message;
     delete details.fault;
   }
 
   // For genieacs-ui
   if (details.context) {
-    details.remoteAddress = details.context.request.ip;
-    if (details.context.state.user)
-      details.user = details.context.state.user.username;
+    details.remoteAddress = getRequestOrigin(
+      details.context["req"],
+    ).remoteAddress;
+    if (details.context["state"].user)
+      details.user = details.context["state"].user.username;
     delete details.context;
   }
 
@@ -210,7 +213,10 @@ export function flatten(details): {} {
   return details;
 }
 
-function formatJson(details, systemd): string {
+function formatJson(
+  details: Record<string, unknown>,
+  systemd: boolean,
+): string {
   if (systemd) {
     let severity = "";
     if (details.severity === "info") severity = "<6>";
@@ -223,14 +229,17 @@ function formatJson(details, systemd): string {
   return `${JSON.stringify(flatten(details))}${os.EOL}`;
 }
 
-function formatSimple(details, systemd): string {
+function formatSimple(
+  details: Record<string, unknown>,
+  systemd: boolean,
+): string {
   const skip = {
     user: true,
     remoteAddress: true,
     severity: true,
     timestamp: true,
     message: true,
-    deviceId: !!details.sessionContext
+    deviceId: !!details.sessionContext,
   };
 
   flatten(details);
@@ -263,12 +272,12 @@ function formatSimple(details, systemd): string {
     return `${severity}${remote}${details.message}${meta}${os.EOL}`;
   }
 
-  return `${details.timestamp} [${details.severity.toUpperCase()}] ${remote}${
-    details.message
-  }${meta}${os.EOL}`;
+  return `${details.timestamp} [${(
+    details.severity as string
+  ).toUpperCase()}] ${remote}${details.message}${meta}${os.EOL}`;
 }
 
-function log(details): void {
+function log(details: Record<string, unknown>): void {
   details.timestamp = new Date().toISOString();
   if (LOG_FORMAT === "json") {
     details = Object.assign({}, defaultMeta, details);
@@ -278,22 +287,22 @@ function log(details): void {
   }
 }
 
-export function info(details): void {
+export function info(details: Record<string, unknown>): void {
   details.severity = "info";
   log(details);
 }
 
-export function warn(details): void {
+export function warn(details: Record<string, unknown>): void {
   details.severity = "warn";
   log(details);
 }
 
-export function error(details): void {
+export function error(details: Record<string, unknown>): void {
   details.severity = "error";
   log(details);
 }
 
-export function accessLog(details): void {
+export function accessLog(details: Record<string, unknown>): void {
   details.timestamp = new Date().toISOString();
   if (ACCESS_LOG_FORMAT === "json") {
     Object.assign(details, defaultMeta);
@@ -303,17 +312,17 @@ export function accessLog(details): void {
   }
 }
 
-export function accessInfo(details): void {
+export function accessInfo(details: Record<string, unknown>): void {
   details.severity = "info";
   accessLog(details);
 }
 
-export function accessWarn(details): void {
+export function accessWarn(details: Record<string, unknown>): void {
   details.severity = "warn";
   accessLog(details);
 }
 
-export function accessError(details): void {
+export function accessError(details: Record<string, unknown>): void {
   details.severity = "error";
   accessLog(details);
 }

lib/mongodb-functions.ts

@@ -1,629 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ObjectID } from "mongodb";
-import { map, parse, stringify } from "./common/expression-parser";
-import { likePatternToRegExp, evaluate } from "./common/expression";
-import { Expression, Fault, Task } from "./types";
-
-const isArray = Array.isArray;
-
-export function processDeviceFilter(filter): Expression {
-  return map(filter, exp => {
-    if (!isArray(exp)) return exp;
-
-    if (exp[0] === "PARAM") {
-      const p = exp[1];
-      if (p === "DeviceID.ID") return ["PARAM", "_id"];
-      else if (p === "DeviceID") return ["PARAM", "_deviceId"];
-      else if (p.startsWith("DeviceID."))
-        return ["PARAM", "_deviceId._" + p.slice(9)];
-      else if (p === "Tags") return ["PARAM", "_tags"];
-      else if (p === "tag") return ["PARAM", "_tags"];
-      else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
-      else if (p === "Events.Registered") return ["PARAM", "_registered"];
-      else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
-      else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"];
-    } else if (
-      isArray(exp[1]) &&
-      exp[1][0] === "PARAM" &&
-      exp[1][1].startsWith("Tags.")
-    ) {
-      const t = exp[1][1].slice(5);
-      if (exp[0] === "IS NULL") return ["<>", ["PARAM", "_tags"], t];
-      else if (exp[0] === "IS NOT NULL") return ["=", ["PARAM", "_tags"], t];
-      else if (exp[0] === "=" && exp[2] === true)
-        return ["=", ["PARAM", "_tags"], t];
-    } else if (
-      ["=", "<>", ">", ">=", "<", "<=", "LIKE", "NOT LIKE"].includes(exp[0])
-    ) {
-      let e = map(exp, ee => {
-        if (
-          isArray(ee) &&
-          ee[0] === "PARAM" &&
-          typeof ee[1] === "string" &&
-          !ee[1].startsWith("_")
-        )
-          return ["PARAM", `${ee[1]}._value`];
-        return ee;
-      });
-
-      if (typeof e[2] === "number") {
-        const alt = (e as any[]).slice();
-        alt[2] = new Date(e[2]);
-        e = ["OR", e, alt];
-      }
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function processTasksFilter(filter): Expression {
-  return map(filter, exp => {
-    if (!isArray(exp)) return exp;
-    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
-      const e = exp.slice();
-      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectID(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
-        e[2] = new Date(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
-        e[2] = new Date(e[2]);
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function processFaultsFilter(filter): Expression {
-  return map(filter, exp => {
-    if (!isArray(exp)) return exp;
-    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
-      const e = exp.slice();
-      if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
-        e[2] = new Date(e[2]);
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function filterToMongoQuery(exp: Expression): {} {
-  const ops = {
-    OR: 0,
-    AND: 0,
-    NOT: 0,
-    "=": 1,
-    "<>": 1,
-    ">": 1,
-    ">=": 1,
-    "<": 1,
-    "<=": 1,
-    LIKE: 1,
-    "NOT LIKE": 1,
-    "IS NULL": 1,
-    "IS NOT NULL": 1
-  };
-
-  function recursive(filter, negate, res = {}): {} {
-    const op = filter[0];
-
-    if (ops[op] === 0) {
-      for (let i = 1; i < filter.length; ++i) {
-        if (!isArray(filter[i]) || ops[filter[i][0]] == null)
-          throw new Error(`Invalid expression in ${op} clause`);
-      }
-
-      if ((!negate && op === "AND") || (negate && op === "OR")) {
-        res["$and"] = res["$and"] || [];
-        for (let i = 1; i < filter.length; ++i)
-          res["$and"].push(recursive(filter[i], negate));
-      } else if ((!negate && op === "OR") || (negate && op === "AND")) {
-        res["$or"] = res["$or"] || [];
-
-        for (let i = 1; i < filter.length; ++i)
-          res["$or"].push(recursive(filter[i], negate));
-      } else if (op === "NOT") {
-        recursive(filter[1], !negate, res);
-      }
-    } else if (ops[op] === 1) {
-      if (isArray(filter[2])) {
-        throw new Error(`Right operand of ${op} clause is not a primitive`);
-      } else if (
-        ["LIKE", "NOT LIKE"].includes(op) &&
-        isArray(filter[1]) &&
-        filter[1][0] === "FUNC" &&
-        ["UPPER", "LOWER"].includes(filter[1][1])
-      ) {
-        if (
-          (filter[1][1] === "UPPER" && filter[2] !== filter[2].toUpperCase()) ||
-          (filter[1][1] === "LOWER" && filter[2] !== filter[2].toLowerCase())
-        ) {
-          throw new Error(
-            `Cannot compare ${
-              filter[1][1]
-            }() against non-${filter[1][1].toLowerCase()}case pattern`
-          );
-        }
-      } else if (!isArray(filter[1]) || filter[1][0] !== "PARAM") {
-        throw new Error(`Left operand of ${op} clause is not a parameter`);
-      }
-
-      if (op === "=") {
-        const param = filter[1][1];
-        const p = (res[param] = res[param] || {});
-        if (negate) p["$ne"] = filter[2];
-        else p["$eq"] = filter[2];
-      } else if (op === "<>") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$ne"] = filter[2];
-        if (param !== "_tags") p["$exists"] = true;
-      } else if (op === ">") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gt"] = filter[2];
-      } else if (op === ">=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$gte"] = filter[2];
-      } else if (op === "<") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lt"] = filter[2];
-      } else if (op === "<=") {
-        const param = filter[1][1];
-        let p = (res[param] = res[param] || {});
-        if (negate) p = p["$not"] = p["$not"] || {};
-        p["$lte"] = filter[2];
-      } else if (op === "IS NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: negate };
-      } else if (op === "IS NOT NULL") {
-        const param = filter[1][1];
-        res[param] = { $exists: !negate };
-      } else {
-        if (op === "NOT LIKE") negate = !negate;
-        let param;
-        let flags;
-        if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
-          param = filter[1][2][1];
-          flags = "i";
-        } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
-          param = filter[1][2][1];
-          flags = "i";
-        } else {
-          param = filter[1][1];
-          flags = "";
-        }
-        const r = likePatternToRegExp(filter[2], filter[3], flags);
-        if (negate) res[param] = { $not: r };
-        else res[param] = r;
-      }
-    } else {
-      throw new Error(`Unrecognized operator ${op}`);
-    }
-
-    return res;
-  }
-
-  const _exp = evaluate(exp, null, Date.now());
-
-  if (!isArray(_exp)) {
-    if (_exp === true) return {};
-    throw new Error("Primitives are not valid queries");
-  }
-
-  return recursive(_exp, false);
-}
-
-export function processDeviceProjection(projection: {}): {} {
-  if (!projection) return projection;
-  const p = {};
-  for (const [k, v] of Object.entries(projection)) {
-    if (k === "DeviceID.ID") {
-      p["_id"] = 1;
-    } else if (k.startsWith("DeviceID")) {
-      p["_deviceId._SerialNumber"] = v;
-      p["_deviceId._OUI"] = v;
-      p["_deviceId._ProductClass"] = v;
-      p["_deviceId._Manufacturer"] = v;
-    } else if (k.startsWith("Tags")) {
-      p["_tags"] = v;
-    } else if (k.startsWith("Events")) {
-      p["_lastInform"] = v;
-      p["_registered"] = v;
-      p["_lastBoot"] = v;
-      p["_lastBootstrap"] = v;
-    } else {
-      p[k] = v;
-    }
-  }
-
-  return p;
-}
-
-export function processDeviceSort(sort: {}): {} {
-  if (!sort) return sort;
-  const s = {};
-  for (const [k, v] of Object.entries(sort)) {
-    if (k === "DeviceID.ID") s["_id"] = v;
-    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
-    else if (k === "Events.Inform") s["_lastInform"] = v;
-    else if (k === "Events.Registered") s["_registered"] = v;
-    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
-    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
-    else s[`${k}._value`] = v;
-  }
-
-  return s;
-}
-
-function parseDate(d): number | string {
-  const n = +d;
-  return isNaN(n) ? "" + d : n;
-}
-
-interface FlatDevice {
-  [param: string]: {
-    object: boolean;
-    objectTimestamp: number;
-    writable: boolean;
-    writableTimestamp: number;
-    value: [string | number | boolean, string];
-    valueTimestamp: number;
-  };
-}
-
-export function flattenDevice(device): FlatDevice {
-  function recursive(input, root, output, timestamp): void {
-    for (const [name, tree] of Object.entries(input)) {
-      if (!root) {
-        if (name === "_lastInform") {
-          output["Events.Inform"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp
-          };
-        } else if (name === "_registered") {
-          output["Events.Registered"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp
-          };
-        } else if (name === "_lastBoot") {
-          output["Events.1_BOOT"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp
-          };
-        } else if (name === "_lastBootstrap") {
-          output["Events.0_BOOTSTRAP"] = {
-            value: [parseDate(tree), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp
-          };
-        } else if (name === "_id") {
-          output["DeviceID.ID"] = {
-            value: [tree, "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp
-          };
-        } else if (name === "_deviceId") {
-          output["DeviceID.Manufacturer"] = {
-            value: [tree["_Manufacturer"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp
-          };
-          output["DeviceID.OUI"] = {
-            value: [tree["_OUI"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp
-          };
-          output["DeviceID.ProductClass"] = {
-            value: [tree["_ProductClass"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp
-          };
-          output["DeviceID.SerialNumber"] = {
-            value: [tree["_SerialNumber"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp
-          };
-        } else if (name === "_tags") {
-          output["Tags"] = {
-            writable: true,
-            writableTimestamp: timestamp,
-            object: true,
-            objectTimestamp: timestamp
-          };
-
-          for (const t of tree as string[]) {
-            output[`Tags.${t}`] = {
-              value: [true, "xsd:boolean"],
-              valueTimestamp: timestamp,
-              writable: false,
-              writableTimestamp: timestamp,
-              object: false,
-              objectTimestamp: timestamp
-            };
-          }
-        }
-      }
-
-      if (name.startsWith("_")) continue;
-
-      let childrenTimestamp = timestamp;
-
-      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
-      else if (+input["_timestamp"] > timestamp)
-        childrenTimestamp = +input["_timestamp"];
-
-      const attrs = {};
-      if (tree["_value"] != null) {
-        attrs["value"] = [
-          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
-          tree["_type"]
-        ];
-        attrs["valueTimestamp"] = +(tree["_timestamp"] || childrenTimestamp);
-        attrs["object"] = false;
-        attrs["objectTimestamp"] = childrenTimestamp;
-      } else if (tree["_object"] != null) {
-        attrs["object"] = tree["_object"];
-        attrs["objectTimestamp"] = childrenTimestamp;
-      }
-
-      if (tree["_writable"] != null) {
-        attrs["writable"] = tree["_writable"];
-        attrs["writableTimestamp"] = childrenTimestamp;
-      }
-
-      const r = root ? `${root}.${name}` : name;
-      output[r] = attrs;
-
-      if (attrs["object"] || tree["object"] == null)
-        recursive(tree, r, output, childrenTimestamp);
-    }
-  }
-
-  const newDevice = {};
-  const timestamp = new Date(device["_lastInform"] || 1).getTime();
-  recursive(device, "", newDevice, timestamp);
-  return newDevice;
-}
-
-export function flattenFault(fault): Fault {
-  const f = Object.assign({}, fault);
-  if (f.timestamp) f.timestamp = +f.timestamp;
-  if (f.expiry) f.expiry = +f.expiry;
-  return f;
-}
-
-export function flattenTask(task): Task {
-  const t = Object.assign({}, task);
-  t._id = "" + t._id;
-  if (t.timestamp) t.timestamp = +t.timestamp;
-  if (t.expiry) t.expiry = +t.expiry;
-  return t;
-}
-
-export function mongoQueryToFilter(query): Expression {
-  function recursive(_query): Expression {
-    const expressions: Expression[] = [];
-    for (const [k, v] of Object.entries(_query)) {
-      if (k[0] === "$") {
-        if (k === "$and") {
-          const and: Expression = ["AND"];
-          for (const vv of Object.values(v)) and.push(recursive(vv));
-          expressions.push(and);
-        } else if (k === "$or") {
-          const or: Expression = ["OR"];
-          for (const vv of Object.values(v)) or.push(recursive(vv));
-          expressions.push(or);
-        } else {
-          throw new Error(`Operator ${k} not supported`);
-        }
-      } else if (k === "_tags") {
-        if (typeof v === "object") {
-          if (isArray(v)) throw new Error(`Invalid type`);
-
-          if (v.hasOwnProperty("$ne"))
-            expressions.push(["IS NULL", ["PARAM", `Tags.${v["$ne"]}`]]);
-          else if (v.hasOwnProperty("$eq"))
-            expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v["$eq"]}`]]);
-          else throw new Error(`Invalid tag query`);
-        } else {
-          expressions.push(["IS NOT NULL", ["PARAM", `Tags.${v}`]]);
-        }
-      } else if (k.startsWith("Tags.")) {
-        let exists: boolean;
-        if (typeof v === "boolean") exists = v;
-        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
-        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
-        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
-        else throw new Error(`Invalid tag query`);
-
-        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
-      } else if (typeof v === "object") {
-        if (isArray(v)) throw new Error(`Invalid type`);
-
-        const exps: Expression[] = [];
-        for (const [kk, vv] of Object.entries(v)) {
-          let op;
-          switch (kk) {
-            case "$eq":
-              op = "=";
-              break;
-            case "$ne":
-              op = "<>";
-              break;
-            case "$lt":
-              op = "<";
-              break;
-            case "$lte":
-              op = "<=";
-              break;
-            case "$gt":
-              op = ">";
-              break;
-            case "$gte":
-              op = ">=";
-              break;
-            default:
-              throw new Error(`Operator ${kk} not supported`);
-          }
-          exps.push([op, ["PARAM", k], vv]);
-        }
-        if (exps.length === 1) return exps[0];
-        const and: Expression = ["AND"];
-        expressions.push(and.concat(exps));
-      } else {
-        expressions.push(["=", ["PARAM", k], v]);
-      }
-    }
-    if (expressions.length === 1) return expressions[0];
-    const and: Expression = ["AND"];
-    return and.concat(expressions);
-  }
-
-  // empty filter
-  if (!Object.keys(query).length) return true;
-
-  return recursive(query);
-}
-
-export function flattenPreset(preset): {} {
-  const p = Object.assign({}, preset);
-  if (p.precondition) {
-    try {
-      // Try parse to check expression validity
-      parse(p.precondition);
-    } catch (error) {
-      p.precondition = mongoQueryToFilter(JSON.parse(p.precondition));
-      p.precondition = p.precondition.length ? stringify(p.precondition) : "";
-    }
-  }
-
-  if (p.events) {
-    const e = [];
-    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
-    p.events = e.join(", ");
-  }
-
-  const provision = p.configurations[0];
-  if (
-    p.configurations.length === 1 &&
-    provision.type === "provision" &&
-    provision.name &&
-    provision.name.length
-  ) {
-    p.provision = provision.name;
-    p.provisionArgs = provision.args
-      ? JSON.stringify(provision.args).slice(1, -1)
-      : "";
-  }
-
-  delete p.configurations;
-  return p;
-}
-
-export function flattenFile(file): {} {
-  const f = {};
-  f["_id"] = file["_id"];
-  if (file.metadata) {
-    f["metadata.fileType"] = file.metadata.fileType || "";
-    f["metadata.oui"] = file.metadata.oui || "";
-    f["metadata.productClass"] = file.metadata.productClass || "";
-    f["metadata.version"] = file.metadata.version || "";
-  }
-  return f;
-}
-
-export function preProcessPreset(data): {} {
-  const preset = Object.assign({}, data);
-
-  if (!preset.precondition) preset.precondition = "";
-  // Try parse to check expression validity
-  parse(preset.precondition);
-
-  preset.weight = parseInt(preset.weight) || 0;
-
-  const events = {};
-  if (preset.events) {
-    for (let e of preset.events.split(",")) {
-      let v = true;
-      e = e.trim();
-      if (e.startsWith("-")) {
-        v = false;
-        e = e.slice(1).trim();
-      }
-      if (e) events[e] = v;
-    }
-  }
-
-  preset.events = events;
-
-  if (!preset.provision) throw new Error("Invalid preset provision");
-
-  const configuration = {
-    type: "provision",
-    name: preset.provision,
-    args: null
-  };
-
-  if (preset.provisionArgs)
-    configuration.args = JSON.parse(`[${preset.provisionArgs}]`);
-
-  delete preset.provision;
-  delete preset.provisionArgs;
-  preset.configurations = [configuration];
-  return preset;
-}

lib/ping.ts

@@ -0,0 +1,98 @@
+import { platform } from "node:os";
+import { exec } from "node:child_process";
+import { domainToASCII } from "node:url";
+
+export interface PingResult {
+  packetsTransmitted: number;
+  packetsReceived: number;
+  packetLoss: number;
+  min: number;
+  avg: number;
+  max: number;
+  mdev: number;
+}
+
+function isValidHost(host: string): boolean {
+  // Valid chars in IPv4, IPv6, domain names
+  if (/^[a-zA-Z0-9\-.:[\]-]+$/.test(host)) return true;
+
+  // Check if input is an IDN convert to Punycode
+  // Can't merge with above because domainToASCII doesn't accept IP addresses
+  return /^[a-zA-Z0-9\-.:[\]-]+$/.test(domainToASCII(host));
+}
+
+export function parsePing(osPlatform: string, stdout: string): PingResult {
+  let parseRegExp1: RegExp, parseRegExp2: RegExp, parsed: PingResult;
+  switch (osPlatform) {
+    case "linux":
+      parseRegExp1 =
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss[^]*= ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+      parseRegExp2 =
+        /(\d+) packets transmitted, (\d+) .*received, ([\d.]+)% .*loss/;
+      break;
+
+    case "freebsd":
+      parseRegExp1 =
+        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
+      parseRegExp2 =
+        /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
+      break;
+  }
+
+  const m1 = stdout.match(parseRegExp1);
+  if (m1) {
+    parsed = {
+      packetsTransmitted: +m1[1],
+      packetsReceived: +m1[2],
+      packetLoss: +m1[3],
+      min: +m1[4],
+      avg: +m1[5],
+      max: +m1[6],
+      mdev: +m1[7],
+    };
+  } else {
+    const m2 = stdout.match(parseRegExp2);
+    if (m2) {
+      parsed = {
+        packetsTransmitted: +m2[1],
+        packetsReceived: +m2[2],
+        packetLoss: +m2[3],
+        min: null,
+        avg: null,
+        max: null,
+        mdev: null,
+      };
+    }
+  }
+  return parsed;
+}
+
+export function ping(
+  host: string,
+  callback: (err: Error, res?: PingResult, stdout?: string) => void,
+): void {
+  // Validate input to prevent possible remote code execution
+  // Credit to Alex Hordijk for reporting this vulnerability
+  if (!isValidHost(host)) return callback(new Error("Invalid host"));
+  host = host.replace("[", "").replace("]", "");
+  let cmd: string;
+
+  switch (platform()) {
+    case "linux":
+      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
+      break;
+    case "freebsd":
+      // Send a single packet because on FreeBSD only superuser can send
+      // packets that are only 200 ms apart.
+      cmd = `ping -t 1 -c 3 ${host}`;
+      break;
+    default:
+      return callback(new Error("Platform not supported"));
+  }
+
+  exec(cmd, (err, stdout) => {
+    if (err) return callback(err);
+    const parsed: PingResult = parsePing(platform(), stdout);
+    return callback(err, parsed, stdout);
+  });
+}

lib/query.ts

@@ -1,23 +1,6 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as common from "./common";
+function isObject(obj: any): boolean {
+  return Object.prototype.toString.call(obj) === "[object Object]";
+}
 
 function stringToRegexp(input, flags?): RegExp | false {
   if (input.indexOf("*") === -1) return false;
@@ -34,8 +17,8 @@ function stringToRegexp(input, flags?): RegExp | false {
 }
 
 function normalize(input): any {
-  if (common.typeOf(input) === common.STRING_TYPE) {
-    const vals = [input];
+  if (typeof input === "string") {
+    const vals: any = [input];
     const m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input);
     if (m) vals.push({ $regex: new RegExp(m[1], m[2]) });
 
@@ -52,14 +35,25 @@ function normalize(input): any {
   return input;
 }
 
-export function expandValue(value): any[] {
-  if (common.typeOf(value) === common.ARRAY_TYPE) {
+const EXPAND_OPS = new Set([
+  "$eq",
+  "$gt",
+  "$gte",
+  "$in",
+  "$lt",
+  "$lte",
+  "$ne",
+  "$nin",
+]);
+
+function expandValue(value: unknown): unknown[] {
+  if (Array.isArray(value)) {
     let a = [];
     for (const j of value) a = a.concat(expandValue(j));
     return [a];
-  } else if (common.typeOf(value) !== common.OBJECT_TYPE) {
+  } else if (!isObject(value)) {
     const n = normalize(value);
-    if (common.typeOf(n) !== common.ARRAY_TYPE) return [n];
+    if (!Array.isArray(n)) return [n];
     else return n;
   }
 
@@ -69,7 +63,8 @@ export function expandValue(value): any[] {
   const values = [];
   for (const [k, v] of Object.entries(value)) {
     keys.push(k);
-    values.push(expandValue(v));
+    if (EXPAND_OPS.has(k)) values.push(expandValue(v));
+    else values.push([v]);
     indices.push(0);
   }
 
@@ -103,12 +98,14 @@ function permute(param, val): any[] {
   return conditions;
 }
 
-export function expand(query): {} {
+export function expand(
+  query: Record<string, unknown>,
+): Record<string, unknown> {
   const newQuery = {};
   for (const [k, v] of Object.entries(query)) {
     if (k[0] === "$") {
       // Operator
-      newQuery[k] = (v as any[]).map(e => expand(e));
+      newQuery[k] = (v as any[]).map((e) => expand(e));
     } else {
       const conditions = permute(k, v);
       if (conditions.length > 1) {
@@ -116,7 +113,7 @@ export function expand(query): {} {
         if (v && (v["$ne"] != null || v["$not"] != null)) {
           if (Object.keys(v).length > 1)
             throw new Error("Cannot mix $ne or $not with other operators");
-          for (const c of conditions) newQuery["$an"].push(c);
+          for (const c of conditions) newQuery["$and"].push(c);
         } else {
           newQuery["$and"].push({ $or: conditions });
         }
@@ -129,100 +126,16 @@ export function expand(query): {} {
   return newQuery;
 }
 
-function testExpressions(params, expressions, lop): boolean {
-  for (const f of expressions) {
-    const res = test(params, f);
-    switch (lop) {
-      case "$and":
-        if (!res) return false;
-        break;
-      case "$or":
-        if (res) return true;
-        break;
-      case "$nor":
-        if (res) return false;
-    }
-  }
-
-  switch (lop) {
-    case "$and":
-      return true;
-    case "$or":
-      return false;
-    case "$nor":
-      return true;
-    default:
-      throw new Error("Unknown logical operator");
-  }
-}
-
-export function test(params, query): boolean {
-  let res;
-  for (const [k, v] of Object.entries(query)) {
-    if (k.charAt(0) === "$") {
-      // Logical operator
-      res = testExpressions(params, v, k);
-    } else {
-      const value = params[k];
-      if (common.typeOf(v) !== common.OBJECT_TYPE) {
-        if (common.typeOf(value) === common.ARRAY_TYPE) {
-          // TODO comparing array to regex, array to array, and object to object
-          res = value.indexOf(v) !== -1;
-        } else {
-          if (common.typeOf(v) === common.REGEXP_TYPE)
-            res = (v as RegExp).test(value);
-          else res = v === value;
-        }
-      } else {
-        for (const [k2, v2] of Object.entries(v)) {
-          switch (k2) {
-            case "$ne":
-              if (common.typeOf(value) === common.ARRAY_TYPE)
-                res = value.indexOf(v2) === -1;
-              else res = value !== v2;
-              break;
-            case "$lt":
-              res = value < v2;
-              break;
-            case "$lte":
-              res = value <= v2;
-              break;
-            case "$gt":
-              res = value > v2;
-              break;
-            case "$gte":
-              res = value >= v2;
-              break;
-            case "$regex":
-              res = v2.test(value);
-              break;
-            case "$in":
-              throw new Error("Operator not supported");
-            case "$nin":
-              throw new Error("Operator not supported");
-            case "$all":
-              throw new Error("Operator not supported");
-            case "$exists":
-              throw new Error("Operator not supported");
-            default:
-              throw new Error("Operator not supported");
-          }
-        }
-      }
-    }
-
-    if (!res) return false;
-  }
-  return true;
-}
-
-export function sanitizeQueryTypes(query, types): {} {
+export function sanitizeQueryTypes(
+  query: Record<string, unknown>,
+  types: Record<string, (v: unknown) => unknown>,
+): Record<string, unknown> {
   for (const [k, v] of Object.entries(query)) {
     if (k[0] === "$") {
       // Logical operator
       for (const vv of v as any[]) sanitizeQueryTypes(vv, types);
     } else if (k in types) {
-      if (common.typeOf(v) === common.OBJECT_TYPE) {
+      if (isObject(v)) {
         for (const [kk, vv] of Object.entries(v)) {
           switch (kk) {
             case "$in":

lib/sandbox.ts

@@ -1,30 +1,11 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as vm from "vm";
+import * as vm from "node:vm";
 import seedrandom from "seedrandom";
-import * as device from "./device";
-import * as extensions from "./extensions";
-import * as logger from "./logger";
-import * as scheduling from "./scheduling";
-import Path from "./common/path";
-import { Fault, SessionContext, ScriptResult } from "./types";
+import * as device from "./device.ts";
+import * as extensions from "./extensions.ts";
+import * as logger from "./logger.ts";
+import * as scheduling from "./scheduling.ts";
+import Path from "./common/path.ts";
+import { Fault, SessionContext, ScriptResult } from "./types.ts";
 
 // Used for throwing to exit user script and commit
 const COMMIT = Symbol();
@@ -34,7 +15,7 @@ const EXT = Symbol();
 
 const UNDEFINED = undefined;
 
-const context = vm.createContext();
+const context = vm.createContext(undefined, { microtaskMode: "afterEvaluate" });
 
 let state;
 
@@ -50,7 +31,7 @@ function runExtension(sessionContext, key, extCall): Promise<Fault> {
   }
 
   let prom = re.get(key);
-  if (!prom) {
+  if (prom == null) {
     re.set(
       key,
       (prom = new Promise((resolve, reject) => {
@@ -62,7 +43,7 @@ function runExtension(sessionContext, key, extCall): Promise<Fault> {
             resolve(fault);
           })
           .catch(reject);
-      }))
+      })),
     );
   }
 
@@ -78,7 +59,7 @@ class SandboxDate {
       number?,
       number?,
       number?,
-      number?
+      number?,
     ]
   ) {
     if (argumentList.length) return new Date(...argumentList);
@@ -109,6 +90,16 @@ class SandboxDate {
 
     return t;
   }
+
+  public static parse(dateString: string): number {
+    return Date.parse(dateString);
+  }
+
+  public static UTC(
+    ...args: [number, number?, number?, number?, number?, number?, number?]
+  ): number {
+    return Date.UTC(...args);
+  }
 }
 
 function random(): number {
@@ -117,7 +108,7 @@ function random(): number {
   return state.rng();
 }
 
-random.seed = function(s) {
+random.seed = function (s) {
   state.rng = seedrandom(s);
 };
 
@@ -125,7 +116,7 @@ class ParameterWrapper {
   public constructor(path: Path, attributes, unpacked?, unpackedRevision?) {
     for (const attrName of attributes) {
       Object.defineProperty(this, attrName, {
-        get: function() {
+        get: function () {
           if (state.uncommitted) commit();
 
           if (state.revision !== unpackedRevision) {
@@ -133,7 +124,7 @@ class ParameterWrapper {
             unpacked = device.unpack(
               state.sessionContext.deviceData,
               path,
-              state.revision
+              state.revision,
             );
           }
 
@@ -141,18 +132,18 @@ class ParameterWrapper {
 
           const attr = state.sessionContext.deviceData.attributes.get(
             unpacked[0],
-            state.revision
+            state.revision,
           )[attrName];
 
           if (!attr) return UNDEFINED;
 
           return attr[1];
-        }
+        },
       });
     }
 
     Object.defineProperty(this, "path", {
-      get: function() {
+      get: function () {
         if (state.uncommitted) commit();
 
         if (state.revision !== unpackedRevision) {
@@ -160,18 +151,18 @@ class ParameterWrapper {
           unpacked = device.unpack(
             state.sessionContext.deviceData,
             path,
-            state.revision
+            state.revision,
           );
         }
 
         if (!unpacked.length) return UNDEFINED;
 
         return unpacked[0].toString();
-      }
+      },
     });
 
     Object.defineProperty(this, "size", {
-      get: function() {
+      get: function () {
         if (state.uncommitted) commit();
 
         if (state.revision !== unpackedRevision) {
@@ -179,17 +170,17 @@ class ParameterWrapper {
           unpacked = device.unpack(
             state.sessionContext.deviceData,
             path,
-            state.revision
+            state.revision,
           );
         }
 
         if (!unpacked.length) return UNDEFINED;
 
         return unpacked.length;
-      }
+      },
     });
 
-    this[Symbol.iterator] = function*() {
+    this[Symbol.iterator] = function* () {
       if (state.uncommitted) commit();
 
       if (state.revision !== unpackedRevision) {
@@ -197,7 +188,7 @@ class ParameterWrapper {
         unpacked = device.unpack(
           state.sessionContext.deviceData,
           path,
-          state.revision
+          state.revision,
         );
       }
 
@@ -210,7 +201,7 @@ class ParameterWrapper {
 function declare(
   path: string,
   timestamps: { [attr: string]: number },
-  values: { [attr: string]: any }
+  values: { [attr: string]: any },
 ): ParameterWrapper {
   state.uncommitted = true;
   if (!timestamps) timestamps = {};
@@ -225,7 +216,7 @@ function declare(
     pathSet: null,
     attrGet: null,
     attrSet: null,
-    defer: true
+    defer: true,
   };
 
   const attrs = new Set();
@@ -245,6 +236,7 @@ function declare(
   }
 
   for (const [attrName, attrTimestamp] of Object.entries(timestamps)) {
+    if (!(attrTimestamp >= 1)) continue;
     if (attrName === "path") {
       declaration.pathGet = attrTimestamp;
     } else {
@@ -275,14 +267,14 @@ function commit(): void {
     throw COMMIT;
   } else if (state.revision > state.maxRevision + 1) {
     throw new Error(
-      "Declare function should not be called from within a try/catch block"
+      "Declare function should not be called from within a try/catch block",
     );
   }
 }
 
-function ext(): any {
+function ext(...args: unknown[]): any {
   ++state.extCounter;
-  const extCall = Array.from(arguments).map(String);
+  const extCall = args.map(String);
   const key = `${state.revision}: ${JSON.stringify(extCall)}`;
 
   if (key in state.sessionContext.extensionsCache)
@@ -292,11 +284,11 @@ function ext(): any {
   throw EXT;
 }
 
-function log(msg: string, meta: {}): void {
+function log(msg: string, meta: Record<string, unknown>): void {
   if (state.revision === state.maxRevision && state.extCounter >= 0) {
     const details = Object.assign({}, meta, {
       sessionContext: state.sessionContext,
-      message: `Script: ${msg}`
+      message: `Script: ${msg}`,
     });
 
     delete details["hostname"];
@@ -332,20 +324,20 @@ function errorToFault(err: Error): Fault {
     message: err.message,
     detail: {
       name: err.name,
-      message: err.message
-    }
+      message: err.message,
+    },
   };
 
   if (err.stack) {
     fault.detail["stack"] = err.stack;
     // Trim the stack trace at the self-executing anonymous wrapper function
     const stackTrimIndex = fault.detail["stack"].match(
-      /\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/
+      /\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/,
     );
     if (stackTrimIndex) {
       fault.detail["stack"] = fault.detail["stack"].slice(
         0,
-        stackTrimIndex.index
+        stackTrimIndex.index,
       );
     }
   }
@@ -355,11 +347,11 @@ function errorToFault(err: Error): Fault {
 
 export async function run(
   script: vm.Script,
-  globals: {},
+  globals: Record<string, unknown>,
   sessionContext: SessionContext,
   startRevision: number,
   maxRevision: number,
-  extCounter = 0
+  extCounter = 0,
 ): Promise<ScriptResult> {
   state = {
     sessionContext: sessionContext,
@@ -370,7 +362,7 @@ export async function run(
     extensions: {},
     clear: [],
     rng: null,
-    extCounter: extCounter
+    extCounter: extCounter,
   };
 
   for (const n of Object.keys(context)) delete context[n];
@@ -380,7 +372,7 @@ export async function run(
   let ret, status;
 
   try {
-    ret = script.runInContext(context, { displayErrors: false });
+    ret = script.runInContext(context, { displayErrors: false, timeout: 50 });
     status = 0;
   } catch (err) {
     if (err === COMMIT) {
@@ -393,7 +385,7 @@ export async function run(
         clear: null,
         declare: null,
         done: false,
-        returnValue: null
+        returnValue: null,
       };
     }
   }
@@ -404,7 +396,7 @@ export async function run(
   await Promise.all(
     Object.entries(_state.extensions).map(async ([k, v]) => {
       fault = (await runExtension(_state.sessionContext, k, v)) || fault;
-    })
+    }),
   );
 
   if (fault) {
@@ -413,7 +405,7 @@ export async function run(
       clear: null,
       declare: null,
       done: false,
-      returnValue: null
+      returnValue: null,
     };
   }
 
@@ -424,7 +416,7 @@ export async function run(
       sessionContext,
       startRevision,
       maxRevision,
-      extCounter - _state.extCounter
+      extCounter - _state.extCounter,
     );
   }
 
@@ -433,6 +425,6 @@ export async function run(
     clear: _state.clear,
     declare: _state.declarations,
     done: status === 0,
-    returnValue: ret
+    returnValue: ret,
   };
 }

lib/scheduling.ts

@@ -1,30 +1,8 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as crypto from "crypto";
-import * as later from "later";
+import * as crypto from "node:crypto";
+import * as later from "@breejs/later";
 
 function md532(str): number {
-  const digest = crypto
-    .createHash("md5")
-    .update(str)
-    .digest();
+  const digest = crypto.createHash("md5").update(str).digest();
   return (
     digest.readUInt32LE(0) ^
     digest.readUInt32LE(4) ^
@@ -33,29 +11,37 @@ function md532(str): number {
   );
 }
 
-export function variance(deviceId, vrnc): number {
+export function variance(deviceId: string, vrnc: number): number {
   return (md532(deviceId) >>> 0) % vrnc;
 }
 
-export function interval(timestamp, intrvl, offset = 0): number {
+export function interval(
+  timestamp: number,
+  intrvl: number,
+  offset = 0,
+): number {
   return Math.trunc((timestamp + offset) / intrvl) * intrvl - offset;
 }
 
-export function parseCron(cronExp): any {
+export function parseCron(cronExp: string): any {
   const parts = cronExp.trim().split(/\s+/);
   if (parts.length === 5) parts.unshift("*");
 
   return later.schedule(later.parse.cron(parts.join(" "), true));
 }
 
-export function cron(timestamp, schedule, offset = 0): number[] {
+export function cron(
+  timestamp: number,
+  schedule: unknown,
+  offset = 0,
+): number[] {
   // TODO later.js doesn't throw erorr if expression is invalid!
   const ret = [0, 0];
 
-  const prev = schedule.prev(1, new Date(timestamp + offset));
+  const prev = (schedule as any).prev(1, new Date(timestamp + offset));
   if (prev) ret[0] = prev.setMilliseconds(0) - offset;
 
-  const next = schedule.next(1, new Date(timestamp + offset + 1000));
+  const next = (schedule as any).next(1, new Date(timestamp + offset + 1000));
   if (next) ret[1] = next.setMilliseconds(0) - offset;
 
   return ret;

lib/server.ts

@@ -1,30 +1,13 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as fs from "fs";
-import * as http from "http";
-import * as https from "https";
-import * as path from "path";
-import { ROOT_DIR } from "./config";
+import { readFileSync } from "node:fs";
+import * as http from "node:http";
+import * as https from "node:https";
+import { Socket } from "node:net";
+import * as path from "node:path";
+import { ROOT_DIR } from "./config.ts";
 
 let server: http.Server | https.Server;
-let listener: (...args) => void;
+let listener: http.RequestListener;
+let stopping = false;
 
 function closeServer(timeout, callback): void {
   if (!server) return void callback();
@@ -46,42 +29,123 @@ function closeServer(timeout, callback): void {
 
     const cb = callback;
     callback = null;
-    cb();
+    // Allow some time for connection close events to fire
+    setTimeout(cb, 50);
+  });
+}
+
+interface ServerOptions {
+  port?: number;
+  host?: string;
+  ssl?: { key: string; cert: string };
+  timeout?: number;
+  keepAliveTimeout?: number;
+  requestTimeout?: number;
+  onConnection?: (socket: Socket) => void;
+  onClientError?: (err: Error, socket: Socket) => void;
+}
+
+interface SocketEndpoint {
+  localAddress: string;
+  localPort: number;
+  remoteAddress: string;
+  remotePort: number;
+  remoteFamily: "IPv4" | "IPv6";
+}
+
+// Save this info as they're not accessible after a socket has been closed
+const socketEndpoints: WeakMap<Socket, SocketEndpoint> = new WeakMap();
+
+type Promisify<T extends (...args: any) => any> = (
+  ...args: Parameters<T>
+) => Promise<ReturnType<T>>;
+
+function getValidPrivKeys(value: string): Buffer[] {
+  return value.split(":").map((str) => {
+    str = str.trim();
+    const buf = str.startsWith("-----BEGIN ")
+      ? Buffer.from(str)
+      : readFileSync(path.resolve(ROOT_DIR, str));
+    return buf;
+  });
+}
+
+function getValidCerts(value: string): Buffer[] {
+  return value.split(":").map((str) => {
+    str = str.trim();
+    const buf = str.startsWith("-----BEGIN ")
+      ? Buffer.from(str)
+      : readFileSync(path.resolve(ROOT_DIR, str));
+    return buf;
   });
 }
 
 export function start(
-  port,
-  networkInterface,
-  ssl,
-  _listener,
-  onConnection?,
-  keepAliveTimeout: number = 0
+  options: ServerOptions,
+  _listener: Promisify<http.RequestListener>,
 ): void {
-  listener = _listener;
+  listener = (req, res) => {
+    if (stopping) res.setHeader("Connection", "close");
+    _listener(req, res).catch((err) => {
+      try {
+        res.socket.unref();
+        if (res.headersSent) {
+          res.writeHead(500, { Connection: "close" });
+          res.end(`${err.name}: ${err.message}`);
+        }
+      } catch {
+        // Ignore
+      }
+      throw err;
+    });
+  };
 
-  if (ssl && ssl.key && ssl.cert) {
-    const options = {
-      key: ssl.key
-        .split(":")
-        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
-      cert: ssl.cert
-        .split(":")
-        .map(f => fs.readFileSync(path.resolve(ROOT_DIR, f.trim())))
+  if (options.ssl) {
+    const opts = {
+      key: getValidPrivKeys(options.ssl.key),
+      cert: getValidCerts(options.ssl.cert),
     };
 
-    server = https.createServer(options, listener);
-    if (onConnection != null) server.on("secureConnection", onConnection);
+    server = https.createServer(opts, listener);
+    if (options.onConnection)
+      server.on("secureConnection", options.onConnection);
   } else {
     server = http.createServer(listener);
-    if (onConnection != null) server.on("connection", onConnection);
+    if (options.onConnection) server.on("connection", options.onConnection);
   }
 
-  if (keepAliveTimeout) server.keepAliveTimeout = keepAliveTimeout;
-  server.listen(port, networkInterface);
+  server.on("connection", (socket: Socket) => {
+    socketEndpoints.set(socket, {
+      localAddress: socket.localAddress,
+      localPort: socket.localPort,
+      remoteAddress: socket.remoteAddress,
+      remotePort: socket.remotePort,
+      remoteFamily: socket.remoteFamily as "IPv4" | "IPv6",
+    });
+  });
+
+  if (options.onClientError) {
+    server.on("clientError", (err, socket: Socket) => {
+      if (err["code"] !== "ECONNRESET" && socket.writable)
+        socket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n");
+
+      // As per Node docs: This event is guaranteed to be passed an instance
+      // of the <net.Socket> class
+      options.onClientError(err, socket as Socket);
+    });
+  }
+
+  server.timeout = options.timeout || 0;
+  if (options.keepAliveTimeout != null)
+    server.keepAliveTimeout = options.keepAliveTimeout;
+  if (options.requestTimeout != null)
+    server.requestTimeout = options.requestTimeout;
+
+  server.listen({ port: options.port, host: options.host });
 }
 
-export function stop(): Promise<void> {
+export function stop(terminateConnections = true): Promise<void> {
+  stopping = terminateConnections;
   return new Promise((resolve, reject) => {
     setTimeout(() => {
       reject(new Error("Could not close server in a timely manner"));
@@ -89,3 +153,8 @@ export function stop(): Promise<void> {
     closeServer(20000, resolve);
   });
 }
+
+export function getSocketEndpoints(socket: Socket): SocketEndpoint {
+  // TLSSocket keeps a reference to the raw TCP socket in _parent
+  return socketEndpoints.get(socket["_parent"] ?? socket);
+}

lib/soap-libxmljs.ts

@@ -1,745 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { parseXml, Document, Element, ParserOptions } from "libxmljs";
-import * as config from "./config";
-import { version as VERSION } from "../package.json";
-import {
-  CpeGetResponse,
-  CpeSetResponse,
-  InformRequest,
-  AcsResponse,
-  CpeRequest,
-  FaultStruct,
-  SpvFault,
-  CpeFault,
-  SoapMessage,
-  TransferCompleteRequest
-} from "./types";
-
-const SERVER_NAME = `GenieACS/${VERSION}`;
-
-const NAMESPACES = {
-  "1.0": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-0"
-  },
-  "1.1": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-1"
-  },
-  "1.2": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
-  },
-  "1.3": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
-  },
-  "1.4": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    xsd: "http://www.w3.org/2001/XMLSchema",
-    xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-3"
-  }
-};
-
-const LIBXMLJS_PARSE_OPTIONS: ParserOptions = {
-  nocdata: true,
-  recover: !!config.get("XML_RECOVER"),
-  /* eslint-disable-next-line @typescript-eslint/camelcase */
-  ignore_enc: !!config.get("XML_IGNORE_ENC")
-};
-
-interface WriterOptions {
-  declaration?: boolean;
-  format?: boolean;
-  selfCloseEmpty?: boolean;
-  whitespace?: boolean;
-  type?: "XML" | "HTML" | "XHTML";
-}
-
-declare module "libxmljs" {
-  interface Document {
-    toString(formatting: boolean | WriterOptions): string;
-  }
-}
-
-const LIBXMLJS_SAVE_OPTIONS: WriterOptions = {
-  format: !!config.get("XML_FORMAT"),
-  declaration: !config.get("XML_NO_DECL"),
-  selfCloseEmpty: !config.get("XML_NO_EMPTY")
-};
-
-let warnings;
-
-function parseBool(v): boolean {
-  v = "" + v;
-  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
-  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
-    return false;
-  else return null;
-}
-
-function event(xml): string[] {
-  return xml
-    .childNodes()
-    .filter(n => n.name() === "EventStruct")
-    .map(c =>
-      c
-        .childNodes()
-        .find(n => n.name() === "EventCode")
-        .text()
-        .trim()
-    );
-}
-
-function parameterInfoList(xml): [string, boolean][] {
-  return xml
-    .childNodes()
-    .filter(e => e.name() === "ParameterInfoStruct")
-    .map(e => {
-      let param, value;
-      for (const c of e.childNodes()) {
-        switch (c.name()) {
-          case "Name":
-            param = c.text();
-            break;
-          case "Writable":
-            value = c.text();
-            break;
-        }
-      }
-
-      let parsed = parseBool(value);
-
-      if (parsed === null) {
-        warnings.push({
-          message: "Invalid writable attribute",
-          parameter: param
-        });
-        parsed = false;
-      }
-
-      return [param, parsed];
-    });
-}
-
-function parameterValueList(
-  xml
-): [string, string | number | boolean, string][] {
-  return xml
-    .childNodes()
-    .filter(e => e.name() === "ParameterValueStruct")
-    .map(e => {
-      let valueElement, param;
-      for (const c of e.childNodes()) {
-        switch (c.name()) {
-          case "Name":
-            param = c.text();
-            break;
-          case "Value":
-            valueElement = c;
-            break;
-        }
-      }
-
-      const valueType = valueElement.attr("type").value();
-      const value = valueElement.text();
-      let parsed = value;
-      if (valueType === "xsd:boolean") {
-        parsed = parseBool(value);
-        if (parsed === null) {
-          warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
-          });
-          parsed = value;
-        }
-      } else if (valueType === "xsd:int" || valueType === "xsd:unsignedInt") {
-        parsed = parseInt(value);
-        if (isNaN(parsed)) {
-          warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
-          });
-          parsed = value;
-        }
-      } else if (valueType === "xsd:dateTime") {
-        parsed = Date.parse(value);
-        if (isNaN(parsed)) {
-          warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
-          });
-          parsed = value;
-        }
-      }
-
-      return [param, parsed, valueType];
-    });
-}
-
-function GetParameterNames(xml, methodRequest): void {
-  const el = xml.node("cwmp:GetParameterNames");
-  el.node("ParameterPath").text(methodRequest.parameterPath);
-  el.node("NextLevel").text(+methodRequest.nextLevel);
-}
-
-function GetParameterNamesResponse(xml): CpeGetResponse {
-  return {
-    name: "GetParameterNamesResponse",
-    parameterList: parameterInfoList(
-      xml.childNodes().find(n => n.name() === "ParameterList")
-    )
-  };
-}
-
-function GetParameterValues(xml, methodRequest): void {
-  const el = xml.node("cwmp:GetParameterValues").node("ParameterNames");
-  el.attr({
-    "soap-enc:arrayType": `xsd:string[${methodRequest.parameterNames.length}]`
-  });
-
-  for (const p of methodRequest.parameterNames) el.node("string").text(p);
-}
-
-function GetParameterValuesResponse(xml): CpeGetResponse {
-  return {
-    name: "GetParameterValuesResponse",
-    parameterList: parameterValueList(
-      xml.childNodes().find(n => n.name() === "ParameterList")
-    )
-  };
-}
-
-function SetParameterValues(xml, methodRequest): void {
-  const el = xml.node("cwmp:SetParameterValues");
-  const paramList = el.node("ParameterList");
-  paramList.attr({
-    "soap-enc:arrayType": `cwmp:ParameterValueStruct[${methodRequest.parameterList.length}]`
-  });
-
-  for (const p of methodRequest.parameterList) {
-    const pvs = paramList.node("ParameterValueStruct");
-    pvs.node("Name").text(p[0]);
-    let val = p[1];
-    if (p[2] === "xsd:dateTime" && typeof val === "number") {
-      val = new Date(val).toISOString();
-      if (methodRequest.DATETIME_MILLISECONDS === false)
-        val = val.replace(".000", "");
-    }
-    if (p[2] === "xsd:boolean" && typeof val === "boolean")
-      if (methodRequest.BOOLEAN_LITERAL === false) val = +val;
-
-    pvs
-      .node("Value")
-      .attr({ "xsi:type": p[2] })
-      .text(val);
-  }
-
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-function SetParameterValuesResponse(xml): CpeSetResponse {
-  return {
-    name: "SetParameterValuesResponse",
-    status: parseInt(
-      xml
-        .childNodes()
-        .find(n => n.name() === "Status")
-        .text()
-    )
-  };
-}
-
-function AddObject(xml, methodRequest): void {
-  const el = xml.node("cwmp:AddObject");
-  el.node("ObjectName").text(methodRequest.objectName);
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-function AddObjectResponse(xml): CpeSetResponse {
-  let instanceNumber, status;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "InstanceNumber":
-        instanceNumber = parseInt(c.text());
-        break;
-      case "Status":
-        status = parseInt(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "AddObjectResponse",
-    instanceNumber: instanceNumber,
-    status: status
-  };
-}
-
-function DeleteObject(xml, methodRequest): void {
-  const el = xml.node("cwmp:DeleteObject");
-  el.node("ObjectName").text(methodRequest.objectName);
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-function DeleteObjectResponse(xml): CpeSetResponse {
-  return {
-    name: "DeleteObjectResponse",
-    status: parseInt(
-      xml
-        .childNodes()
-        .find(n => n.name() === "Status")
-        .text()
-    )
-  };
-}
-
-function Reboot(xml, methodRequest): void {
-  const el = xml.node("cwmp:Reboot");
-  el.node("CommandKey").text(methodRequest.commandKey || "");
-}
-
-function RebootResponse(): CpeSetResponse {
-  return {
-    name: "RebootResponse"
-  };
-}
-
-function FactoryReset(xml): void {
-  xml.node("cwmp:FactoryReset");
-}
-
-function FactoryResetResponse(): CpeSetResponse {
-  return {
-    name: "FactoryResetResponse"
-  };
-}
-
-function Download(xml, methodRequest): void {
-  const el = xml.node("cwmp:Download");
-  el.node("CommandKey").text(methodRequest.commandKey || "");
-  el.node("FileType").text(methodRequest.fileType);
-  el.node("URL").text(methodRequest.url);
-  el.node("Username").text(methodRequest.username || "");
-  el.node("Password").text(methodRequest.password || "");
-  el.node("FileSize").text(methodRequest.fileSize || "0");
-  el.node("TargetFileName").text(methodRequest.targetFileName || "");
-  el.node("DelaySeconds").text(methodRequest.delaySeconds || "0");
-  el.node("SuccessURL").text(methodRequest.successUrl || "");
-  el.node("FailureURL").text(methodRequest.failureUrl || "");
-}
-
-function DownloadResponse(xml): CpeSetResponse {
-  let status, startTime, completeTime;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "Status":
-        status = parseInt(c.text());
-        break;
-      case "StartTime":
-        startTime = Date.parse(c.text());
-        break;
-      case "CompleteTime":
-        completeTime = Date.parse(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "DownloadResponse",
-    status: status,
-    startTime: startTime,
-    completeTime: completeTime
-  };
-}
-
-function Inform(xml): InformRequest {
-  let retryCount, evnt, parameterList;
-  const deviceId = {
-    Manufacturer: null,
-    OUI: null,
-    ProductClass: null,
-    SerialNumber: null
-  };
-
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "ParameterList":
-        parameterList = parameterValueList(c);
-        break;
-      case "DeviceId":
-        for (const cc of c.childNodes()) {
-          const n = cc.name();
-          if (n in deviceId) deviceId[n] = cc.text();
-        }
-        break;
-      case "Event":
-        evnt = event(c);
-        break;
-      case "RetryCount":
-        retryCount = parseInt(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "Inform",
-    parameterList: parameterList,
-    deviceId: deviceId,
-    event: evnt,
-    retryCount: retryCount
-  };
-}
-
-function InformResponse(xml): void {
-  xml
-    .node("cwmp:InformResponse")
-    .node("MaxEnvelopes")
-    .text(1);
-}
-
-function GetRPCMethods(): AcsResponse {
-  return { name: "GetRPCMethods" };
-}
-
-function GetRPCMethodsResponse(xml, methodResponse): void {
-  const el = xml.node("cwmp:GetRPCMethodsResponse").node("MethodList");
-  el.attr({
-    "soap-enc:arrayType": `xsd:string[${methodResponse.methodList.length}]`
-  });
-
-  for (const m of methodResponse.methodList) el.node("string").text(m);
-}
-
-function TransferComplete(xml): TransferCompleteRequest {
-  let commandKey, _faultStruct, startTime, completeTime;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "CommandKey":
-        commandKey = c.text();
-        break;
-      case "FaultStruct":
-        _faultStruct = faultStruct(c);
-        break;
-      case "StartTime":
-        startTime = Date.parse(c.text());
-        break;
-      case "CompleteTime":
-        completeTime = Date.parse(c.text());
-        break;
-    }
-  }
-
-  return {
-    name: "TransferComplete",
-    commandKey: commandKey,
-    faultStruct: _faultStruct,
-    startTime: startTime,
-    completeTime: completeTime
-  };
-}
-
-function TransferCompleteResponse(xml): void {
-  xml.node("cwmp:TransferCompleteResponse").text("");
-}
-
-function RequestDownload(xml): CpeRequest {
-  return {
-    name: "RequestDownload",
-    fileType: xml
-      .childNodes()
-      .find(n => n.name() === "FileType")
-      .text()
-  };
-}
-
-function RequestDownloadResponse(xml): void {
-  xml.node("cwmp:RequestDownloadResponse").text("");
-}
-
-function faultStruct(xml): FaultStruct {
-  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "FaultCode":
-        faultCode = c.text();
-        break;
-      case "FaultString":
-        faultString = c.text();
-        break;
-      case "SetParameterValuesFault":
-        setParameterValuesFault = setParameterValuesFault || [];
-        for (const cc of c.childNodes()) {
-          switch (cc.name()) {
-            case "ParameterName":
-              pn = cc.text();
-              break;
-            case "FaultCode":
-              fc = cc.text();
-              break;
-            case "FaultString":
-              fs = cc.text();
-              break;
-          }
-        }
-        setParameterValuesFault.push({
-          parameterName: pn,
-          faultCode: fc,
-          faultString: fs
-        });
-    }
-  }
-
-  return { faultCode, faultString, setParameterValuesFault };
-}
-
-function fault(xml): CpeFault {
-  let faultCode, faultString, detail;
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "faultcode":
-        faultCode = c.text();
-        break;
-      case "faultstring":
-        faultString = c.text();
-        break;
-      case "detail":
-        detail = faultStruct(c.childNodes().find(n => n.name() === "Fault"));
-        break;
-    }
-  }
-
-  return { faultCode, faultString, detail };
-}
-
-export function request(data, cwmpVersion, warn): SoapMessage {
-  warnings = warn;
-
-  const rpc = {
-    id: null,
-    cwmpVersion: cwmpVersion,
-    sessionTimeout: null,
-    cpeRequest: null,
-    cpeFault: null,
-    cpeResponse: null
-  };
-
-  if (!data.length) return rpc;
-
-  const xml = parseXml(data, LIBXMLJS_PARSE_OPTIONS);
-
-  let headerElement, bodyElement;
-
-  for (const c of xml.childNodes()) {
-    switch (c.name()) {
-      case "Header":
-        headerElement = c;
-        break;
-      case "Body":
-        bodyElement = c;
-        break;
-    }
-  }
-
-  if (headerElement) {
-    for (const c of headerElement.childNodes()) {
-      switch (c.name()) {
-        case "ID":
-          rpc.id = c.text();
-          break;
-        case "sessionTimeout":
-          rpc.sessionTimeout = parseInt(c.text());
-          break;
-      }
-    }
-  }
-
-  for (const methodElement of bodyElement.childNodes()) {
-    const methodName = methodElement.name();
-    if (methodName === "text") continue;
-
-    if (!rpc.cwmpVersion && methodName !== "Fault") {
-      switch (methodElement.namespace().href()) {
-        case "urn:dslforum-org:cwmp-1-0":
-          rpc.cwmpVersion = "1.0";
-          break;
-        case "urn:dslforum-org:cwmp-1-1":
-          rpc.cwmpVersion = "1.1";
-          break;
-        case "urn:dslforum-org:cwmp-1-2":
-          if (rpc.sessionTimeout) rpc.cwmpVersion = "1.3";
-          else rpc.cwmpVersion = "1.2";
-
-          break;
-        case "urn:dslforum-org:cwmp-1-3":
-          rpc.cwmpVersion = "1.4";
-          break;
-        default:
-          throw new Error("Unrecognized CWMP version");
-      }
-    }
-
-    switch (methodName) {
-      case "Inform":
-        rpc.cpeRequest = Inform(methodElement);
-        break;
-      case "GetRPCMethods":
-        rpc.cpeRequest = GetRPCMethods();
-        break;
-      case "TransferComplete":
-        rpc.cpeRequest = TransferComplete(methodElement);
-        break;
-      case "RequestDownload":
-        rpc.cpeRequest = RequestDownload(methodElement);
-        break;
-      case "GetParameterNamesResponse":
-        rpc.cpeResponse = GetParameterNamesResponse(methodElement);
-        break;
-      case "GetParameterValuesResponse":
-        rpc.cpeResponse = GetParameterValuesResponse(methodElement);
-        break;
-      case "SetParameterValuesResponse":
-        rpc.cpeResponse = SetParameterValuesResponse(methodElement);
-        break;
-      case "AddObjectResponse":
-        rpc.cpeResponse = AddObjectResponse(methodElement);
-        break;
-      case "DeleteObjectResponse":
-        rpc.cpeResponse = DeleteObjectResponse(methodElement);
-        break;
-      case "RebootResponse":
-        rpc.cpeResponse = RebootResponse();
-        break;
-      case "FactoryResetResponse":
-        rpc.cpeResponse = FactoryResetResponse();
-        break;
-      case "DownloadResponse":
-        rpc.cpeResponse = DownloadResponse(methodElement);
-        break;
-      case "Fault":
-        rpc.cpeFault = fault(methodElement);
-        break;
-      default:
-        throw new Error(`8000 Method not supported ${methodName}`);
-    }
-  }
-
-  return rpc;
-}
-
-function createSoapEnv(cwmpVersion: string): Element {
-  const xml = new Document();
-  const env = xml.node("soap-env:Envelope");
-
-  for (const prefix of Object.keys(NAMESPACES[cwmpVersion]))
-    env.defineNamespace(prefix, NAMESPACES[cwmpVersion][prefix]);
-
-  return env;
-}
-
-export function response(rpc): { code: number; headers: {}; data: string } {
-  const headers = {
-    Server: SERVER_NAME,
-    SOAPServer: SERVER_NAME
-  };
-
-  if (!rpc) return { code: 204, headers: headers, data: "" };
-
-  const env = createSoapEnv(rpc.cwmpVersion);
-  const header = env.node("soap-env:Header");
-  header
-    .node("cwmp:ID")
-    .attr({
-      "soap-env:mustUnderstand": "1"
-    })
-    .text(rpc.id);
-
-  const body = env.node("soap-env:Body");
-
-  if (rpc.acsResponse) {
-    switch (rpc.acsResponse.name) {
-      case "InformResponse":
-        InformResponse(body);
-        break;
-      case "GetRPCMethodsResponse":
-        GetRPCMethodsResponse(body, rpc.acsResponse);
-        break;
-      case "TransferCompleteResponse":
-        TransferCompleteResponse(body);
-        break;
-      case "RequestDownloadResponse":
-        RequestDownloadResponse(body);
-        break;
-      default:
-        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
-    }
-  } else if (rpc.acsRequest) {
-    switch (rpc.acsRequest.name) {
-      case "GetParameterNames":
-        GetParameterNames(body, rpc.acsRequest);
-        break;
-      case "GetParameterValues":
-        GetParameterValues(body, rpc.acsRequest);
-        break;
-      case "SetParameterValues":
-        SetParameterValues(body, rpc.acsRequest);
-        break;
-      case "AddObject":
-        AddObject(body, rpc.acsRequest);
-        break;
-      case "DeleteObject":
-        DeleteObject(body, rpc.acsRequest);
-        break;
-      case "Reboot":
-        Reboot(body, rpc.acsRequest);
-        break;
-      case "FactoryReset":
-        FactoryReset(body);
-        break;
-      case "Download":
-        Download(body, rpc.acsRequest);
-        break;
-      default:
-        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
-    }
-  }
-
-  headers["Content-Type"] = 'text/xml; charset="utf-8"';
-  return {
-    code: 200,
-    headers: headers,
-    data: env.doc().toString(LIBXMLJS_SAVE_OPTIONS)
-  };
-}

lib/soap.ts

@@ -1,43 +1,36 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 import {
   parseXml,
   Element,
   parseAttrs,
   encodeEntities,
-  decodeEntities
-} from "./xml-parser";
-import memoize from "./common/memoize";
+  decodeEntities,
+} from "./xml-parser.ts";
+import memoize from "./common/memoize.ts";
 import { version as VERSION } from "../package.json";
 import {
-  CpeGetResponse,
-  CpeSetResponse,
   InformRequest,
-  AcsResponse,
-  CpeRequest,
   FaultStruct,
   SpvFault,
   CpeFault,
   SoapMessage,
-  TransferCompleteRequest
-} from "./types";
+  TransferCompleteRequest,
+  AcsRequest,
+  type GetParameterNamesResponse,
+  type GetParameterValuesResponse,
+  type GetParameterAttributesResponse,
+  type SetParameterValuesResponse,
+  type SetParameterAttributesResponse,
+  type AddObjectResponse,
+  type DeleteObject,
+  type DeleteObjectResponse,
+  type RebootResponse,
+  type FactoryResetResponse,
+  type DownloadResponse,
+  GetRPCMethodsRequest,
+  RequestDownloadRequest,
+  AcsResponse,
+} from "./types.ts";
+import Path from "./common/path.ts";
 
 const SERVER_NAME = `GenieACS/${VERSION}`;
 
@@ -47,60 +40,58 @@ const NAMESPACES = {
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-0"
+    cwmp: "urn:dslforum-org:cwmp-1-0",
   },
   "1.1": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-1"
+    cwmp: "urn:dslforum-org:cwmp-1-1",
   },
   "1.2": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
+    cwmp: "urn:dslforum-org:cwmp-1-2",
   },
   "1.3": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-2"
+    cwmp: "urn:dslforum-org:cwmp-1-2",
   },
   "1.4": {
     "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
     "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
     xsd: "http://www.w3.org/2001/XMLSchema",
     xsi: "http://www.w3.org/2001/XMLSchema-instance",
-    cwmp: "urn:dslforum-org:cwmp-1-3"
-  }
+    cwmp: "urn:dslforum-org:cwmp-1-3",
+  },
 };
 
-let warnings;
+let warnings: Record<string, unknown>[];
 
 const memoizedParseAttrs = memoize(parseAttrs);
 
-function parseBool(v): boolean {
-  v = "" + v;
-  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
-  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
-    return false;
-  else return null;
+function parseBool(v: string): boolean {
+  if (v === "true" || v === "1") return true;
+  if (v === "false" || v === "0") return false;
+  return null;
 }
 
 function event(xml: Element): string[] {
   return xml.children
-    .filter(n => n.localName === "EventStruct")
-    .map(c => c.children.find(n => n.localName === "EventCode").text.trim());
+    .filter((n) => n.localName === "EventStruct")
+    .map((c) => c.children.find((n) => n.localName === "EventCode").text);
 }
 
-function parameterInfoList(xml: Element): [string, boolean][] {
+function parameterInfoList(xml: Element): [Path, boolean, boolean][] {
   return xml.children
-    .filter(e => e.localName === "ParameterInfoStruct")
-    .map<[string, boolean]>(e => {
+    .map<[Path, boolean, boolean]>((e) => {
+      if (e.localName !== "ParameterInfoStruct") return null;
       let param: string, value: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -115,30 +106,44 @@ function parameterInfoList(xml: Element): [string, boolean][] {
 
       let parsed: boolean = parseBool(value);
 
-      if (parsed === null) {
+      if (parsed == null) {
         warnings.push({
-          message: "Invalid writable attribute",
-          parameter: param
+          message: "Missing or invalid XML node",
+          element: "Writable",
+          parameter: param,
         });
         parsed = false;
       }
 
-      return [param, parsed];
-    });
+      try {
+        if (param && !param.endsWith("."))
+          return [Path.parse(param), false, parsed];
+        else return [Path.parse(param.slice(0, -1)), true, parsed];
+      } catch {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
-const getValueType = memoize((str: string) =>
-  parseAttrs(str)
-    .find(s => s.localName === "type")
-    .value.trim()
-);
+const getValueType = memoize((str: string) => {
+  const attrs = parseAttrs(str);
+  for (const attr of attrs) if (attr.localName === "type") return attr.value;
+
+  return null;
+});
 
 function parameterValueList(
-  xml: Element
-): [string, string | number | boolean, string][] {
+  xml: Element,
+): [Path, string | number | boolean, string][] {
   return xml.children
-    .filter(e => e.localName === "ParameterValueStruct")
-    .map<[string, string | number | boolean, string]>(e => {
+    .map<[Path, string | number | boolean, string]>((e) => {
+      if (e.localName !== "ParameterValueStruct") return null;
       let valueElement: Element, param: string;
       for (const c of e.children) {
         switch (c.localName) {
@@ -151,16 +156,25 @@ function parameterValueList(
         }
       }
 
-      const valueType = getValueType(valueElement.attrs);
+      let valueType = getValueType(valueElement.attrs);
+      if (!valueType) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          attribute: "type",
+          parameter: param,
+        });
+        valueType = "xsd:string";
+      }
 
       const value = decodeEntities(valueElement.text);
       let parsed: string | number | boolean = value;
       if (valueType === "xsd:boolean") {
         parsed = parseBool(value);
-        if (parsed === null) {
+        if (parsed == null) {
           warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
+            message: "Missing or invalid XML node",
+            element: "Value",
+            parameter: param,
           });
           parsed = value;
         }
@@ -168,8 +182,9 @@ function parameterValueList(
         parsed = parseInt(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
+            message: "Missing or invalid XML node",
+            element: "Value",
+            parameter: param,
           });
           parsed = value;
         }
@@ -177,15 +192,74 @@ function parameterValueList(
         parsed = Date.parse(value);
         if (isNaN(parsed)) {
           warnings.push({
-            message: "Invalid value attribute",
-            parameter: param
+            message: "Missing or invalid XML node",
+            element: "Value",
+            parameter: param,
           });
           parsed = value;
         }
       }
+      try {
+        return [Path.parse(param), parsed, valueType];
+      } catch {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
+}
 
-      return [param, parsed, valueType];
-    });
+function parameterAttributeList(xml: Element): [Path, number, string[]][] {
+  return xml.children
+    .map<[Path, number, string[]]>((e) => {
+      if (e.localName !== "ParameterAttributeStruct") return null;
+      let notificationElement: Element,
+        accessListElement: Element,
+        param: string;
+      for (const c of e.children) {
+        switch (c.localName) {
+          case "Name":
+            param = c.text;
+            break;
+          case "Notification":
+            notificationElement = c;
+            break;
+          case "AccessList":
+            accessListElement = c;
+            break;
+        }
+      }
+
+      let notification = parseInt(notificationElement.text);
+      if (isNaN(notification)) {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Notification",
+          parameter: param,
+        });
+        notification = 0;
+      }
+
+      const accessList = accessListElement.children
+        .filter((c) => c.localName === "string")
+        .map((c) => decodeEntities(c.text));
+
+      try {
+        return [Path.parse(param), notification, accessList];
+      } catch {
+        warnings.push({
+          message: "Missing or invalid XML node",
+          element: "Name",
+          parameter: param,
+        });
+        return null;
+      }
+    })
+    .filter((e) => e != null);
 }
 
 function GetParameterNames(methodRequest): string {
@@ -194,12 +268,12 @@ function GetParameterNames(methodRequest): string {
   }</ParameterPath><NextLevel>${+methodRequest.nextLevel}</NextLevel></cwmp:GetParameterNames>`;
 }
 
-function GetParameterNamesResponse(xml): CpeGetResponse {
+function GetParameterNamesResponse(xml): GetParameterNamesResponse {
   return {
     name: "GetParameterNamesResponse",
     parameterList: parameterInfoList(
-      xml.children.find(n => n.localName === "ParameterList")
-    )
+      xml.children.find((n) => n.localName === "ParameterList"),
+    ),
   };
 }
 
@@ -207,21 +281,40 @@ function GetParameterValues(methodRequest): string {
   return `<cwmp:GetParameterValues><ParameterNames soap-enc:arrayType="xsd:string[${
     methodRequest.parameterNames.length
   }]">${methodRequest.parameterNames
-    .map(p => `<string>${p}</string>`)
+    .map((p) => `<string>${p}</string>`)
     .join("")}</ParameterNames></cwmp:GetParameterValues>`;
 }
 
-function GetParameterValuesResponse(xml: Element): CpeGetResponse {
+function GetParameterValuesResponse(xml: Element): GetParameterValuesResponse {
   return {
     name: "GetParameterValuesResponse",
     parameterList: parameterValueList(
-      xml.children.find(n => n.localName === "ParameterList")
-    )
+      xml.children.find((n) => n.localName === "ParameterList"),
+    ),
+  };
+}
+
+function GetParameterAttributes(methodRequest): string {
+  return `<cwmp:GetParameterAttributes><ParameterNames soap-enc:arrayType="xsd:string[${
+    methodRequest.parameterNames.length
+  }]">${methodRequest.parameterNames
+    .map((p) => `<string>${p}</string>`)
+    .join("")}</ParameterNames></cwmp:GetParameterAttributes>`;
+}
+
+function GetParameterAttributesResponse(
+  xml: Element,
+): GetParameterAttributesResponse {
+  return {
+    name: "GetParameterAttributesResponse",
+    parameterList: parameterAttributeList(
+      xml.children.find((n) => n.localName === "ParameterList"),
+    ),
   };
 }
 
 function SetParameterValues(methodRequest): string {
-  const params = methodRequest.parameterList.map(p => {
+  const params = methodRequest.parameterList.map((p) => {
     let val = p[1];
     if (p[2] === "xsd:dateTime" && typeof val === "number") {
       val = new Date(val).toISOString();
@@ -237,32 +330,80 @@ function SetParameterValues(methodRequest): string {
 
   return `<cwmp:SetParameterValues><ParameterList soap-enc:arrayType="cwmp:ParameterValueStruct[${
     methodRequest.parameterList.length
-  }]">${params.join(
-    ""
-  )}</ParameterList><ParameterKey>${methodRequest.parameterKey ||
-    ""}</ParameterKey></cwmp:SetParameterValues>`;
+  }]">${params.join("")}</ParameterList><ParameterKey>${
+    methodRequest.parameterKey || ""
+  }</ParameterKey></cwmp:SetParameterValues>`;
 }
 
-function SetParameterValuesResponse(xml: Element): CpeSetResponse {
+function SetParameterValuesResponse(xml: Element): SetParameterValuesResponse {
+  let status: number;
+
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "Status":
+        status = parseInt(c.text);
+        break;
+    }
+  }
+
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
   return {
     name: "SetParameterValuesResponse",
-    status: parseInt(xml.children.find(n => n.localName === "Status").text)
+    status: status,
+  };
+}
+
+function SetParameterAttributes(methodRequest): string {
+  const params = methodRequest.parameterList.map((p) => {
+    return `<SetParameterAttributesStruct><Name>${
+      p[0]
+    }</Name><NotificationChange>${
+      p[1] == null ? "false" : "true"
+    }</NotificationChange><Notification>${
+      p[1] == null ? "" : p[1]
+    }</Notification><AccessListChange>${
+      p[2] == null ? "false" : "true"
+    }</AccessListChange><AccessList soap-enc:arrayType="xsd:string[${
+      (p[2] || []).length
+    }]">${
+      p[2] == null
+        ? ""
+        : p[2].map((s) => `<string>${encodeEntities(s)}</string>`).join("")
+    }</AccessList></SetParameterAttributesStruct>`;
+  });
+
+  return `<cwmp:SetParameterAttributes><ParameterList soap-enc:arrayType="cwmp:SetParameterAttributesStruct[${
+    methodRequest.parameterList.length
+  }]">${params.join("")}</ParameterList></cwmp:SetParameterAttributes>`;
+}
+
+function SetParameterAttributesResponse(): SetParameterAttributesResponse {
+  return {
+    name: "SetParameterAttributesResponse",
   };
 }
 
 function AddObject(methodRequest): string {
   return `<cwmp:AddObject><ObjectName>${
     methodRequest.objectName
-  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
-    ""}</ParameterKey></cwmp:AddObject>`;
+  }</ObjectName><ParameterKey>${
+    methodRequest.parameterKey || ""
+  }</ParameterKey></cwmp:AddObject>`;
 }
 
-function AddObjectResponse(xml: Element): CpeSetResponse {
-  let instanceNumber, status;
+function AddObjectResponse(xml: Element): AddObjectResponse {
+  let instanceNumber: string, status: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "InstanceNumber":
-        instanceNumber = parseInt(c.text);
+        instanceNumber = c.text;
         break;
       case "Status":
         status = parseInt(c.text);
@@ -270,35 +411,66 @@ function AddObjectResponse(xml: Element): CpeSetResponse {
     }
   }
 
+  if (!/^[0-9]+$/.test(instanceNumber))
+    throw new Error("Missing or invalid instance number");
+
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
   return {
     name: "AddObjectResponse",
     instanceNumber: instanceNumber,
-    status: status
+    status: status,
   };
 }
 
 function DeleteObject(methodRequest): string {
   return `<cwmp:DeleteObject><ObjectName>${
     methodRequest.objectName
-  }</ObjectName><ParameterKey>${methodRequest.parameterKey ||
-    ""}</ParameterKey></cwmp:DeleteObject>`;
+  }</ObjectName><ParameterKey>${
+    methodRequest.parameterKey || ""
+  }</ParameterKey></cwmp:DeleteObject>`;
 }
 
-function DeleteObjectResponse(xml: Element): CpeSetResponse {
+function DeleteObjectResponse(xml: Element): DeleteObjectResponse {
+  let status: number;
+
+  for (const c of xml.children) {
+    switch (c.localName) {
+      case "Status":
+        status = parseInt(c.text);
+        break;
+    }
+  }
+
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
   return {
     name: "DeleteObjectResponse",
-    status: parseInt(xml.children.find(n => n.localName === "Status").text)
+    status: status,
   };
 }
 
 function Reboot(methodRequest): string {
-  return `<cwmp:Reboot><CommandKey>${methodRequest.commandKey ||
-    ""}</CommandKey></cwmp:Reboot>`;
+  return `<cwmp:Reboot><CommandKey>${
+    methodRequest.commandKey || ""
+  }</CommandKey></cwmp:Reboot>`;
 }
 
-function RebootResponse(): CpeSetResponse {
+function RebootResponse(): RebootResponse {
   return {
-    name: "RebootResponse"
+    name: "RebootResponse",
   };
 }
 
@@ -306,33 +478,36 @@ function FactoryReset(): string {
   return "<cwmp:FactoryReset></cwmp:FactoryReset>";
 }
 
-function FactoryResetResponse(): CpeSetResponse {
+function FactoryResetResponse(): FactoryResetResponse {
   return {
-    name: "FactoryResetResponse"
+    name: "FactoryResetResponse",
   };
 }
 
 function Download(methodRequest): string {
-  return `<cwmp:Download><CommandKey>${methodRequest.commandKey ||
-    ""}</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
+  return `<cwmp:Download><CommandKey>${
+    methodRequest.commandKey || ""
+  }</CommandKey><FileType>${methodRequest.fileType}</FileType><URL>${
     methodRequest.url
   }</URL><Username>${encodeEntities(
-    methodRequest.username || ""
+    methodRequest.username || "",
   )}</Username><Password>${encodeEntities(
-    methodRequest.password || ""
-  )}</Password><FileSize>${methodRequest.fileSize ||
-    "0"}</FileSize><TargetFileName>${encodeEntities(
-    methodRequest.targetFileName || ""
-  )}</TargetFileName><DelaySeconds>${methodRequest.delaySeconds ||
-    "0"}</DelaySeconds><SuccessURL>${encodeEntities(
-    methodRequest.successUrl || ""
+    methodRequest.password || "",
+  )}</Password><FileSize>${
+    methodRequest.fileSize || "0"
+  }</FileSize><TargetFileName>${encodeEntities(
+    methodRequest.targetFileName || "",
+  )}</TargetFileName><DelaySeconds>${
+    methodRequest.delaySeconds || "0"
+  }</DelaySeconds><SuccessURL>${encodeEntities(
+    methodRequest.successUrl || "",
   )}</SuccessURL><FailureURL>${encodeEntities(
-    methodRequest.failureUrl || ""
+    methodRequest.failureUrl || "",
   )}</FailureURL></cwmp:Download>`;
 }
 
-function DownloadResponse(xml: Element): CpeSetResponse {
-  let status, startTime, completeTime;
+function DownloadResponse(xml: Element): DownloadResponse {
+  let status: number, startTime: number, completeTime: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "Status":
@@ -347,21 +522,46 @@ function DownloadResponse(xml: Element): CpeSetResponse {
     }
   }
 
+  if (!(status >= 0)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "Status",
+    });
+    status = 0;
+  }
+
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "DownloadResponse",
     status: status,
     startTime: startTime,
-    completeTime: completeTime
+    completeTime: completeTime,
   };
 }
 
 function Inform(xml: Element): InformRequest {
-  let retryCount, evnt, parameterList;
+  let retryCount: number, evnt: string[];
+  let parameterList: [Path, string | number | boolean, string][];
   const deviceId = {
     Manufacturer: null,
     OUI: null,
     ProductClass: null,
-    SerialNumber: null
+    SerialNumber: null,
   };
 
   for (const c of xml.children) {
@@ -384,12 +584,36 @@ function Inform(xml: Element): InformRequest {
     }
   }
 
+  if (!deviceId || !deviceId.SerialNumber || !deviceId.OUI)
+    throw new Error("Missing or invalid DeviceId element");
+
+  if (!parameterList) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "ParameterList",
+    });
+    parameterList = [];
+  }
+
+  if (!evnt) {
+    warnings.push({ message: "Missing or invalid XML node", element: "Event" });
+    evnt = [];
+  }
+
+  if (retryCount == null || isNaN(retryCount)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "RetryCount",
+    });
+    retryCount = 0;
+  }
+
   return {
     name: "Inform",
     parameterList: parameterList,
     deviceId: deviceId,
     event: evnt,
-    retryCount: retryCount
+    retryCount: retryCount,
   };
 }
 
@@ -397,7 +621,7 @@ function InformResponse(): string {
   return "<cwmp:InformResponse><MaxEnvelopes>1</MaxEnvelopes></cwmp:InformResponse>";
 }
 
-function GetRPCMethods(): AcsResponse {
+function GetRPCMethods(): GetRPCMethodsRequest {
   return { name: "GetRPCMethods" };
 }
 
@@ -405,12 +629,15 @@ function GetRPCMethodsResponse(methodResponse): string {
   return `<cwmp:GetRPCMethodsResponse><MethodList soap-enc:arrayType="xsd:string[${
     methodResponse.methodList.length
   }]">${methodResponse.methodList
-    .map(m => `<string>${m}</string>`)
+    .map((m) => `<string>${m}</string>`)
     .join("")}</MethodList></cwmp:GetRPCMethodsResponse>`;
 }
 
 function TransferComplete(xml: Element): TransferCompleteRequest {
-  let commandKey, _faultStruct, startTime, completeTime;
+  let commandKey: string,
+    _faultStruct: FaultStruct,
+    startTime: number,
+    completeTime: number;
   for (const c of xml.children) {
     switch (c.localName) {
       case "CommandKey":
@@ -428,12 +655,44 @@ function TransferComplete(xml: Element): TransferCompleteRequest {
     }
   }
 
+  if (commandKey == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CommandKey",
+    });
+    commandKey = "";
+  }
+
+  if (!_faultStruct) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultStruct",
+    });
+    _faultStruct = { faultCode: "0", faultString: "" };
+  }
+
+  if (startTime == null || isNaN(startTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "StartTime",
+    });
+    startTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
+  if (completeTime == null || isNaN(completeTime)) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "CompleteTime",
+    });
+    completeTime = Date.parse("0001-01-01T00:00:00Z");
+  }
+
   return {
     name: "TransferComplete",
     commandKey: commandKey,
     faultStruct: _faultStruct,
     startTime: startTime,
-    completeTime: completeTime
+    completeTime: completeTime,
   };
 }
 
@@ -441,10 +700,10 @@ function TransferCompleteResponse(): string {
   return "<cwmp:TransferCompleteResponse></cwmp:TransferCompleteResponse>";
 }
 
-function RequestDownload(xml: Element): CpeRequest {
+function RequestDownload(xml: Element): RequestDownloadRequest {
   return {
     name: "RequestDownload",
-    fileType: xml.children.find(n => n.localName === "FileType").text
+    fileType: xml.children.find((n) => n.localName === "FileType").text,
   };
 }
 
@@ -452,8 +711,25 @@ function RequestDownloadResponse(): string {
   return "<cwmp:RequestDownloadResponse></cwmp:RequestDownloadResponse>";
 }
 
+function AcsFault(f: CpeFault): string {
+  return `<soap-env:Body:Fault><faultcode>${encodeEntities(
+    f.faultCode,
+  )}</faultcode><faultstring>${encodeEntities(
+    f.faultString,
+  )}</faultstring><detail><cwmp:Fault><FaultCode>${encodeEntities(
+    f.detail.faultCode,
+  )}</FaultCode><FaultString>${encodeEntities(
+    f.detail.faultString,
+  )}</FaultString></cwmp:Fault></detail></soap-env:Body:Fault>`;
+}
+
 function faultStruct(xml: Element): FaultStruct {
-  let faultCode, faultString, setParameterValuesFault: SpvFault[], pn, fc, fs;
+  let faultCode: string,
+    faultString: string,
+    setParameterValuesFault: SpvFault[],
+    pn: string,
+    fc: string,
+    fs: string;
   for (const c of xml.children) {
     switch (c.localName) {
       case "FaultCode":
@@ -464,6 +740,7 @@ function faultStruct(xml: Element): FaultStruct {
         break;
       case "SetParameterValuesFault":
         setParameterValuesFault = setParameterValuesFault || [];
+        pn = fc = fs = null;
         for (const cc of c.children) {
           switch (cc.localName) {
             case "ParameterName":
@@ -480,16 +757,32 @@ function faultStruct(xml: Element): FaultStruct {
         setParameterValuesFault.push({
           parameterName: pn,
           faultCode: fc,
-          faultString: fs
+          faultString: fs,
         });
     }
   }
 
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultCode",
+    });
+    faultCode = "";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "FaultString",
+    });
+    faultString = "";
+  }
+
   return { faultCode, faultString, setParameterValuesFault };
 }
 
 function fault(xml: Element): CpeFault {
-  let faultCode, faultString, detail;
+  let faultCode: string, faultString: string, detail: FaultStruct;
   for (const c of xml.children) {
     switch (c.localName) {
       case "faultcode":
@@ -499,24 +792,46 @@ function fault(xml: Element): CpeFault {
         faultString = decodeEntities(c.text);
         break;
       case "detail":
-        detail = faultStruct(c.children.find(n => n.localName === "Fault"));
+        detail = faultStruct(c.children.find((n) => n.localName === "Fault"));
         break;
     }
   }
 
-  return { faultCode, faultString, detail };
+  if (!detail) throw new Error("Missing detail element");
+
+  if (faultCode == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultcode",
+    });
+    faultCode = "Client";
+  }
+
+  if (faultString == null) {
+    warnings.push({
+      message: "Missing or invalid XML node",
+      element: "faultstring",
+    });
+    faultString = "CWMP fault";
+  }
+
+  return { faultCode, faultString, detail } as CpeFault;
 }
 
-export function request(body: string, cwmpVersion, warn): SoapMessage {
+export function request(
+  body: string,
+  warn: Record<string, unknown>[],
+): SoapMessage {
   warnings = warn;
 
   const rpc = {
     id: null,
-    cwmpVersion: cwmpVersion,
+    cwmpVersion: null,
     sessionTimeout: null,
     cpeRequest: null,
     cpeFault: null,
-    cpeResponse: null
+    cpeResponse: null,
+    unknownMethod: null,
   };
 
   if (!body.length) return rpc;
@@ -555,7 +870,7 @@ export function request(body: string, cwmpVersion, warn): SoapMessage {
 
   const methodElement = bodyElement.children[0];
 
-  if (!rpc.cwmpVersion && methodElement.localName !== "Fault") {
+  if (methodElement.localName === "Inform") {
     let namespace, namespaceHref;
     for (const e of [methodElement, bodyElement, envelope]) {
       namespace = namespace || e.namespace;
@@ -563,9 +878,9 @@ export function request(body: string, cwmpVersion, warn): SoapMessage {
         const attrs = memoizedParseAttrs(e.attrs);
         const attr = namespace
           ? attrs.find(
-              s => s.namespace === "xmlns" && s.localName === namespace
+              (s) => s.namespace === "xmlns" && s.localName === namespace,
             )
-          : attrs.find(s => s.name === "xmlns");
+          : attrs.find((s) => s.name === "xmlns");
 
         if (attr) namespaceHref = attr.value;
       }
@@ -610,9 +925,15 @@ export function request(body: string, cwmpVersion, warn): SoapMessage {
     case "GetParameterValuesResponse":
       rpc.cpeResponse = GetParameterValuesResponse(methodElement);
       break;
+    case "GetParameterAttributesResponse":
+      rpc.cpeResponse = GetParameterAttributesResponse(methodElement);
+      break;
     case "SetParameterValuesResponse":
       rpc.cpeResponse = SetParameterValuesResponse(methodElement);
       break;
+    case "SetParameterAttributesResponse":
+      rpc.cpeResponse = SetParameterAttributesResponse();
+      break;
     case "AddObjectResponse":
       rpc.cpeResponse = AddObjectResponse(methodElement);
       break;
@@ -632,7 +953,8 @@ export function request(body: string, cwmpVersion, warn): SoapMessage {
       rpc.cpeFault = fault(methodElement);
       break;
     default:
-      throw new Error(`8000 Method not supported ${methodElement.localName}`);
+      rpc.unknownMethod = methodElement.localName;
+      break;
   }
 
   return rpc;
@@ -653,13 +975,19 @@ const namespacesAttrs = {
     .join(" "),
   "1.4": Object.entries(NAMESPACES["1.4"])
     .map(([k, v]) => `xmlns:${k}="${v}"`)
-    .join(" ")
+    .join(" "),
 };
 
-export function response(rpc): { code: number; headers: {}; data: string } {
+export function response(rpc: {
+  id: string;
+  acsRequest?: AcsRequest;
+  acsResponse?: AcsResponse;
+  acsFault?: CpeFault;
+  cwmpVersion?: string;
+}): { code: number; headers: Record<string, string>; data: string } {
   const headers = {
     Server: SERVER_NAME,
-    SOAPServer: SERVER_NAME
+    SOAPServer: SERVER_NAME,
   };
 
   if (!rpc) return { code: 204, headers: headers, data: "" };
@@ -680,7 +1008,11 @@ export function response(rpc): { code: number; headers: {}; data: string } {
         body = RequestDownloadResponse();
         break;
       default:
-        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
+        throw new Error(
+          `Unknown method response type ${
+            (rpc.acsResponse as AcsResponse).name
+          }`,
+        );
     }
   } else if (rpc.acsRequest) {
     switch (rpc.acsRequest.name) {
@@ -690,9 +1022,15 @@ export function response(rpc): { code: number; headers: {}; data: string } {
       case "GetParameterValues":
         body = GetParameterValues(rpc.acsRequest);
         break;
+      case "GetParameterAttributes":
+        body = GetParameterAttributes(rpc.acsRequest);
+        break;
       case "SetParameterValues":
         body = SetParameterValues(rpc.acsRequest);
         break;
+      case "SetParameterAttributes":
+        body = SetParameterAttributes(rpc.acsRequest);
+        break;
       case "AddObject":
         body = AddObject(rpc.acsRequest);
         break;
@@ -709,8 +1047,12 @@ export function response(rpc): { code: number; headers: {}; data: string } {
         body = Download(rpc.acsRequest);
         break;
       default:
-        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
+        throw new Error(
+          `Unknown method request ${(rpc.acsRequest as AcsRequest).name}`,
+        );
     }
+  } else if (rpc.acsFault) {
+    body = AcsFault(rpc.acsFault);
   }
 
   headers["Content-Type"] = 'text/xml; charset="utf-8"';
@@ -721,6 +1063,6 @@ export function response(rpc): { code: number; headers: {}; data: string } {
       namespacesAttrs[rpc.cwmpVersion]
     }><soap-env:Header><cwmp:ID soap-env:mustUnderstand="1">${
       rpc.id
-    }</cwmp:ID></soap-env:Header><soap-env:Body>${body}</soap-env:Body></soap-env:Envelope>`
+    }</cwmp:ID></soap-env:Header><soap-env:Body>${body}</soap-env:Body></soap-env:Envelope>`,
   };
 }

lib/types.ts

@@ -1,29 +1,10 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
-import PathSet from "./common/path-set";
-import VersionedMap from "./versioned-map";
-import InstanceSet from "./instance-set";
-import { IncomingMessage, ServerResponse } from "http";
-
-export type Expression = string | number | boolean | null | any[];
+import { IncomingMessage, ServerResponse } from "node:http";
+import { Script } from "node:vm";
+import Path from "./common/path.ts";
+import PathSet from "./common/path-set.ts";
+import VersionedMap from "./versioned-map.ts";
+import InstanceSet from "./instance-set.ts";
+import Expression, { Value } from "./common/expression.ts";
 
 export interface Fault {
   code: string;
@@ -51,18 +32,24 @@ export interface Attributes {
   object?: [number, 1 | 0];
   writable?: [number, 1 | 0];
   value?: [number, [string | number | boolean, string]];
+  notification?: [number, number];
+  accessList?: [number, string[]];
 }
 
 export interface AttributeTimestamps {
   object?: number;
   writable?: number;
   value?: number;
+  notification?: number;
+  accessList?: number;
 }
 
 export interface AttributeValues {
   object?: boolean;
   writable?: boolean;
   value?: [string | number | boolean, string?];
+  notification?: number;
+  accessList?: string[];
 }
 
 export interface DeviceData {
@@ -75,13 +62,22 @@ export interface DeviceData {
 
 export type VirtualParameterDeclaration = [
   Path,
-  { path?: number; object?: number; writable?: number; value?: number }?,
+  {
+    path?: number;
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  }?,
   {
     path?: [number, number];
     object?: boolean;
     writable?: boolean;
     value?: [string | number | boolean, string?];
-  }?
+    notification?: number;
+    accessList?: string[];
+  }?,
 ];
 
 export interface SyncState {
@@ -90,8 +86,11 @@ export interface SyncState {
     object: Set<Path>;
     writable: Set<Path>;
     value: Set<Path>;
+    notification: Set<Path>;
+    accessList: Set<Path>;
   };
   spv: Map<Path, [string | number | boolean, string]>;
+  spa: Map<Path, { notification: number; accessList: string[] }>;
   gpn: Set<Path>;
   gpnPatterns: Map<Path, number>;
   tags: Map<Path, boolean>;
@@ -120,7 +119,7 @@ export interface SessionContext {
     AttributeTimestamps,
     AttributeValues,
     AttributeTimestamps,
-    AttributeValues
+    AttributeValues,
   ][][];
   revisions: number[];
   rpcCount: number;
@@ -131,7 +130,6 @@ export interface SessionContext {
   faults?: { [channel: string]: SessionFault };
   retries?: { [channel: string]: number };
   cacheSnapshot?: string;
-  configContext?: { [name: string]: any };
   httpResponse?: ServerResponse;
   httpRequest?: IncomingMessage;
   faultsTouched?: { [channel: string]: boolean };
@@ -139,11 +137,12 @@ export interface SessionContext {
   new?: boolean;
   debug?: boolean;
   state: number;
+  authState: number;
   tasks?: Task[];
   operations?: { [commandKey: string]: Operation };
-  cacheUntil?: number;
   syncState?: SyncState;
   lastActivity?: number;
+  extendLock?: number;
   rpcRequest?: AcsRequest;
   operationsTouched?: { [commandKey: string]: 1 | 0 };
   provisionsRet?: any[];
@@ -151,7 +150,7 @@ export interface SessionContext {
 }
 
 export interface Task {
-  _id: string;
+  _id?: string;
   name: string;
   parameterNames?: string[];
   parameterValues?: [string, string | number | boolean, string?][];
@@ -160,6 +159,7 @@ export interface Task {
   fileName?: string;
   targetFileName?: string;
   expiry?: number;
+  provisions?: [string, ...Value[]][];
 }
 
 export interface Operation {
@@ -176,44 +176,84 @@ export interface Operation {
   };
 }
 
-export interface AcsRequest {
-  name: string;
-  next?: string;
+export type AcsRequest =
+  | GetParameterNames
+  | GetParameterValues
+  | GetParameterAttributes
+  | SetParameterValues
+  | SetParameterAttributes
+  | AddObject
+  | DeleteObject
+  | FactoryReset
+  | Reboot
+  | Download;
+
+export interface GetParameterNames {
+  name: "GetParameterNames";
+  parameterPath: string;
+  nextLevel: boolean;
 }
 
-export interface GetAcsRequest extends AcsRequest {
-  name: "GetParameterNames" | "GetParameterValues";
-  objectName?: string;
-  parameterNames?: string[];
-  parameterPath?: string;
-  nextLevel?: boolean;
-  instanceValues?: { [name: string]: string };
+export interface GetParameterValues {
+  name: "GetParameterValues";
+  parameterNames: string[];
 }
 
-export interface SetAcsRequest extends AcsRequest {
-  name:
-    | "SetParameterValues"
-    | "AddObject"
-    | "DeleteObject"
-    | "FactoryReset"
-    | "Reboot"
-    | "Download";
-  parameterList?: [string, string | number | boolean, string][];
-  instanceValues?: { [param: string]: string | number | boolean };
-  objectName?: string;
+export interface GetParameterAttributes {
+  name: "GetParameterAttributes";
+  parameterNames: string[];
+}
+
+export interface SetParameterValues {
+  name: "SetParameterValues";
+  parameterList: [string, boolean | number | string, string][];
+  parameterKey?: string;
   DATETIME_MILLISECONDS?: boolean;
   BOOLEAN_LITERAL?: boolean;
-  commandKey?: string;
-  instance?: string;
-  fileType?: string;
-  fileSize?: number;
-  url?: string;
-  fileName?: string;
-  targetFileName?: string;
 }
 
-export interface CpeResponse {
-  name: string;
+export interface SetParameterAttributes {
+  name: "SetParameterAttributes";
+  parameterList: [string, number, string[]][];
+}
+
+export interface AddObject {
+  name: "AddObject";
+  objectName: string;
+  parameterKey?: string;
+  instanceValues: Record<string, string>;
+}
+
+export interface DeleteObject {
+  name: "DeleteObject";
+  objectName: string;
+  parameterKey?: string;
+}
+
+export interface FactoryReset {
+  name: "FactoryReset";
+  commandKey?: string;
+}
+
+export interface Reboot {
+  name: "Reboot";
+  commandKey?: string;
+}
+
+export interface Download {
+  name: "Download";
+  commandKey: string;
+  instance: string;
+  fileType: string;
+  fileName?: string;
+  url?: string;
+  username?: string;
+  password?: string;
+  fileSize?: number;
+  targetFileName?: string;
+  delaySecods?: number;
+  successUrl?: string;
+  failureUrl?: string;
 }
 
 export interface SpvFault {
@@ -229,38 +269,80 @@ export interface FaultStruct {
 }
 
 export interface CpeFault {
-  faultCode: string;
-  faultString: string;
+  faultCode: "Client" | "Server";
+  faultString: "CWMP fault";
   detail?: FaultStruct;
 }
 
-export interface CpeGetResponse extends CpeResponse {
-  name: "GetParameterNamesResponse" | "GetParameterValuesResponse";
-  parameterList?:
-    | [string, boolean][]
-    | [string, string | number | boolean, string][];
+export type CpeResponse =
+  | GetParameterNamesResponse
+  | GetParameterValuesResponse
+  | GetParameterAttributesResponse
+  | SetParameterValuesResponse
+  | SetParameterAttributesResponse
+  | AddObjectResponse
+  | DeleteObjectResponse
+  | RebootResponse
+  | FactoryResetResponse
+  | DownloadResponse;
+
+export interface GetParameterNamesResponse {
+  name: "GetParameterNamesResponse";
+  parameterList: [Path, boolean, boolean][];
 }
 
-export interface CpeSetResponse extends CpeResponse {
-  name:
-    | "SetParameterValuesResponse"
-    | "AddObjectResponse"
-    | "DeleteObjectResponse"
-    | "RebootResponse"
-    | "FactoryResetResponse"
-    | "DownloadResponse";
-  status?: number;
-  instanceNumber?: string;
+export interface GetParameterValuesResponse {
+  name: "GetParameterValuesResponse";
+  parameterList: [Path, string | number | boolean, string][];
+}
+
+export interface GetParameterAttributesResponse {
+  name: "GetParameterAttributesResponse";
+  parameterList: [Path, number, string[]][];
+}
+
+export interface SetParameterValuesResponse {
+  name: "SetParameterValuesResponse";
+  status: number;
+}
+
+export interface SetParameterAttributesResponse {
+  name: "SetParameterAttributesResponse";
+}
+
+export interface AddObjectResponse {
+  name: "AddObjectResponse";
+  instanceNumber: string;
+  status: number;
+}
+
+export interface DeleteObjectResponse {
+  name: "DeleteObjectResponse";
+  status: number;
+}
+
+export interface RebootResponse {
+  name: "RebootResponse";
+}
+
+export interface FactoryResetResponse {
+  name: "FactoryResetResponse";
+}
+
+export interface DownloadResponse {
+  name: "DownloadResponse";
+  status: number;
   startTime?: number;
   completeTime?: number;
 }
 
-export interface CpeRequest {
-  name: string;
-  fileType?: string;
-}
+export type CpeRequest =
+  | InformRequest
+  | TransferCompleteRequest
+  | GetRPCMethodsRequest
+  | RequestDownloadRequest;
 
-export interface InformRequest extends CpeRequest {
+export interface InformRequest {
   name: "Inform";
   deviceId: {
     Manufacturer: string;
@@ -270,10 +352,10 @@ export interface InformRequest extends CpeRequest {
   };
   event: string[];
   retryCount: number;
-  parameterList: [string, string | number | boolean, string][];
+  parameterList: [Path, string | number | boolean, string][];
 }
 
-export interface TransferCompleteRequest extends CpeRequest {
+export interface TransferCompleteRequest {
   name: "TransferComplete";
   commandKey?: string;
   faultStruct?: FaultStruct;
@@ -281,10 +363,36 @@ export interface TransferCompleteRequest extends CpeRequest {
   completeTime?: number;
 }
 
-export interface AcsResponse {
-  name: string;
-  commandKey?: string;
-  faultStruct?: FaultStruct;
+export interface GetRPCMethodsRequest {
+  name: "GetRPCMethods";
+}
+
+export interface RequestDownloadRequest {
+  name: "RequestDownload";
+  fileType: string;
+}
+
+export type AcsResponse =
+  | InformResponse
+  | GetRPCMethodsResponse
+  | TransferCompleteResponse
+  | RequestDownloadResponse;
+
+export interface InformResponse {
+  name: "InformResponse";
+}
+
+export interface GetRPCMethodsResponse {
+  name: "GetRPCMethodsResponse";
+  methodList: string[];
+}
+
+export interface TransferCompleteResponse {
+  name: "TransferCompleteResponse";
+}
+
+export interface RequestDownloadResponse {
+  name: "RequestDownloadResponse";
 }
 
 export interface QueryOptions {
@@ -299,12 +407,20 @@ export interface QueryOptions {
 export interface Declaration {
   path: Path;
   pathGet: number;
-  pathSet?: [number, number];
-  attrGet?: { object?: number; writable?: number; value?: number };
+  pathSet?: number | [number, number];
+  attrGet?: {
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  };
   attrSet?: {
     object?: boolean;
     writable?: boolean;
     value?: [string | number | boolean, string?];
+    notification?: number;
+    accessList?: string[];
   };
   defer: boolean;
 }
@@ -312,8 +428,14 @@ export interface Declaration {
 export type Clear = [
   Path,
   number,
-  { object?: number; writable?: number; value?: number }?,
-  number?
+  {
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  }?,
+  number?,
 ];
 
 export interface Preset {
@@ -321,18 +443,52 @@ export interface Preset {
   channel: string;
   schedule?: { md5: string; duration: number; schedule: any };
   events?: { [event: string]: boolean };
-  precondition?: {};
-  provisions: string[][];
+  precondition?: Expression;
+  provisions: [string, ...Expression[]][];
+}
+
+export interface Provisions {
+  [name: string]: { md5: string; script: Script };
+}
+
+export interface VirtualParameters {
+  [name: string]: { md5: string; script: Script };
+}
+
+export interface Files {
+  [name: string]: { length: number };
+}
+
+export interface Users {
+  [name: string]: { password: string; salt: string; roles: string[] };
+}
+
+export interface Permissions {
+  [role: string]: {
+    [access: number]: {
+      [resource: string]: {
+        access: number;
+        filter: Expression;
+        validate: Expression;
+      };
+    };
+  };
 }
 
 export type PermissionSet = {
   [resource: string]: {
     access: number;
-    validate?: Expression;
+    validate: Expression;
     filter: Expression;
   };
 }[];
 
+export interface Config {
+  [name: string]: Expression;
+}
+
+export type UiConfig = Record<string, string>;
+
 export interface SoapMessage {
   id: string;
   cwmpVersion: string;
@@ -340,6 +496,7 @@ export interface SoapMessage {
   cpeRequest?: CpeRequest;
   cpeFault?: CpeFault;
   cpeResponse?: CpeResponse;
+  unknownMethod?: string;
 }
 
 export interface ScriptResult {

lib/ui.ts

@@ -1,45 +1,22 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { Z_SYNC_FLUSH } from "zlib";
+import { constants } from "node:zlib";
 import Koa from "koa";
-import Router from "koa-router";
+import Router from "@koa/router";
 import * as jwt from "jsonwebtoken";
-import koaStatic from "koa-static";
+import koaSend from "koa-send";
 import koaCompress from "koa-compress";
-import koaBodyParser from "koa-bodyparser";
+import koaBodyParser from "@koa/bodyparser";
 import koaJwt from "koa-jwt";
-import * as config from "./config";
-import api from "./ui/api";
-import Authorizer from "./common/authorizer";
-import * as logger from "./logger";
-import * as localCache from "./local-cache";
-import { PermissionSet } from "./types";
-import { authSimple } from "./ui/api-functions";
-import * as init from "./init";
+import * as config from "./config.ts";
+import api from "./ui/api.ts";
+import Authorizer from "./common/authorizer.ts";
+import * as logger from "./logger.ts";
+import * as localCache from "./ui/local-cache.ts";
+import { PermissionSet } from "./types.ts";
+import { authLocal } from "./api-functions.ts";
+import * as init from "./init.ts";
 import { version as VERSION } from "../package.json";
-
-declare module "koa" {
-  interface Request {
-    body: any;
-  }
-}
+import memoize from "./common/memoize.ts";
+import { APP_JS, APP_CSS, FAVICON_PNG } from "../build/assets.ts";
 
 const koa = new Koa();
 const router = new Router();
@@ -47,22 +24,31 @@ const router = new Router();
 const JWT_SECRET = "" + config.get("UI_JWT_SECRET");
 const JWT_COOKIE = "genieacs-ui-jwt";
 
-function getPermissionSets(ctx): PermissionSet[] {
-  const allPermissions = localCache.getPermissions(ctx.state.configSnapshot);
-  const users = localCache.getUsers(ctx.state.configSnapshot);
-  const user = users[ctx.state.user.username];
-  const permissionSets = user.roles.map(role =>
-    Object.values(allPermissions[role] || {})
-  );
-  return permissionSets;
-}
+const getAuthorizer = memoize(
+  (snapshot: string, rolesStr: string): Authorizer => {
+    const roles: string[] = JSON.parse(rolesStr);
+    const allPermissions = localCache.getPermissions(snapshot);
+    const permissionSets: PermissionSet[] = roles.map((r) =>
+      Object.values(allPermissions[r] || {}),
+    );
+    return new Authorizer(permissionSets);
+  },
+);
 
-koa.on("error", async err => {
-  throw err;
+koa.on("error", (err, ctx) => {
+  setTimeout(() => {
+    // Ignored errors resulting from aborted requests
+    if (ctx?.req.aborted) return;
+
+    // Ignore client errors (e.g. malicious path)
+    if (err.status === 400) return;
+
+    throw err;
+  });
 });
 
 koa.use(async (ctx, next) => {
-  const configSnapshot = await localCache.getCurrentSnapshot();
+  const configSnapshot = await localCache.getRevision();
   ctx.state.configSnapshot = configSnapshot;
   ctx.set("X-Config-Snapshot", configSnapshot);
   ctx.set("GenieACS-Version", VERSION);
@@ -73,19 +59,43 @@ koa.use(
   koaJwt({
     secret: JWT_SECRET,
     passthrough: true,
-    cookie: JWT_COOKIE
-  })
+    cookie: JWT_COOKIE,
+    isRevoked: async (ctx, token) => {
+      if (token["authMethod"] === "local") {
+        return !localCache.getUsers(ctx.state.configSnapshot)[
+          token["username"]
+        ];
+      }
+
+      return true;
+    },
+  }),
 );
 
 koa.use(async (ctx, next) => {
-  if (ctx.state.user && ctx.state.user.username)
-    ctx.state.authorizer = new Authorizer(getPermissionSets(ctx));
-  else ctx.state.authorizer = new Authorizer([]);
+  let roles: string[] = [];
+
+  if (ctx.state.user?.username) {
+    let user;
+    if (ctx.state.user.authMethod === "local") {
+      user = localCache.getUsers(ctx.state.configSnapshot)[
+        ctx.state.user.username
+      ];
+    } else {
+      throw new Error("Invalid auth method");
+    }
+    roles = user.roles || [];
+  }
+
+  ctx.state.authorizer = getAuthorizer(
+    ctx.state.configSnapshot,
+    JSON.stringify(roles),
+  );
 
   return next();
 });
 
-router.post("/login", async ctx => {
+router.post("/login", async (ctx) => {
   if (!JWT_SECRET) {
     ctx.status = 500;
     ctx.body = "UI_JWT_SECRET is not set";
@@ -95,18 +105,27 @@ router.post("/login", async ctx => {
 
   const username = ctx.request.body.username;
   const password = ctx.request.body.password;
+  const remember = ctx.request.body.remember;
+  const TWO_WEEKS_SECS = 1209600;
+  const ONE_DAY_SECS = 86400;
 
   const log = {
     message: "Log in",
     context: ctx,
     username: username,
-    method: null
+    method: null,
   };
 
-  function success(method): void {
-    log.method = method;
-    const token = jwt.sign({ username }, JWT_SECRET);
-    ctx.cookies.set(JWT_COOKIE, token, { sameSite: "lax" });
+  function success(authMethod): void {
+    log.method = authMethod;
+    const expiresIn = remember ? TWO_WEEKS_SECS : ONE_DAY_SECS;
+    const token = jwt.sign({ username, authMethod }, JWT_SECRET, {
+      expiresIn,
+    });
+    ctx.cookies.set(JWT_COOKIE, token, {
+      sameSite: "lax",
+      maxAge: remember ? expiresIn * 1000 : null,
+    });
     ctx.body = JSON.stringify(token);
     logger.accessInfo(log);
   }
@@ -118,20 +137,19 @@ router.post("/login", async ctx => {
     logger.accessWarn(log);
   }
 
-  const roles = await authSimple(ctx.state.configSnapshot, username, password);
-
-  if (roles) return void success("simple");
+  if (await authLocal(ctx.state.configSnapshot, username, password))
+    return void success("local");
 
   failure();
 });
 
-router.post("/logout", async ctx => {
+router.post("/logout", async (ctx) => {
   ctx.cookies.set(JWT_COOKIE); // Delete cookie
   ctx.body = "";
 
   logger.accessInfo({
     message: "Log out",
-    context: ctx
+    context: ctx,
   });
 });
 
@@ -145,11 +163,11 @@ koa.use(async (ctx, next) => {
 koa.use(koaBodyParser());
 router.use("/api", api.routes(), api.allowedMethods());
 
-router.get("/status", ctx => {
+router.get("/status", (ctx) => {
   ctx.body = "OK";
 });
 
-router.get("/init", async ctx => {
+router.get("/init", async (ctx) => {
   const status = await init.getStatus();
   if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
     if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
@@ -170,7 +188,7 @@ router.get("/init", async ctx => {
   ctx.body = status;
 });
 
-router.post("/init", async ctx => {
+router.post("/init", async (ctx) => {
   const status = ctx.request.body;
   if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
     if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
@@ -191,36 +209,46 @@ router.post("/init", async ctx => {
   ctx.body = "";
 });
 
-router.get("/", async ctx => {
-  let permissionSets = [];
-  if (ctx.state.user && ctx.state.user.username)
-    permissionSets = getPermissionSets(ctx);
+router.get("/", async (ctx) => {
+  // koa-router seems to tolerate double slashes in the URL but that can
+  // be problematic when using relatives asset paths in HTML
+  if (ctx.path.endsWith("//")) return;
+
+  const ps: PermissionSet[] = ctx.state.authorizer.getPermissionSets();
+  const permissionSets = ps.map((p) =>
+    p.map((s) =>
+      Object.fromEntries(
+        Object.entries(s).map(([resource, { access, validate, filter }]) => [
+          resource,
+          { access, validate: validate.toString(), filter: filter.toString() },
+        ]),
+      ),
+    ),
+  );
 
   let wizard = "";
   if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
     wizard = '<script>window.location.hash = "#!/wizard";</script>';
 
-  ctx.body = `
+  ctx.body = `<!DOCTYPE html>
   <html>
     <head>
       <title>GenieACS</title>
-      <link rel="shortcut icon" type="image/png" href="favicon.png" />
-      <link rel="stylesheet" href="app.css">
+      <link rel="shortcut icon" type="image/png" href="${FAVICON_PNG}" />
+      <link rel="stylesheet" href="${APP_CSS}">
     </head>
-    <body>
+    <body class="h-full bg-stone-100">
     <noscript>GenieACS UI requires JavaScript to work. Please enable JavaScript in your browser.</noscript>
       <script>
-        window.clientConfig = ${JSON.stringify({
-          ui: localCache.getUiConfig(ctx.state.configSnapshot)
-        })};
+        window.clientConfig = ${JSON.stringify(localCache.getUiConfig(ctx.state.configSnapshot))};
         window.configSnapshot = ${JSON.stringify(ctx.state.configSnapshot)};
         window.genieacsVersion = ${JSON.stringify(VERSION)};
         window.username = ${JSON.stringify(
-          ctx.state.user ? ctx.state.user.username : ""
+          ctx.state.user ? ctx.state.user.username : "",
         )};
         window.permissionSets = ${JSON.stringify(permissionSets)};
       </script>
-      <script src="app.js"></script>${wizard} 
+      <script type="module" src="${APP_JS}"></script>${wizard} 
     </body>
   </html>
   `;
@@ -228,11 +256,33 @@ router.get("/", async ctx => {
 
 koa.use(
   koaCompress({
-    flush: Z_SYNC_FLUSH
-  })
+    gzip: {
+      flush: constants.Z_SYNC_FLUSH,
+    },
+    deflate: {
+      flush: constants.Z_SYNC_FLUSH,
+    },
+    br: {
+      flush: constants.BROTLI_OPERATION_FLUSH,
+      params: {
+        [constants.BROTLI_PARAM_QUALITY]: 5,
+      },
+    },
+  }),
 );
 
 koa.use(router.routes());
-koa.use(koaStatic(config.ROOT_DIR + "/public"));
+koa.use(async (ctx, next) => {
+  await next();
+  if (ctx.method !== "HEAD" && ctx.method !== "GET") return;
+  if (ctx.body != null || ctx.status !== 404) return;
+  if (/(?:^|[\\/])\.\.(?:[\\/]|$)/.test(ctx.path)) return;
+
+  try {
+    await koaSend(ctx, ctx.path, { root: config.ROOT_DIR + "/public" });
+  } catch (err) {
+    if (err.status !== 404) throw err;
+  }
+});
 
 export const listener = koa.callback();

lib/ui/api-functions.ts

@@ -1,207 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { exec } from "child_process";
-import { ObjectID } from "mongodb";
-import * as db from "./db";
-import { del } from "../cache";
-import { getUsers } from "../local-cache";
-import { hashPassword } from "../auth";
-import {
-  insertTasks,
-  watchTask,
-  connectionRequest,
-  deleteDevice
-} from "../api-functions";
-
-async function deleteFault(id): Promise<void> {
-  const deviceId = id.split(":", 1)[0];
-  const channel = id.slice(deviceId.length + 1);
-
-  await Promise.all([
-    db.deleteFault(id),
-    channel.startsWith("task_")
-      ? db.deleteTask(new ObjectID(channel.slice(5)))
-      : null
-  ]);
-
-  await del(`${deviceId}_tasks_faults_operations`);
-}
-
-export async function deleteResource(resource, id): Promise<void> {
-  switch (resource) {
-    case "devices":
-      await deleteDevice(id);
-      break;
-
-    case "files":
-      await db.deleteFile(id);
-      break;
-
-    case "faults":
-      await deleteFault(id);
-      break;
-
-    case "provisions":
-      await db.deleteProvision(id);
-      break;
-
-    case "presets":
-      await db.deletePreset(id);
-      break;
-
-    case "virtualParameters":
-      await db.deleteVirtualParameter(id);
-      break;
-
-    case "config":
-      await db.deleteConfig(id);
-      break;
-
-    case "permissions":
-      await db.deletePermission(id);
-      break;
-
-    case "users":
-      await db.deleteUser(id);
-      break;
-  }
-
-  await del("presets_hash");
-}
-
-export async function postTasks(
-  deviceId,
-  tasks,
-  timeout
-): Promise<{ connectionRequest: string; tasks: any[] }> {
-  for (const task of tasks) {
-    delete task._id;
-    task.device = deviceId;
-  }
-
-  tasks = await insertTasks(tasks);
-  const statuses = tasks.map(t => {
-    return { _id: t._id, status: "pending" };
-  });
-
-  await del(`${deviceId}_tasks_faults_operations`);
-
-  try {
-    await connectionRequest(deviceId);
-  } catch (err) {
-    return {
-      connectionRequest: err.message,
-      tasks: statuses
-    };
-  }
-
-  const sample = tasks[tasks.length - 1];
-
-  // Waiting for session to finish or timeout
-  await watchTask(deviceId, sample._id, timeout);
-
-  const promises = [];
-  for (const s of statuses) {
-    promises.push(db.query("tasks", ["=", ["PARAM", "_id"], s._id]));
-    promises.push(
-      db.query("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
-    );
-  }
-
-  const res = await Promise.all(promises);
-  for (const [i, r] of statuses.entries()) {
-    if (res[i * 2].length === 0) {
-      r.status = "done";
-    } else if (res[i * 2 + 1].length === 1) {
-      r.status = "fault";
-      r.fault = res[i * 2 + 1][0];
-    }
-    db.deleteTask(r._id);
-  }
-
-  return { connectionRequest: "OK", tasks: statuses };
-}
-
-interface PingResponse {
-  packetsTransmitted: number;
-  packetsReceived: number;
-  packetLoss: string;
-  min: number;
-  avg: number;
-  max: number;
-  mdev: number;
-}
-
-export function ping(host): Promise<PingResponse> {
-  return new Promise((resolve, reject) => {
-    exec(`ping -w 1 -i 0.2 -c 3 ${host}`, (err, stdout) => {
-      if (err) return void reject(err);
-
-      const m = stdout.match(
-        /(\d) packets transmitted, (\d) received, ([\d.%]+) packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/
-      );
-      if (!m) return void reject(new Error("Could not parse ping response"));
-
-      resolve({
-        packetsTransmitted: +m[1],
-        packetsReceived: +m[2],
-        packetLoss: m[3],
-        min: +m[4],
-        avg: +m[5],
-        max: +m[6],
-        mdev: +m[7]
-      });
-    });
-  });
-}
-
-export async function putResource(resource, id, data): Promise<void> {
-  if (resource === "presets") {
-    await db.putPreset(id, data);
-  } else if (resource === "provisions") {
-    await db.putProvision(id, data);
-  } else if (resource === "virtualParameters") {
-    await db.putVirtualParameter(id, data);
-  } else if (resource === "config") {
-    await db.putConfig(id, data);
-  } else if (resource === "permissions") {
-    await db.putPermission(id, data);
-  } else if (resource === "users") {
-    delete data.password;
-    delete data.salt;
-    await db.putUser(id, data);
-  }
-
-  await del("presets_hash");
-}
-
-export function authSimple(snapshot, username, password): Promise<string[]> {
-  return new Promise((resolve, reject) => {
-    const users = getUsers(snapshot);
-    const user = users[username];
-    if (!user || !user.password) return void resolve(null);
-    hashPassword(password, user.salt)
-      .then(hash => {
-        if (hash === user.password) resolve(user.roles);
-        else resolve(null);
-      })
-      .catch(reject);
-  });
-}

lib/ui/api.ts

@@ -1,33 +1,22 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as stream from "stream";
-import Router from "koa-router";
-import * as db from "./db";
-import * as apiFunctions from "./api-functions";
-import { evaluate, and, extractParams } from "../common/expression";
-import { parse } from "../common/expression-parser";
-import * as logger from "../logger";
-import { getConfig } from "../local-cache";
-import { QueryOptions, Expression } from "../types";
-import { generateSalt, hashPassword } from "../auth";
-import { del } from "../cache";
+import { Readable } from "node:stream";
+import Router from "@koa/router";
+import { ObjectId } from "mongodb";
+import * as db from "./db.ts";
+import * as apiFunctions from "../api-functions.ts";
+import Expression, { extractPaths } from "../common/expression.ts";
+import Path from "../common/path.ts";
+import * as logger from "../logger.ts";
+import { getConfig } from "../ui/local-cache.ts";
+import { Task } from "../types.ts";
+import { generateSalt, hashPassword } from "../auth.ts";
+import { del } from "../cache.ts";
+import Authorizer from "../common/authorizer.ts";
+import { ping } from "../ping.ts";
+import { decodeTag } from "../util.ts";
+import { stringify as yamlStringify } from "../common/yaml.ts";
+import { ResourceLockedError } from "../common/errors.ts";
+import { acquireLock, releaseLock } from "../lock.ts";
+import { collections } from "../db/db.ts";
 
 const router = new Router();
 export default router;
@@ -50,7 +39,7 @@ const RESOURCE_IDS = {
   permissions: "_id",
   users: "_id",
   faults: "_id",
-  tasks: "_id"
+  tasks: "_id",
 };
 
 const resources = {
@@ -63,79 +52,131 @@ const resources = {
   permissions: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   users: 0 | RESOURCE_DELETE | RESOURCE_PUT,
   faults: 0 | RESOURCE_DELETE,
-  tasks: 0
+  tasks: 0,
 };
 
-router.get(`/devices/:id.csv`, async ctx => {
+function singleParam(p: string | string[]): string {
+  return Array.isArray(p) ? p[p.length - 1] : p;
+}
+
+router.get(`/devices/:id.csv`, async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Query device (CSV)",
     context: ctx,
-    id: ctx.params.id
+    id: ctx.params.id,
   };
 
-  const filter = ["=", ["PARAM", RESOURCE_IDS.devices], ctx.params.id];
+  const filter = Expression.and(
+    authorizer.getFilter("devices", 2),
+    new Expression.Binary(
+      "=",
+      new Expression.Parameter(Path.parse(RESOURCE_IDS.devices)),
+      new Expression.Literal(ctx.params.id),
+    ),
+  );
 
-  if (!ctx.state.authorizer.hasAccess("devices", 2)) {
+  if (!authorizer.hasAccess("devices", 2)) {
     logUnauthorizedWarning(log);
-    return void (ctx.status = 404);
+    return void (ctx.status = 403);
   }
 
-  const res = await db.query("devices", filter);
-  if (!res[0]) return void (ctx.status = 404);
+  const { value: device } = await db.query("devices", filter).next();
+  if (!device) return void (ctx.status = 404);
 
   ctx.type = "text/csv";
   ctx.attachment(
     `device-${ctx.params.id}-${new Date()
       .toISOString()
-      .replace(/[:.]/g, "")}.csv`
+      .replace(/[:.]/g, "")}.csv`,
   );
 
-  ctx.body = new stream.PassThrough();
-  ctx.body.write(
-    "Parameter,Object,Object timestamp,Writable,Writable timestamp,Value,Value type,Value timestamp\n"
-  );
+  const lines: string[] = [
+    "Parameter,Object,Writable,Value,Value type,Timestamp,Value timestamp,Notification,Access list,Attributes timestamp",
+  ];
+
+  const keys = Object.keys(device).sort();
+  let prevParam = "";
+  let attrs: Record<string, string | number | boolean | null> = {};
+
+  function flushRow(): void {
+    if (!prevParam) return;
+    let value: string | number | boolean | null = attrs["value"] ?? "";
+    if (attrs["type"] === "xsd:dateTime" && typeof value === "number")
+      value = new Date(value).toJSON();
 
-  for (const k of Object.keys(res[0]).sort()) {
-    const p = res[0][k];
     const row = [
-      k,
-      p.object,
-      p.objectTimestamp,
-      p.writable,
-      p.writableTimestamp,
-      p.value != null ? `"${p.value[0].toString().replace(/"/g, '""')}"` : "",
-      p.value != null ? p.value[1] : "",
-      p.valueTimestamp
+      prevParam,
+      attrs["object"] ?? "",
+      attrs["writable"] ?? "",
+      `"${String(value).replace(/"/g, '""')}"`,
+      attrs["type"] ?? "",
+      attrs["timestamp"] != null ? new Date(+attrs["timestamp"]).toJSON() : "",
+      attrs["valueTimestamp"] != null
+        ? new Date(+attrs["valueTimestamp"]).toJSON()
+        : "",
+      attrs["notification"] ?? "",
+      attrs["accessList"] ?? "",
+      attrs["attributesTimestamp"] != null
+        ? new Date(+attrs["attributesTimestamp"]).toJSON()
+        : "",
     ];
-    ctx.body.write(row.map(r => (r != null ? r : "")).join(",") + "\n");
+    lines.push(row.map((r) => (r != null ? r : "")).join(","));
   }
-  ctx.body.end();
+
+  for (const k of keys) {
+    const colonIdx = k.lastIndexOf(":");
+    const param = colonIdx === -1 ? k : k.slice(0, colonIdx);
+    const attr = colonIdx === -1 ? "value" : k.slice(colonIdx + 1);
+
+    if (param !== prevParam) {
+      flushRow();
+      prevParam = param;
+      attrs = {};
+    }
+    attrs[attr] = device[k];
+  }
+  flushRow();
+  ctx.body = lines.join("\n");
   logger.accessInfo(log);
 });
 
 for (const [resource, flags] of Object.entries(resources)) {
-  router.head(`/${resource}`, async (ctx, next) => {
-    let filter: Expression = true;
-    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+  router.head(`/${resource}`, async (ctx) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
+    let filter: Expression = authorizer.getFilter(resource, 1);
+    if (ctx.request.query.filter)
+      filter = Expression.and(
+        filter,
+        Expression.parse(singleParam(ctx.request.query.filter)),
+      );
 
     const log = {
       message: `Count ${resource}`,
       context: ctx,
       filter: ctx.request.query.filter,
-      count: null
+      count: null,
     };
 
-    if (!ctx.state.authorizer.hasAccess(resource, 1)) {
+    if (!authorizer.hasAccess(resource, 1)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
-      filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000]
-      ]);
+      const p = new Expression.Parameter(Path.parse("expiry"));
+      filter = Expression.and(
+        filter,
+        Expression.or(
+          new Expression.Binary(
+            ">=",
+            p,
+            new Expression.Literal(Date.now() + 60000),
+          ),
+          new Expression.Unary("IS NULL", p),
+        ),
+      );
     }
 
     const count = await db.count(resource, filter);
@@ -147,16 +188,21 @@ for (const [resource, flags] of Object.entries(resources)) {
     logger.accessInfo(log);
   });
 
-  router.get(`/${resource}`, async (ctx, next) => {
-    const options: QueryOptions = {};
-    let filter: Expression = true;
-    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+  router.get(`/${resource}`, async (ctx) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
+    const options: Parameters<typeof db.query>[2] = {};
+    let filter: Expression = authorizer.getFilter(resource, 2);
+    if (ctx.request.query.filter)
+      filter = Expression.and(
+        filter,
+        Expression.parse(singleParam(ctx.request.query.filter)),
+      );
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
-      options.sort = JSON.parse(ctx.request.query.sort);
+      options.sort = JSON.parse(singleParam(ctx.request.query.sort));
     if (ctx.request.query.projection) {
-      options.projection = ctx.request.query.projection
+      options.projection = singleParam(ctx.request.query.projection)
         .split(",")
         .reduce((obj, k) => Object.assign(obj, { [k]: 1 }), {});
     }
@@ -168,44 +214,57 @@ for (const [resource, flags] of Object.entries(resources)) {
       limit: options.limit,
       skip: options.skip,
       sort: options.sort,
-      projection: options.projection
+      projection: options.projection,
     };
 
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
-      filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000]
-      ]);
+      const p = new Expression.Parameter(Path.parse("expiry"));
+      filter = Expression.and(
+        filter,
+        Expression.or(
+          new Expression.Binary(
+            ">=",
+            p,
+            new Expression.Literal(Date.now() + 60000),
+          ),
+          new Expression.Unary("IS NULL", p),
+        ),
+      );
     }
 
-    ctx.body = new stream.PassThrough();
-    ctx.type = "application/json";
-
-    let c = 0;
-    ctx.body.write("[\n");
-    await db.query(resource, filter, options, obj => {
-      ctx.body.write((c++ ? "," : "") + JSON.stringify(obj) + "\n");
-    });
-    ctx.body.end("]");
-
     logger.accessInfo(log);
+    ctx.type = "application/json";
+    ctx.body = Readable.from(
+      (async function* () {
+        let c = 0;
+        yield "[\n";
+        for await (const obj of db.query(resource, filter, options))
+          yield (c++ ? "," : "") + JSON.stringify(obj) + "\n";
+        yield "]";
+      })(),
+    );
   });
 
   // CSV download
-  router.get(`/${resource}.csv`, async (ctx, next) => {
-    const options: QueryOptions = { projection: {} };
-    let filter: Expression = true;
-    if (ctx.request.query.filter) filter = parse(ctx.request.query.filter);
+  router.get(`/${resource}.csv`, async (ctx) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
+    const options: Parameters<typeof db.query>[2] = { projection: {} };
+    let filter: Expression = authorizer.getFilter(resource, 2);
+    if (ctx.request.query.filter)
+      filter = Expression.and(
+        filter,
+        Expression.parse(singleParam(ctx.request.query.filter)),
+      );
     if (ctx.request.query.limit) options.limit = +ctx.request.query.limit;
     if (ctx.request.query.skip) options.skip = +ctx.request.query.skip;
     if (ctx.request.query.sort)
-      options.sort = JSON.parse(ctx.request.query.sort);
+      options.sort = JSON.parse(singleParam(ctx.request.query.sort));
 
     const log = {
       message: `Query ${resource} (CSV)`,
@@ -213,144 +272,202 @@ for (const [resource, flags] of Object.entries(resources)) {
       filter: ctx.request.query.filter,
       limit: options.limit,
       skip: options.skip,
-      sort: options.sort
+      sort: options.sort,
     };
 
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
-    const columns = JSON.parse(ctx.request.query.columns);
+    const columnsStr: Record<string, string> = JSON.parse(
+      singleParam(ctx.request.query.columns),
+    );
+
     const now = Date.now();
-
-    for (const [k, v] of Object.entries(columns)) {
-      const e = evaluate(parse(v as string), null, now);
-      columns[k] = e;
-      for (const p of extractParams(e))
-        if (typeof p === "string") options.projection[p] = 1;
-    }
+    const columns: Record<string, Expression> = Object.fromEntries(
+      Object.entries(columnsStr).map(([k, v]) => {
+        let exp = Expression.parse(v);
+        exp = exp.evaluate((e) => {
+          if (e instanceof Expression.FunctionCall && e.name === "NOW")
+            return new Expression.Literal(now);
+          return e;
+        });
+        for (const p of extractPaths(exp)) options.projection[p.toString()] = 1;
+        return [k, exp];
+      }),
+    );
 
     // Exclude temporary tasks and faults
     if (resource === "tasks" || resource === "faults") {
-      filter = and(filter, [
-        "NOT",
-        ["<", ["PARAM", "expiry"], Date.now() + 60000]
-      ]);
+      const p = new Expression.Parameter(Path.parse("expiry"));
+      filter = Expression.and(
+        filter,
+        Expression.or(
+          new Expression.Binary(
+            ">=",
+            p,
+            new Expression.Literal(Date.now() + 60000),
+          ),
+          new Expression.Unary("IS NULL", p),
+        ),
+      );
     }
 
-    ctx.body = new stream.PassThrough();
+    logger.accessInfo(log);
     ctx.type = "text/csv";
     ctx.attachment(
-      `${resource}-${new Date(now).toISOString().replace(/[:.]/g, "")}.csv`
+      `${resource}-${new Date(now).toISOString().replace(/[:.]/g, "")}.csv`,
     );
 
-    ctx.body.write(
-      Object.keys(columns).map(k => `"${k.replace(/"/, '""')}"`) + "\n"
-    );
-    await db.query(resource, filter, options, obj => {
-      const arr = Object.values(columns).map(exp => {
-        const v = evaluate(exp, obj, null, e => {
-          if (Array.isArray(e)) {
-            if (e[0] === "PARAM") {
-              if (resource === "devices") {
-                if (e[1] === "Tags") {
-                  const tags = [];
-                  for (const p in obj)
-                    if (p.startsWith("Tags.")) tags.push(p.slice(5));
-
-                  return tags.join(", ");
+    ctx.body = Readable.from(
+      (async function* () {
+        yield Object.keys(columns).map((k) => `"${k.replace(/"/, '""')}"`) +
+          "\n";
+        for await (const obj of db.query(resource, filter, options)) {
+          const arr = Object.values(columns).map((exp) => {
+            return exp.evaluate((e) => {
+              if (e instanceof Expression.Literal) return e;
+              else if (e instanceof Expression.FunctionCall) {
+                if (e.name === "NOW") return new Expression.Literal(now);
+                if (e.name === "DATE_STRING") {
+                  if (e.args[0] instanceof Expression.Literal)
+                    return new Expression.Literal(
+                      new Date(e.args[0].value as number).toJSON(),
+                    );
                 }
+              } else if (e instanceof Expression.Parameter) {
+                let v = obj[e.path.toString()];
+                if (resource === "devices") {
+                  if (e.path.toString() === "Tags") {
+                    const tags = [];
+                    for (const p in obj)
+                      if (p.startsWith("Tags.") && p.lastIndexOf(":") === -1)
+                        tags.push(decodeTag(p.slice(5)));
+                    v = tags.join(", ");
+                  }
+                  if (e === exp) {
+                    const type = obj[e.path.toString() + ":type"];
+                    if (type === "xsd:dateTime" && typeof v === "number")
+                      v = new Date(v).toJSON();
+                  }
+                } else if (resource === "faults") {
+                  if (e.path.toString() === "detail") v = yamlStringify(v);
+                }
+
+                if (typeof v === "string") v = `"${v.replace(/"/g, '""')}"`;
+                if (v != null) return new Expression.Literal(v);
               }
-            } else if (e[0] === "FUNC") {
-              if (e[1] === "DATE_STRING") {
-                if (e[2] && !Array.isArray(e[2]))
-                  return new Date(e[2]).toISOString();
-              }
-            }
-          }
-
-          return e;
-        });
-
-        if (Array.isArray(v) || v == null) return "";
-        if (typeof v === "string") return `"${v.replace(/"/g, '""')}"`;
-        return v;
-      });
-      ctx.body.write(arr.join(",") + "\n");
-    });
-    ctx.body.end();
-
-    logger.accessInfo(log);
+              return new Expression.Literal(null);
+            }).value;
+          });
+          yield arr.join(",") + "\n";
+        }
+      })(),
+    );
   });
 
-  router.head(`/${resource}/:id`, async (ctx, next) => {
+  router.head(`/${resource}/:id`, async (ctx) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Count ${resource}`,
       context: ctx,
-      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`,
     };
 
-    const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+    const filter = Expression.and(
+      authorizer.getFilter(resource, 2),
+      new Expression.Binary(
+        "=",
+        new Expression.Parameter(Path.parse(RESOURCE_IDS[resource])),
+        new Expression.Literal(ctx.params.id),
+      ),
+    );
+
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
-    const res = await db.query(resource, filter);
+    const count = await db.count(resource, filter);
 
-    if (!res.length) return void next();
+    if (!count) return void (ctx.status = 404);
 
     logger.accessInfo(log);
     ctx.body = "";
   });
 
-  router.get(`/${resource}/:id`, async (ctx, next) => {
+  router.get(`/${resource}/:id`, async (ctx) => {
+    const authorizer: Authorizer = ctx.state.authorizer;
     const log = {
       message: `Query ${resource}`,
       context: ctx,
-      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`
+      filter: `${RESOURCE_IDS[resource]} = "${ctx.params.id}"`,
     };
 
-    const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
-    if (!ctx.state.authorizer.hasAccess(resource, 2)) {
+    const filter = Expression.and(
+      authorizer.getFilter(resource, 2),
+      new Expression.Binary(
+        "=",
+        new Expression.Parameter(Path.parse(RESOURCE_IDS[resource])),
+        new Expression.Literal(ctx.params.id),
+      ),
+    );
+    if (!authorizer.hasAccess(resource, 2)) {
       logUnauthorizedWarning(log);
-      return void next();
+      return void (ctx.status = 403);
     }
 
-    const res = await db.query(resource, filter);
+    const { value: res } = await db.query(resource, filter).next();
 
-    if (!res.length) return void next();
+    if (!res) return void (ctx.status = 404);
 
     logger.accessInfo(log);
-    ctx.body = res[0];
+    ctx.body = res;
   });
 
-  // TODO add PUT, PATCH routes
   if (flags & RESOURCE_DELETE) {
-    router.delete(`/${resource}/:id`, async (ctx, next) => {
+    router.delete(`/${resource}/:id`, async (ctx) => {
+      const authorizer: Authorizer = ctx.state.authorizer;
       const log = {
         message: `Delete ${resource}`,
         context: ctx,
-        id: ctx.params.id
+        id: ctx.params.id,
       };
 
-      const authorizer = ctx.state.authorizer;
-      const filter = ["=", ["PARAM", RESOURCE_IDS[resource]], ctx.params.id];
+      const filter = Expression.and(
+        authorizer.getFilter(resource, 3),
+        new Expression.Binary(
+          "=",
+          new Expression.Parameter(Path.parse(RESOURCE_IDS[resource])),
+          new Expression.Literal(ctx.params.id),
+        ),
+      );
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
-      const res = await db.query(resource, filter);
-      if (!res.length) return void next();
+      const { value: res } = await db.query(resource, filter).next();
+      if (!res) return void (ctx.status = 404);
 
-      const validate = authorizer.getValidator(resource, res[0]);
+      const validate = authorizer.getValidator(resource, res);
       if (!validate("delete")) {
         logUnauthorizedWarning(log);
         return void (ctx.status = 403);
       }
 
-      await apiFunctions.deleteResource(resource, ctx.params.id);
+      try {
+        await apiFunctions.deleteResource(resource, ctx.params.id);
+      } catch (err) {
+        if (err instanceof ResourceLockedError) {
+          log.message += " failed";
+          logger.accessWarn(log);
+          ctx.status = 503;
+          ctx.body = err.message;
+          return;
+        }
+        throw err;
+      }
 
       logger.accessInfo(log);
 
@@ -359,19 +476,19 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 
   if (flags & RESOURCE_PUT) {
-    router.put(`/${resource}/:id`, async (ctx, next) => {
+    router.put(`/${resource}/:id`, async (ctx) => {
+      const authorizer: Authorizer = ctx.state.authorizer;
       const id = ctx.params.id;
 
       const log = {
         message: `Put ${resource}`,
         context: ctx,
-        id: id
+        id: id,
       };
 
-      const authorizer = ctx.state.authorizer;
       if (!authorizer.hasAccess(resource, 3)) {
         logUnauthorizedWarning(log);
-        return void next();
+        return void (ctx.status = 403);
       }
 
       const obj = ctx.request.body;
@@ -398,7 +515,41 @@ for (const [resource, flags] of Object.entries(resources)) {
   }
 }
 
-router.put("/files/:id", async (ctx, next) => {
+router.get("/blob/files/:id", async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
+  const resource = "files";
+  const id = ctx.params.id;
+
+  const log = {
+    message: `Download ${resource}`,
+    context: ctx,
+    id: id,
+  };
+
+  const filter = Expression.and(
+    authorizer.getFilter(resource, 2),
+    new Expression.Binary(
+      "=",
+      new Expression.Parameter(Path.parse(RESOURCE_IDS[resource])),
+      new Expression.Literal(ctx.params.id),
+    ),
+  );
+
+  if (!authorizer.hasAccess(resource, 2)) {
+    logUnauthorizedWarning(log);
+    return void (ctx.status = 403);
+  }
+
+  const count = await db.count(resource, filter);
+  if (!count) return void (ctx.status = 404);
+
+  logger.accessInfo(log);
+  ctx.body = db.downloadFile(id);
+  ctx.attachment(id);
+});
+
+router.put("/files/:id", async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const resource = "files";
   const id = ctx.params.id;
 
@@ -406,20 +557,20 @@ router.put("/files/:id", async (ctx, next) => {
     message: `Upload ${resource}`,
     context: ctx,
     id: id,
-    metadata: null
+    metadata: null,
   };
 
-  const authorizer = ctx.state.authorizer;
   if (!authorizer.hasAccess(resource, 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
   const metadata = {
-    fileType: ctx.request.headers["metadata.filetype"] || "",
-    oui: ctx.request.headers["metadata.oui"] || "",
-    productClass: ctx.request.headers["metadata.productclass"] || "",
-    version: ctx.request.headers["metadata.version"] || ""
+    fileType: singleParam(ctx.request.headers["metadata-filetype"]) || "",
+    oui: singleParam(ctx.request.headers["metadata-oui"]) || "",
+    productClass:
+      singleParam(ctx.request.headers["metadata-productclass"]) || "",
+    version: singleParam(ctx.request.headers["metadata-version"]) || "",
   };
 
   const validate = authorizer.getValidator(resource, metadata);
@@ -428,6 +579,12 @@ router.put("/files/:id", async (ctx, next) => {
     return void (ctx.status = 403);
   }
 
+  try {
+    await db.deleteFile(id);
+  } catch {
+    // File doesn't exist, ignore
+  }
+
   await db.putFile(id, metadata, ctx.req);
   log.metadata = metadata;
   logger.accessInfo(log);
@@ -435,69 +592,163 @@ router.put("/files/:id", async (ctx, next) => {
   ctx.body = "";
 });
 
-router.post("/devices/:id/tasks", async (ctx, next) => {
+router.post("/devices/:id/tasks", async (ctx) => {
+  const deviceId = ctx.params.id;
+  const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Commit tasks",
     context: ctx,
-    deviceId: ctx.params.id,
-    tasks: null
+    deviceId: deviceId,
+    tasks: null,
   };
 
-  const authorizer = ctx.state.authorizer;
-  const filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
-  const devices = await db.query("devices", filter);
-  if (!devices.length) return void next();
 
-  const validate = authorizer.getValidator("devices", devices[0]);
-  for (const t of ctx.request.body) {
-    if (!validate("task", t)) {
-      logUnauthorizedWarning(log);
-      return void (ctx.status = 403);
+  const socketTimeout: number = ctx.socket.timeout;
+
+  // Extend socket timeout while waiting for session
+  if (socketTimeout) ctx.socket.setTimeout(300000);
+
+  const token = await acquireLock(`cwmp_session_${deviceId}`, 5000, 30000);
+  if (!token) {
+    log.message += " failed";
+    logger.accessWarn(log);
+    ctx.body = "Device is in session";
+    ctx.status = 503;
+    // Restore socket timeout
+    if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
+    return;
+  }
+
+  let device;
+
+  let statuses: { _id: string; status: string }[];
+
+  try {
+    const filter = Expression.and(
+      authorizer.getFilter("devices", 3),
+      new Expression.Binary(
+        "=",
+        new Expression.Parameter(Path.parse(RESOURCE_IDS["devices"])),
+        new Expression.Literal(ctx.params.id),
+      ),
+    );
+    device = (await db.query("devices", filter).next()).value;
+    if (!device) return void (ctx.status = 404);
+
+    const validate = authorizer.getValidator("devices", device);
+    for (const t of ctx.request.body) {
+      if (!validate("task", t)) {
+        logUnauthorizedWarning(log);
+        return void (ctx.status = 403);
+      }
+    }
+
+    let tasks = ctx.request.body as Task[];
+
+    for (const task of tasks) {
+      delete task._id;
+      task["device"] = deviceId;
+    }
+
+    tasks = await apiFunctions.insertTasks(tasks);
+
+    statuses = tasks.map((t) => ({ _id: t._id, status: "pending" }));
+  } finally {
+    await releaseLock(`cwmp_session_${deviceId}`, token);
+  }
+
+  const now = Date.now();
+
+  const onlineThreshold = getConfig(
+    ctx.state.configSnapshot,
+    "cwmp.deviceOnlineThreshold",
+    4000,
+    (exp) => {
+      if (exp instanceof Expression.Literal) return exp;
+      else if (exp instanceof Expression.Parameter) {
+        const p = device[exp.path.toString()];
+        if (p != null) return new Expression.Literal(p);
+      } else if (exp instanceof Expression.FunctionCall) {
+        if (exp.name === "NOW") return new Expression.Literal(now);
+        if (exp.name === "REMOTE_ADDRESS") {
+          for (const root of ["InternetGatewayDevice", "Device"]) {
+            const p = device[`${root}.ManagementServer.ConnectionRequestURL`];
+            if (p != null) return new Expression.Literal(new URL(p).hostname);
+          }
+        }
+      }
+      return new Expression.Literal(null);
+    },
+  );
+
+  const lastInform = device["Events.Inform"] as number;
+
+  let status = await apiFunctions.connectionRequest(deviceId, device);
+  if (!status) {
+    const sessionStarted = await apiFunctions.awaitSessionStart(
+      deviceId,
+      lastInform,
+      onlineThreshold,
+    );
+    if (!sessionStarted) {
+      status = "No contact from CPE";
+    } else {
+      const sessionEnded = await apiFunctions.awaitSessionEnd(deviceId, 120000);
+      if (!sessionEnded) status = "Session took too long to complete";
     }
   }
 
-  // TODO Pass context and timestamp
-  const onlineThreshold = getConfig(
-    ctx.state.configSnapshot,
-    "cwmp.deviceOnlineThreshold"
-  );
+  if (!status) {
+    const promises = statuses.map((t) =>
+      collections.faults.count({ _id: `${deviceId}:task_${t._id}` }),
+    );
 
-  const res = await apiFunctions.postTasks(
-    ctx.params.id,
-    ctx.request.body,
-    onlineThreshold
-  );
+    const res = await Promise.all(promises);
+    for (const [i, r] of statuses.entries())
+      r.status = res[i] ? "fault" : "done";
+  }
 
-  log.tasks = res.tasks.map(t => t._id).join(",");
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectId(t._id))));
 
+  // Restore socket timeout
+  if (socketTimeout) ctx.socket.setTimeout(socketTimeout);
+
+  log.tasks = statuses.map((t) => t._id).join(",");
   logger.accessInfo(log);
 
-  ctx.set("Connection-Request", res.connectionRequest);
-  ctx.body = res.tasks;
+  ctx.set("Connection-Request", status || "OK");
+  ctx.body = statuses;
 });
 
-router.post("/devices/:id/tags", async (ctx, next) => {
+router.post("/devices/:id/tags", async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const log = {
     message: "Update tags",
     context: ctx,
     deviceId: ctx.params.id,
-    tags: ctx.request.body
+    tags: ctx.request.body,
   };
 
-  const authorizer = ctx.state.authorizer;
-  const filter = ["=", ["PARAM", "DeviceID.ID"], ctx.params.id];
+  const filter = Expression.and(
+    authorizer.getFilter("devices", 3),
+    new Expression.Binary(
+      "=",
+      new Expression.Parameter(Path.parse(RESOURCE_IDS["devices"])),
+      new Expression.Literal(ctx.params.id),
+    ),
+  );
   if (!authorizer.hasAccess("devices", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
-  const res = await db.query("devices", filter);
-  if (!res.length) return void next();
+  const { value: res } = await db.query("devices", filter).next();
+  if (!res) return void (ctx.status = 404);
 
-  const validate = authorizer.getValidator("devices", res[0]);
+  const validate = authorizer.getValidator("devices", res);
   if (!validate("tags", ctx.request.body)) {
     logUnauthorizedWarning(log);
     return void (ctx.status = 403);
@@ -517,33 +768,37 @@ router.post("/devices/:id/tags", async (ctx, next) => {
   ctx.body = "";
 });
 
-router.get("/ping/:host", async ctx => {
-  try {
-    ctx.body = await apiFunctions.ping(ctx.params.host);
-  } catch (err) {
-    ctx.status = 404;
-    ctx.response.headers["Cache-Control"] = "no-cache";
-    ctx.body = `${err.name}: ${err.message}`;
-  }
+router.get("/ping/:host", async (ctx) => {
+  if (!ctx.state.user) return void (ctx.status = 401);
+  return new Promise<void>((resolve) => {
+    ping(ctx.params.host, (err, parsed) => {
+      if (parsed) {
+        ctx.body = parsed;
+      } else {
+        ctx.status = 500;
+        ctx.body = err ? `${err.name}: ${err.message}` : "Unknown error";
+      }
+      resolve();
+    });
+  });
 });
 
-router.put("/users/:id/password", async (ctx, next) => {
+router.put("/users/:id/password", async (ctx) => {
+  const authorizer: Authorizer = ctx.state.authorizer;
   const username = ctx.params.id;
   const log = {
     message: "Change password",
     context: ctx,
-    username: username
+    username: username,
   };
 
-  const authorizer = ctx.state.authorizer;
-
   if (!ctx.state.user) {
     // User not logged in
     if (
-      !(await apiFunctions.authSimple(
+      !(await apiFunctions.authLocal(
         ctx.state.configSnapshot,
         username,
-        ctx.request.body.authPassword
+        ctx.request.body.authPassword,
       ))
     ) {
       logUnauthorizedWarning(log);
@@ -553,16 +808,23 @@ router.put("/users/:id/password", async (ctx, next) => {
     }
   } else if (!authorizer.hasAccess("users", 3)) {
     logUnauthorizedWarning(log);
-    return void next();
+    return void (ctx.status = 403);
   }
 
-  const filter = ["=", ["PARAM", RESOURCE_IDS.users], username];
-  const res = await db.query("users", filter);
-  if (!res.length) return void next();
-
   const newPassword = ctx.request.body.newPassword;
+
   if (ctx.state.user) {
-    const validate = authorizer.getValidator("users", res[0]);
+    const filter = Expression.and(
+      authorizer.getFilter("users", 3),
+      new Expression.Binary(
+        "=",
+        new Expression.Parameter(Path.parse(RESOURCE_IDS["users"])),
+        new Expression.Literal(username),
+      ),
+    );
+    const { value: res } = await db.query("users", filter).next();
+    if (!res) return void (ctx.status = 404);
+    const validate = authorizer.getValidator("users", res);
     if (!validate("password", { password: newPassword })) {
       logUnauthorizedWarning(log);
       return void (ctx.status = 403);
@@ -573,7 +835,7 @@ router.put("/users/:id/password", async (ctx, next) => {
   const password = await hashPassword(newPassword, salt);
   await db.putUser(username, { password, salt });
 
-  await del("presets_hash");
+  await del("ui-local-cache-hash");
 
   logger.accessInfo(log);
   ctx.body = "";