Complete implementation of ChangeDUState

Lookbehind in regex is not supported in Safari browsers. This implementation
should be more efficient so why not.

.eslintrc.json

@@ -0,0 +1,38 @@
+{
+  "env": {
+    "es6": true,
+    "node": true
+  },
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "project": "./tsconfig.json"
+  },
+  "plugins": ["@typescript-eslint"],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
+  "rules": {
+    "no-shadow": ["error", { "allow": ["err"] }],
+    "handle-callback-err": "error",
+    "prefer-arrow-callback": "error",
+    "curly": ["error", "multi-or-nest", "consistent"],
+    "no-buffer-constructor": "error",
+    "prefer-const": ["error", { "destructuring": "all" }],
+    "eqeqeq": ["error", "always", { "null": "ignore" }],
+    "no-return-await": "error",
+    "@typescript-eslint/no-explicit-any": 0,
+    "@typescript-eslint/no-use-before-define": [
+      "error",
+      { "functions": false }
+    ],
+    "@typescript-eslint/explicit-function-return-type": [
+      "error",
+      { "allowExpressions": true }
+    ],
+    "@typescript-eslint/no-floating-promises": "error",
+    "require-atomic-updates": 0,
+    "no-prototype-builtins": 0
+  }
+}

.gitignore

@@ -1,28 +1,4 @@
 *~
-lib/*.map
-lib/api-functions.js
-lib/auth.js
-lib/cache.js
-lib/common.js
-lib/config.js
-lib/cwmp.js
-lib/db.js
-lib/fs.js
-lib/nbi.js
-lib/query.js
-lib/server.js
-lib/session.js
 node_modules
-debug
-config/auth.js
-config/config.json
-config/cwmp.key
-config/cwmp.crt
-config/cwmp.ca-bundle
-config/nbi.key
-config/nbi.crt
-config/nbi.ca-bundle
-config/fs.key
-config/fs.crt
-config/fs.ca-bundle
-config/ext
+dist
+docs/_build

.npmignore

@@ -1,21 +0,0 @@
-*~
-.npmignore
-
-config/auth.js
-config/config.json
-config/cwmp.key
-config/cwmp.crt
-config/cwmp.ca-bundle
-config/nbi.key
-config/nbi.crt
-config/nbi.ca-bundle
-config/fs.key
-config/fs.crt
-config/fs.ca-bundle
-config/ext
-
-debug/*
-!debug/.gitkeep
-
-lib/*.coffee
-lib/*.map

.prettierignore

@@ -0,0 +1,2 @@
+dist
+npm-shrinkwrap.json

CHANGELOG.md

@@ -1,33 +1,214 @@
 # Change Log
 
+## 1.2.5 (2021-03-12)
+
+- Support specifying custom types when uploading files.
+- Fix JS compatibility issue with Safari browser.
+
+## 1.2.4 (2021-02-24)
+
+- The data model state of a CPE is no longer forgotten after unsuccessful
+  session termination (e.g. timeout). This addresses a number of undesired side
+  effects that arise when a CPE does not terminate the session properly.
+- Executing tasks that take a long time to complete (e.g. refreshing the entire
+  data model) no longer shows a timeout error while the task is still being
+  processed.
+- New function `ROUND()` available to expressions. It works similar to the
+  function by the same name in SQLite and PostgreSQL.
+- Log access events for `genieacs-nbi` service.
+- Pipe stdout/stderr from extension scripts to the `genieacs-cwmp` process log.
+- Parameter values of type `xsd:dateTime` are now displayed in the UI and CSV
+  downloads as a date string rather than a numeric value.
+- Add file download link in the files listing page.
+- Display spinner loading animation throughout the UI.
+- Display GenieACS version and build number underneath the logo.
+- New option to specify how many parameters are displayed at a time in the
+  all-parameters component. Simply set `limit` property in the component config.
+- Reduce overly strong Brotli compression level which was causing significant
+  page load slowdown when Brotli is used.
+- Retire dump-data-model tool. `genieacs-sim` can now use a CSV file as its data
+  model.
+- Reduce the number of concurrent database connections from each process.
+- Remove dependency on 'hostInfo' MongoDB command which is a privileged action.
+  It is now possible to use shared MongoDB instances with limited privileged.
+- Fix bug in NBI where querying files returns 404 error.
+- Fix ping not working for devices with an IPv6 address.
+- Fix an elusive memory leak in `genieacs-fs` that slowly eats up memory and can
+  go unnoticed for long periods of time.
+- Fix a rare edge case where a `declare()` call to set a parameter value may not
+  work as intended if the parameter was originally received as part of the
+  Inform message.
+- A number of other fixes and stability improvements.
+
+## 1.2.3 (2020-10-26)
+
+- New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
+  report writable parameters as non-writable. When set to true, the scripts will
+  no longer respect the 'writable' attribute of the CPE parameters and will send
+  a SetParamteerValues, AddObject, or DeleteObject request anyway.
+- Tags no longer restrict what characters are allowed. Any character other than
+  alphanumeric characters, hyphen, or underscore is now encoded in the data
+  model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
+- Ask for a confirmation before closing a pop-up dialog with unsaved changes.
+- Better XML validation to avoid crashes caused by invalid CPE requests.
+- Fix confusing 404 error message when the user attempts to modify a resource
+  when they don't have the necessary permissions.
+- Fix a rare issue where genieacs-cwmp stops accepting new connections after
+  running for a few weeks.
+- Fix exception when IS NULL operator is used in certain situations.
+
+## 1.2.2 (2020-10-03)
+
+- Added button to push files to selected devices from device listing page.
+- A few minor UI improvements.
+- Fix exception that can happen and persist after a Download request.
+- Fix validation bug preventing running refreshObject task on data model root.
+- Fix invalid arguments fault in refresh preset configuration when upgrading
+  from v1.1.
+
+## 1.2.1 (2020-09-08)
+
+- Fix bug causing faults to not be displayed in the UI.
+- Fix bug where deleting objects does not get reflected immediately in the UI.
+- Improve conversion between filters written in the expression format and
+  MongoDB queries. There should now be fewer edge cases where the two are not
+  equisatisfiable.
+
+## 1.2.0 (2020-09-01)
+
+- Support GetParameterAttributes and SetParameterAttributes TR-069 methods.
+- Support CASE statement and COALESCE function in expressions.
+- Provision arguments can now be a list of expressions that are dynamically
+  evaluated.
+- Support Forwarded HTTP header to display in the logs the correct IP of CPEs
+  behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
+- Config expressions can now access all available device parameters, not only
+  serial number, product class, and OUI.
+- Use relative URLs throughout the UI to allow serving from a subdirectory using
+  a reverse proxy.
+- Make Date.parse() and Date.UTC() available to provision scripts.
+- libxmljs has been entirely removed in favor of our bespoke XML parser.
+- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now used
+  to determine the TCP connection timeout.
+- The all-parameters component now limits the number of parameters displayed for
+  better performance.
+- The process genieacs-cwmp is now much less likely to throw exceptions as a
+  result of invalid requests from CPE.
+- A large number of bug fixes and stability improvements.
+
+## 1.2.0-beta.0 (2019-07-30)
+
+- A brand new UI superseding genieacs-gui.
+- New initialization wizard on first run.
+- New expression/query language used in search filters and preset preconditions.
+- CPE -> ACS authentication is now supported.
+- New config option (CWMP_KEEP_ALIVE_TIMEOUT) to specify how long to wait for a
+  reply from the CPE before closing the TCP connection.
+- Debug logging has been reimplemented utilizing YAML format for logs.
+- Handle 9005 faults (Invalid Parameter Name) gracefully by attempting to
+  rediscover the path of the missing parameter recursively.
+- declare() statements not followed by an explicit commit() are now deferred
+  until all currently active scripts have been executed.
+- FS_HOSTNAME now defaults to the server's hostname or IP.
+- The API now validates the structure of task objects before saving.
+- New XML parser implementation for better performance. You can revert to the
+  old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
+  v10.
+- Performance optimizations. While performance has improved for the majority of
+  use cases, there may be situations where performance has degraded. It's
+  recommended to revisit your hardware requirements.
+- Connection request authentication no longer uses 'auth.js' file. Instead, the
+  connection request authentication behavior can now be customized using an
+  'expression'.
+- The config file (config.json) has been deprecated. System configuration (e.g.
+  listen ports, worker count) are now recommended to be passed as environment.
+  variables. Other general configuration options are stored in the database so
+  as to not require service restart for changes to take effect.
+- Optional redis dependency has been removed completely.
+- Tags now allow only alphanumeric characters and underscore.
+- Supported versions of Node.js and MongoDB are 10.x and up and 2.6 and up
+  respectively.
+
+## 1.1.3 (2018-10-23)
+
+- New config option (MAX_COMMIT_ITERATIONS) to avoid max commit iterations
+  faults for more complex scripts.
+- Support base64 and hexBinary parameter types.
+- Strict parsing of number values in queries (e.g. "123abc" no longer accepted
+  as 123).
+- Mixing $ne and $not operators is not allowed. Now it throws an error instead
+  of returning incorrect results.
+- When a task expires, any associated fault is also deleted.
+- API now accepts 'timeout' argument when posting a task.
+- A number of stability fixes.
+
+## 1.1.2 (2018-02-24)
+
+- A large number of bug fixes as well as stability and performance improvements.
+- Three security vulnerabilities disclosed by Maximilian Hils have been patched.
+- New config option UDP_CONNECTION_REQUEST_PORT to specify binding port for UDP
+  connection requests.
+- New config option DATETIME_MILLISECONDS to strip milliseconds from dateTime
+  values.
+- New config option BOOLEAN_LITERAL to use 1/0 or true/false for boolean values.
+- Parameter values that cannot be parsed according to the reported type now show
+  a warning message.
+- Virtual parameter scripts now use the variable 'args' instead of the special
+  'TIMESTAMPS' and 'VALUES' variables. The content of the args array is:
+  {declare timestamps}, {declare values}, {current timestamps}, {current
+  values}.
+- Virtual parameter value types are now inferred from the JavaScript type if the
+  returned value attribute is not a value-type pair.
+- Show a fault when a virtual parameter script doesn't return the required
+  attributes.
+- Redis is now optional (and disabled by default), reducing the complexity of
+  scalable deployments.
+- Better detection of cyclical presets resulting in fewer faults for complex
+  provisioning scripts.
+- Math.random() is now deterministic on per-device basis. A function has been
+  added to allow specifying a seed value (e.g. Math.random.seed(Date.now())).
+- Overload spikes are now handled gracefully by refusing to accept new sessions
+  temporarily when under abnormal load.
+- Added log messages for session timeouts, connection drops, and XML parsing
+  errors.
+- Date.now() now takes an optional argument to specify "time steps" (in
+  milliseconds). This can be used to ensure a group of parameters are all
+  refreshed at the same time intervals.
+- Only the non-default configuration options are now logged at process start.
+- Faults caused by errors from extensions now show a cleaner stack trace.
+- Exit main process if there are too many worker crashes (e.g. when DB is down).
+- Updated dependencies and included a lockfile to ensure installations get the
+  exact dependencies it was tested against.
+
 ## 1.1.1 (2017-03-23)
 
 - Avoid crashing when connection request credentials are missing.
 - Show a warning instead of crashing when failing to parse parameter values
-according to the expected value type.
+  according to the expected value type.
 - Add missing "Registered" event.
 - Fix bug where in certain cases many more instances than declared are created.
 - Fix parameter discovery bug when declared path timestamp is 1 or is not set.
 - Fix preset precondition failing when testing against datetime parameters and
-certain other parameters like \_deviceId.\_ProductClass.
+  certain other parameters like \_deviceId.\_ProductClass.
 
 ## 1.1.0 (2017-03-10)
 
 - Provisions enable implementing dynamic device configuration or complex device
-provisioning work flow using arbitrary scripts.
+  provisioning work flow using arbitrary scripts.
 - Virtual parameters are user-defined parameters whose values are evaluated from
-a custom script.
+  a custom script.
 - Extensions are sandboxed Node.js scripts that are accessible from provision
-and virtual parameter scripts to facilitate integration with external entities.
-- Support for UDP/STUN based connection requests for reaching devices behind
-NAT (TR-069 Annex G).
+  and virtual parameter scripts to facilitate integration with external
+  entities.
+- Support for UDP/STUN based connection requests for reaching devices behind NAT
+  (TR-069 Annex G).
 - Presets can now be scheduled using a cron-like expression.
 - Presets can now be tied to specific device events (e.g. boot).
-- Presets precondition queries no longer support "$or" or other MongoDB logical
-operators.
+- Presets precondition queries no longer support "\$or" or other MongoDB logical
+  operators.
 - Faults are no longer a part of tasks but are now first class objects.
 - Presets are now assigned to channels. A fault in one channel only blocks
-presets in that channel.
+  presets in that channel.
 - New API CRUD functions for provisions, virtual parameters, and faults.
 - New config options for XML output.
 - API responses now include "GenieACS-Version" header.
@@ -36,19 +217,19 @@ presets in that channel.
 - Supported Node.js versions are 6.x and 7.x.
 - Supported MongoDB versions are 2.6 through 3.4.
 - Expect performance differences due to major under the hood changes. Some
-operations are faster and some are slower. Overall performance is improved.
+  operations are faster and some are slower. Overall performance is improved.
 - GenieACS will no longer fetch the entire device data model upon first contact
-but will instead only fetch the parameters it needs to fulfill the presets.
+  but will instead only fetch the parameters it needs to fulfill the presets.
 - Logs have been overhauled and split into two streams: process log (stderr) and
-access log (stdout). Also added config options to dump logs to files rather than
-standard streams.
+  access log (stdout). Also added config options to dump logs to files rather
+  than standard streams.
 - Connection request authentication credentials are picked up from the device
-data model if available. config/auth.js is still supported as a fallback and now
-supports an optional callback argument.
+  data model if available. config/auth.js is still supported as a fallback and
+  now supports an optional callback argument.
 - Custom commands have been removed. Use virtual parameters and/or extensions.
 - Aliases and value normalizers (config/parameters.json) have been removed. Use
-virtual parameters.
+  virtual parameters.
 - The API /devices/<device_id>/preset has been removed.
 - Rarely used RequestDownload method no longer supported.
 - The TR-069 client simulator has moved to its own repo at
-https://github.com/zaidka/genieacs-sim
+  https://github.com/zaidka/genieacs-sim

CONTRIBUTING.md

@@ -1,22 +1,21 @@
 # Have a question?
 
-Please ask in the [Users mailing list][mailing-lists] instead of opening a
-Github Issue. The Github issue tracker is strictly for bug reports.
+Please ask in the [forum] instead of opening a Github Issue. The Github issue
+tracker is strictly for bug reports.
 
 # Issues
 
-We use GitHub issues to track bugs. Please ensure your bug description is
-clear and has sufficient instructions to be able to reproduce the issue.
+We use GitHub issues to track bugs. Please ensure your bug description is clear
+and has sufficient instructions to be able to reproduce the issue.
 
-If you face interoperability issues with your device, more often than not it's
-a device related issue. Please consult the [Users mailing list][mailing-lists]
-before submitting an Issue.
+If you face interoperability issues with your CPE, more often than not it's a
+device related issue. Please consult the [forum] before submitting an Issue.
 
 # Pull Requests
 
 Pull requests are welcome. If you have a fix for a bug, go ahead and create a
-pull request. For other kinds of enhancements, please discuss in the [Users
-mailing list][mailing-lists] first.
+pull request. For other kinds of enhancements, please discuss in the [forum]
+first.
 
 Please follow the guidelines [here][commit-guidelines] when committing your
 changes.
@@ -27,13 +26,14 @@ By submitting a pull request to this repository, you acknowledge that, while
 maintaining copyright, you grant Zaid Abdulla a perpetual, worldwide, non-
 exclusive, no-charge, royalty-free, irrevocable license to reproduce, prepare
 derivative works of, publicly display, publicly perform, sublicense, and
-distribute your contributions and such derivative works under the AGPLv3
-license or any other license terms, including, but not limited to, proprietary
-or commercial license terms.
+distribute your contributions and such derivative works under the AGPLv3 license
+or any other license terms, including, but not limited to, proprietary or
+commercial license terms.
 
-You confirm that you own or have rights to distribute and sublicense the
-source code contained therein, and that your content does not infringe upon
-the intellectual property rights of a third party.
+You confirm that you own or have rights to distribute and sublicense the source
+code contained therein, and that your content does not infringe upon the
+intellectual property rights of a third party.
 
-[commit-guidelines]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
-[mailing-lists]: http://lists.genieacs.com
+[commit-guidelines]:
+  http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
+[forum]: https://forum.genieacs.com

README.md

@@ -1,75 +1,76 @@
 # GenieACS
 
-GenieACS is a fast and lightweight TR-069 auto configuration server (ACS). This
-is the core back end component. A GUI front end is available at
-https://github.com/zaidka/genieacs-gui.
-
-## Features
-
-- **Battle-tested at scale**: Can scale to manage hundreds of thousands and
-potentially millions of devices
-- **Powerful provisioning system**: Scriptable configuration to handle complex
-device provisioning scenarios and other automated operations such as firmware
-upgrade
-- **Extensive API**: A rich HTTP-based API allows easy integration with other
-systems
+GenieACS is a high performance Auto Configuration Server (ACS) for remote
+management of TR-069 enabled devices. It utilizes a declarative and fault
+tolerant configuration engine for automating complex provisioning scenarios at
+scale. It's battle-tested to handle hundreds of thousands and potentially
+millions of concurrent devices.
 
 ## Quick Start
 
-Install [Node.js](http://nodejs.org/), [Redis](http://redis.io/), and
-[MongoDB](http://www.mongodb.org/). Refer to the corresponding documentation for
-installation guides. Supported versions are:
+Install [Node.js](http://nodejs.org/) and [MongoDB](http://www.mongodb.org/).
+Refer to their corresponding documentation for installation instructions. The
+supported versions are:
 
-- Node.js: 6.x and 7.x (7.x recommended)
-- Redis: 2.4 through 3.2
-- MongoDB: 2.6 through 3.4
+- Node.js: 12.3+
+- MongoDB: 3.6+
 
-Then install GenieACS using NPM:
+Install GenieACS from NPM:
 
-    npm install -g genieacs
+    sudo npm install -g genieacs
 
-Alternatively, you can install from source by cloning the git repository:
+To build from source instead, clone this repo or download the source archive
+then _cd_ into the source directory then run:
 
-    git clone https://github.com/zaidka/genieacs.git
-    cd genieacs
-    git checkout $(git tag -l v1.1.* --sort=-v:refname | head -n 1)
     npm install
-    npm run compile
+    npm run build
 
-Finally, run the following (from bin directory if installing from source):
+Finally, run the following services (found under `./dist/bin/` if building from
+source):
 
-    genieacs-cwmp
+### genieacs-cwmp
 
-This is the service that the CPEs will communicate with. It listens to port 7547
-by default (see config/config.json). Configure the ACS URL of your devices
-accordingly.
+This is the service that the CPEs will communicate with. It listens on port 7547
+by default. Configure the ACS URL in your devices accordingly.
 
-    genieacs-nbi
+You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
+as a dummy TR-069 simulator if you don't have a CPE at hand.
+
+### genieacs-nbi
 
 This is the northbound interface module. It exposes a REST API on port 7557 by
-default. This must be running for the GUI front end to work.
+default. This one is only required if you have an external system integrating
+with GenieACS using this API.
 
-    genieacs-fs
+### genieacs-fs
 
 This is the file server from which the CPEs will download firmware images and
-such.
+such. It listens on port 7567 by default.
 
-Note: For production deployment make sure to run those as background services.
+### genieacs-ui
 
-For further details about installation and configuration, refer to the
-[wiki section](https://github.com/zaidka/genieacs/wiki).
+This serves the web based user interface. It listens on port 3000 by default.
+You must pass _--ui-jwt-secret_ argument to supply the secret key used for
+signing browser cookies:
+
+    genieacs-ui --ui-jwt-secret secret
+
+The UI has plenty of configuration options. When you open GenieACS's UI in a
+browser you'll be greeted with a database initialization wizard to help you
+populate some initial configuration.
+
+Visit [docs.genieacs.com](https://docs.genieacs.com) for more documentation and
+a complete installation guide for production deployments.
 
 ## Support
 
-The [Users mailing list](http://lists.genieacs.com) is a good place to get
-guidance and help from the community. Head on over and join the conversation!
-In addition, the [wiki](https://github.com/zaidka/genieacs/wiki) provides useful
-documentation and tips from GenieACS users.
+The [forum](https://forum.genieacs.com) is a good place to get guidance and help
+from the community. Head on over and join the conversation!
 
-For commercial support options and professional services, please visit
-[genieacs.com](https://genieacs.com).
+For commercial support options, please visit
+[genieacs.com](https://genieacs.com/support/).
 
 ## License
 
-Copyright 2013-2017 Zaid Abdulla. GenieACS is released under the
-[AGPLv3 license terms](https://raw.githubusercontent.com/zaidka/genieacs/master/LICENSE).
+Copyright 2013-2020 GenieACS Inc. GenieACS is released under the
+[AGPLv3 license terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

ava.config.js

@@ -0,0 +1,4 @@
+export default {
+  extensions: ["ts"],
+  require: ["esm", "ts-node/register/transpile-only"],
+};

bin/genieacs-cwmp

@@ -1,9 +0,0 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
-
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
-
-var cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('cwmp');

bin/genieacs-cwmp.ts

@@ -0,0 +1,122 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import * as cwmp from "../lib/cwmp";
+import * as db from "../lib/db";
+import * as extensions from "../lib/extensions";
+import { version as VERSION } from "../package.json";
+
+logger.init("cwmp", VERSION);
+
+const SERVICE_ADDRESS = config.get("CWMP_INTERFACE") as string;
+const SERVICE_PORT = config.get("CWMP_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ]).catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  extensions.killAll().finally(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-cwmp starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("CWMP_SSL_KEY") as string;
+  const cert = config.get("CWMP_SSL_CERT") as string;
+
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    onConnection: cwmp.onConnection,
+    timeout: 30000,
+    keepAliveTimeout: 0,
+  };
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, cwmp.listener);
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-ext.ts

@@ -0,0 +1,96 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Fault } from "../lib/types";
+
+const jobs = new Set();
+const fileName = process.argv[2];
+let script;
+
+function errorToFault(err: Error): Fault {
+  if (!err) return null;
+
+  if (!err.name) return { code: "ext", message: `${err}` };
+
+  const fault: Fault = {
+    code: `ext.${err.name}`,
+    message: err.message,
+    detail: {
+      name: err.name,
+      message: err.message,
+    },
+  };
+
+  if (err.stack) {
+    fault.detail["stack"] = err.stack;
+    // Trim the stack trace
+    const stackTrimIndex = fault.detail["stack"].match(
+      /\s+at\s[^\s]+\s\(.*genieacs-ext:.+\)/
+    );
+    if (stackTrimIndex) {
+      fault.detail["stack"] = fault.detail["stack"].slice(
+        0,
+        stackTrimIndex.index
+      );
+    }
+  }
+
+  return fault;
+}
+
+process.on("uncaughtException", (err) => {
+  const fault = errorToFault(err);
+  jobs.forEach((jobId) => {
+    process.send([jobId, fault, null]);
+  });
+  jobs.clear();
+  process.disconnect();
+});
+
+process.on("message", (message) => {
+  jobs.add(message[0]);
+
+  if (!script) {
+    const cwd = process.env["GENIEACS_EXT_DIR"];
+    process.chdir(cwd);
+    script = require(`${cwd}/${fileName}`);
+  }
+
+  const funcName = message[1][0];
+
+  if (!script[funcName]) {
+    const fault = {
+      code: "ext",
+      message: `No such function '${funcName}' in extension '${fileName}'`,
+    };
+    process.send([message[0], fault, null]);
+    return;
+  }
+
+  script[funcName](message[1].slice(1), (err, res) => {
+    if (!jobs.delete(message[0])) return;
+
+    process.send([message[0], errorToFault(err), res]);
+  });
+});
+
+// Ignore SIGINT
+process.on("SIGINT", () => {
+  // Ignore
+});

bin/genieacs-fs

@@ -1,9 +0,0 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
-
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
-
-var cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('fs');

bin/genieacs-fs.ts

@@ -0,0 +1,124 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import { listener } from "../lib/fs";
+import * as db from "../lib/db";
+import { version as VERSION } from "../package.json";
+
+logger.init("fs", VERSION);
+
+const SERVICE_ADDRESS = config.get("FS_INTERFACE") as string;
+const SERVICE_PORT = config.get("FS_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([db.disconnect(), cluster.worker.disconnect()]).catch(
+    exitWorkerUngracefully
+  );
+}
+
+function exitWorkerUngracefully(): void {
+  process.exit(1);
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-fs starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("FS_SSL_KEY") as string;
+  const cert = config.get("FS_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
+  };
+
+  let stopping = false;
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    stopping = true;
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const _listener = (req, res): void => {
+    if (stopping) res.setHeader("Connection", "close");
+    void listener(req, res);
+  };
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, _listener);
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-nbi

@@ -1,9 +0,0 @@
-#!/usr/bin/env node
-var path = require('path');
-var fs = require('fs');
-
-var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
-process.chdir(dir);
-
-var cluster = require(path.resolve(dir, 'lib/cluster'));
-cluster.start('nbi');

bin/genieacs-nbi.ts

@@ -0,0 +1,129 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import { listener } from "../lib/nbi";
+import * as db from "../lib/db";
+import * as extensions from "../lib/extensions";
+import { version as VERSION } from "../package.json";
+
+logger.init("nbi", VERSION);
+
+const SERVICE_ADDRESS = config.get("NBI_INTERFACE") as string;
+const SERVICE_PORT = config.get("NBI_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ]).catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  extensions.killAll().finally(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-nbi starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("NBI_SSL_KEY") as string;
+  const cert = config.get("NBI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
+  };
+
+  let stopping = false;
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    stopping = true;
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const _listener = (req, res): void => {
+    if (stopping) res.setHeader("Connection", "close");
+    listener(req, res);
+  };
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, _listener);
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+}

bin/genieacs-ui.ts

@@ -0,0 +1,129 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as config from "../lib/config";
+import * as logger from "../lib/logger";
+import * as cluster from "../lib/cluster";
+import * as server from "../lib/server";
+import { listener } from "../lib/ui";
+import * as extensions from "../lib/extensions";
+import * as db from "../lib/db";
+import { version as VERSION } from "../package.json";
+
+logger.init("ui", VERSION);
+
+const SERVICE_ADDRESS = config.get("UI_INTERFACE") as string;
+const SERVICE_PORT = config.get("UI_PORT") as number;
+
+function exitWorkerGracefully(): void {
+  setTimeout(exitWorkerUngracefully, 5000).unref();
+  Promise.all([
+    db.disconnect(),
+    extensions.killAll(),
+    cluster.worker.disconnect(),
+  ]).catch(exitWorkerUngracefully);
+}
+
+function exitWorkerUngracefully(): void {
+  extensions.killAll().finally(() => {
+    process.exit(1);
+  });
+}
+
+if (!cluster.worker) {
+  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES") as number;
+
+  logger.info({
+    message: `genieacs-ui starting`,
+    pid: process.pid,
+    version: VERSION,
+  });
+
+  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
+
+  process.on("SIGINT", () => {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+
+  process.on("SIGTERM", () => {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid,
+    });
+
+    cluster.stop();
+  });
+} else {
+  const key = config.get("UI_SSL_KEY") as string;
+  const cert = config.get("UI_SSL_CERT") as string;
+  const options = {
+    port: SERVICE_PORT,
+    host: SERVICE_ADDRESS,
+    ssl: key && cert ? { key, cert } : null,
+    timeout: 30000,
+  };
+
+  let stopping = false;
+
+  process.on("uncaughtException", (err) => {
+    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
+    logger.error({
+      message: "Uncaught exception",
+      exception: err,
+      pid: process.pid,
+    });
+    stopping = true;
+    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+  });
+
+  const _listener = (req, res): void => {
+    if (stopping) res.setHeader("Connection", "close");
+    listener(req, res);
+  };
+
+  const initPromise = db
+    .connect()
+    .then(() => {
+      server.start(options, _listener);
+    })
+    .catch((err) => {
+      setTimeout(() => {
+        throw err;
+      });
+    });
+
+  process.on("SIGINT", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+
+  process.on("SIGTERM", () => {
+    stopping = true;
+    initPromise.finally(() => {
+      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
+    });
+  });
+}

build/build.ts

@@ -0,0 +1,335 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as path from "path";
+import * as fs from "fs";
+import { rollup, WarningHandler } from "rollup";
+import rollupJson from "@rollup/plugin-json";
+import typescript from "@rollup/plugin-typescript";
+import { terser } from "rollup-plugin-terser";
+import nodeResolve from "@rollup/plugin-node-resolve";
+import commonjs from "@rollup/plugin-commonjs";
+import postcss from "postcss";
+import postcssImport from "postcss-import";
+import postcssPresetEnv from "postcss-preset-env";
+import cssnano from "cssnano";
+import { optimize, extendDefaultPlugins } from "svgo";
+import * as xmlParser from "../lib/xml-parser";
+
+const MODE = process.env["NODE_ENV"] || "production";
+
+const BUILD_METADATA = new Date()
+  .toISOString()
+  .split(".")[0]
+  .replace(/[^0-9]/g, "");
+
+const INPUT_DIR = path.resolve(__dirname, "..");
+const OUTPUT_DIR = path.resolve(__dirname, "../dist");
+
+const builtins = [
+  "path",
+  "fs",
+  "cluster",
+  "os",
+  "tls",
+  "http",
+  "https",
+  "zlib",
+  "crypto",
+  "util",
+  "vm",
+  "querystring",
+  "child_process",
+  "dgram",
+  "url",
+  "readline",
+  "stream",
+];
+
+function rmDirSync(dirPath): void {
+  if (!fs.existsSync(dirPath)) return;
+  const files = fs.readdirSync(dirPath);
+
+  for (const file of files) {
+    const filePath = `${dirPath}/${file}`;
+    if (fs.statSync(filePath).isFile()) fs.unlinkSync(filePath);
+    else rmDirSync(filePath);
+  }
+  fs.rmdirSync(dirPath);
+}
+
+function stripDevDeps(deps): void {
+  if (!deps["dependencies"]) return;
+  for (const [k, v] of Object.entries(deps["dependencies"])) {
+    if (v["dev"]) delete deps["dependencies"][k];
+    else stripDevDeps(v);
+  }
+  if (!Object.keys(deps["dependencies"]).length) delete deps["dependencies"];
+}
+
+function xmlTostring(xml): string {
+  const children = [];
+  for (const c of xml.children || []) children.push(xmlTostring(c));
+
+  return xml.name === "root" && xml.bodyIndex === 0
+    ? children.join("")
+    : `<${xml.name} ${xml.attrs}>${children.join("")}</${xml.name}>`;
+}
+
+function generateSymbol(id: string, svgStr: string): string {
+  const xml = xmlParser.parseXml(svgStr);
+  const svg = xml.children[0];
+  const svgAttrs = xmlParser.parseAttrs(svg.attrs);
+  let viewBox = "";
+  for (const a of svgAttrs) {
+    if (a.name === "viewBox") {
+      viewBox = `viewBox="${a.value}"`;
+      break;
+    }
+  }
+  const symbolBody = xml.children[0].children
+    .map((c) => xmlTostring(c))
+    .join("");
+  return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
+}
+
+async function init(): Promise<string[]> {
+  // Delete any old output directory
+  rmDirSync(OUTPUT_DIR);
+
+  // Create output directory layout
+  fs.mkdirSync(OUTPUT_DIR);
+  fs.mkdirSync(OUTPUT_DIR + "/bin");
+  fs.mkdirSync(OUTPUT_DIR + "/public");
+
+  // Create package.json
+  const packageJson = JSON.parse(
+    fs.readFileSync(path.resolve(INPUT_DIR, "package.json")).toString()
+  );
+  delete packageJson["devDependencies"];
+  delete packageJson["private"];
+  packageJson["scripts"] = {
+    install: packageJson["scripts"].install,
+    configure: packageJson["scripts"].configure,
+  };
+  packageJson["version"] = `${packageJson["version"]}+${BUILD_METADATA}`;
+  fs.writeFileSync(
+    path.resolve(OUTPUT_DIR, "package.json"),
+    JSON.stringify(packageJson, null, 2)
+  );
+
+  // Create npm-shrinkwrap.json
+  const npmShrinkwrapJson = JSON.parse(
+    fs.readFileSync(path.resolve(INPUT_DIR, "npm-shrinkwrap.json")).toString()
+  );
+  npmShrinkwrapJson["version"] = packageJson["version"];
+  stripDevDeps(npmShrinkwrapJson);
+  fs.writeFileSync(
+    path.resolve(OUTPUT_DIR, "npm-shrinkwrap.json"),
+    JSON.stringify(npmShrinkwrapJson, null, 2)
+  );
+  return Object.keys(packageJson["dependencies"]);
+}
+
+async function copyStatic(): Promise<void> {
+  const files = [
+    "LICENSE",
+    "README.md",
+    "CHANGELOG.md",
+    "public/logo.svg",
+    "public/favicon.png",
+  ];
+
+  for (const file of files) {
+    fs.copyFileSync(
+      path.resolve(INPUT_DIR, file),
+      path.resolve(OUTPUT_DIR, file)
+    );
+  }
+}
+
+async function generateCss(): Promise<void> {
+  const cssInPath = path.resolve(INPUT_DIR, "ui/css/app.css");
+  const cssOutPath = path.resolve(OUTPUT_DIR, "public/app.css");
+  const cssIn = fs.readFileSync(cssInPath);
+  const cssOut = await postcss([
+    postcssImport,
+    postcssPresetEnv({
+      stage: 3,
+      features: {
+        "nesting-rules": true,
+        "color-mod-function": true,
+      },
+    }),
+    cssnano,
+  ]).process(cssIn, { from: cssInPath, to: cssOutPath });
+  fs.writeFileSync(cssOutPath, cssOut.css);
+}
+
+async function generateBackendJs(externals: string[]): Promise<void> {
+  for (const bin of [
+    "genieacs-cwmp",
+    "genieacs-ext",
+    "genieacs-nbi",
+    "genieacs-fs",
+    "genieacs-ui",
+  ]) {
+    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}.ts`);
+    const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
+    const bundle = await rollup({
+      input: inputFile,
+      external: [...builtins, ...externals],
+      acorn: {
+        allowHashBang: true,
+      },
+      treeshake: {
+        propertyReadSideEffects: false,
+        moduleSideEffects: false,
+      },
+      plugins: [
+        rollupJson({ preferConst: true }),
+        {
+          name: "",
+          resolveId: (importee, importer) => {
+            if (importee.endsWith("/package.json")) {
+              const p = path.resolve(path.dirname(importer), importee);
+              if (p === path.resolve(INPUT_DIR, "package.json"))
+                return path.resolve(OUTPUT_DIR, "package.json");
+            }
+            return null;
+          },
+        },
+        typescript({
+          tsconfig: "./tsconfig.json",
+          include: [`bin/${bin}.ts`, "lib/**/*.ts"],
+        }),
+        MODE === "production" ? terser() : null,
+      ],
+    });
+
+    await bundle.write({
+      format: "cjs",
+      preferConst: true,
+      sourcemap: "inline",
+      sourcemapExcludeSources: true,
+      banner: "#!/usr/bin/env node",
+      file: outputFile,
+    });
+
+    // Mark as executable
+    const mode = fs.statSync(outputFile).mode;
+    fs.chmodSync(outputFile, mode | 73);
+  }
+}
+
+async function generateFrontendJs(externals: string[]): Promise<void> {
+  const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
+  const outputDir = path.resolve(OUTPUT_DIR, "public");
+
+  const inlineDeps = ["parsimmon", "espresso-iisojs"];
+  const bundle = await rollup({
+    input: inputFile,
+    external: [
+      ...builtins,
+      ...externals.filter((e) => !inlineDeps.includes(e)),
+    ],
+    plugins: [
+      rollupJson({ preferConst: true }),
+      {
+        name: "",
+        resolveId: function (importee, importer) {
+          if (importee.endsWith("/package.json")) {
+            const p = path.resolve(path.dirname(importer), importee);
+            if (p === path.resolve(INPUT_DIR, "package.json"))
+              return path.resolve(OUTPUT_DIR, "package.json");
+          }
+          return null;
+        },
+      },
+      typescript({ tsconfig: "./tsconfig.json" }),
+      nodeResolve(),
+      commonjs(),
+      MODE === "production" ? terser() : null,
+    ],
+    preserveEntrySignatures: false,
+    treeshake: {
+      propertyReadSideEffects: false,
+      moduleSideEffects: false,
+    },
+    onwarn: ((warning, warn) => {
+      // Ignore circular dependency warnings
+      if (warning.code !== "CIRCULAR_DEPENDENCY") warn(warning);
+    }) as WarningHandler,
+  });
+
+  await bundle.write({
+    manualChunks: (id) => {
+      if (id.includes("node_modules/codemirror")) return "codemirror";
+      else if (id.includes("node_modules/yaml")) return "yaml";
+      return "app";
+    },
+    preferConst: true,
+    format: "es",
+    sourcemap: true,
+    sourcemapExcludeSources: true,
+    dir: outputDir,
+  });
+}
+
+async function generateIconsSprite(): Promise<void> {
+  const symbols = [];
+  const iconsDir = path.resolve(INPUT_DIR, "ui/icons");
+  for (const file of fs.readdirSync(iconsDir)) {
+    const id = path.parse(file).name;
+    const filePath = path.join(iconsDir, file);
+    const { data } = await optimize(fs.readFileSync(filePath).toString(), {
+      plugins: extendDefaultPlugins([
+        {
+          name: "removeViewBox",
+          active: false,
+        },
+      ]),
+    });
+    symbols.push(generateSymbol(id, data));
+  }
+  fs.writeFileSync(
+    path.resolve(OUTPUT_DIR, "public/icons.svg"),
+    `<svg xmlns="http://www.w3.org/2000/svg">${symbols.join("")}</svg>`
+  );
+}
+
+init()
+  .then((externals) => {
+    Promise.all([
+      copyStatic(),
+      generateCss(),
+      generateIconsSprite(),
+      generateBackendJs(externals),
+      generateFrontendJs(externals),
+    ])
+      .then(() => {
+        // Ignore
+      })
+      .catch((err) => {
+        process.stderr.write(err.stack + "\n");
+      });
+  })
+  .catch((err) => {
+    process.stderr.write(err.stack + "\n");
+  });

build/spellcheck-dict.pws

@@ -0,0 +1,167 @@
+personal_ws-1.1 en 0
+genieacs
+GenieACS
+javascript
+sudo
+config
+cwmp
+CPE
+systemctl
+nbi
+CWMP
+NBI
+ACS
+SSL
+xsd
+args
+DeviceID
+GENIEACS
+env
+param
+arg
+AUTH
+igd
+TCP
+toctree
+auth
+EnvironmentFile
+JSON
+json
+parameterValues
+setParameterValues
+stderr
+stdout
+attr
+cfg
+CPEs
+ExecStart
+faq
+FileType
+GenieACS's
+JWT
+maxdepth
+npm
+objectName
+refreshObject
+SerialNumber
+systemd
+usr
+WantedBy
+yaml
+chown
+Config
+getParameterValues
+https
+InternetGatewayDevice
+mkdir
+OUI
+sql
+TODO
+UDP
+addObject
+AGPLv
+const
+cpe
+cron
+deleteObject
+FactoryReset
+factoryReset
+FileName
+fileType
+Github
+hostname
+HTTPS
+jwt
+latlong
+NPM
+oui
+parameterNames
+ProductClass
+productClass
+repo
+SSID
+sublicense
+TLS
+WANIPConnection
+WPA
+YourProvisionName
+abc
+Abdulla
+accessors
+AddObject
+AddressingType
+api
+APIs
+chmod
+cwmp's
+dateext
+DATETIME
+dateTime
+datetime
+declaratively
+delaycompress
+DeleteObject
+deviceId
+DHCP
+dir
+distro
+equisatisfiable
+ExternalIPAddress
+failover
+favor
+func
+GetParameterAttributes
+getPassword
+gte
+gui
+hexBinary
+Hils
+HOSTNAME
+init
+journalctl
+journald
+LastFileName
+LastFileType
+lastInform
+LIBXMLJS
+libxmljs
+literalinclude
+lockfile
+logrotate
+lte
+mipsbe
+normalizers
+pre
+quickstart
+README
+redis
+reimplemented
+RequestDownload
+RESTful
+rollout
+sandboxed
+scalable
+SetParameterAttributes
+SetParamteerValues
+SIGINT
+SIGTERM
+skipWritableCheck
+stringify
+subdirectory
+TargetFileName
+useradd
+VirtualParameters
+vparam
+xml
+YAML
+Zaid
+cacheExpire
+iss
+http
+statusCode
+rawData
+pos
+Brotli
+CSV
+hostInfo
+IPv

build/spellcheck.sh

@@ -0,0 +1,12 @@
+#! /bin/sh
+
+cd "$(dirname "$0")"
+
+FILES=`ls ../docs/*.rst ../docs/*.js ../*.md`
+
+for FILE in $FILES
+do
+    echo $FILE
+    cat $FILE | aspell list --lang=en --add-extra-dicts=./spellcheck-dict.pws --ignore 2
+    echo
+done

config/auth-sample.js

@@ -1,7 +0,0 @@
-"use strict";
-
-function connectionRequest(deviceId, url, username, password, callback) {
-  return callback(username || deviceId, password || "");
-}
-
-exports.connectionRequest = connectionRequest;

config/config-sample.json

@@ -1,15 +0,0 @@
-{
-  "MONGODB_CONNECTION_URL" : "mongodb://127.0.0.1/genieacs",
-  "REDIS_PORT" : "6379",
-  "REDIS_HOST" : "127.0.0.1",
-  "CWMP_INTERFACE" : "0.0.0.0",
-  "CWMP_PORT" : 7547,
-  "CWMP_SSL" : false,
-  "NBI_INTERFACE" : "0.0.0.0",
-  "NBI_PORT" : 7557,
-  "FS_INTERFACE" : "0.0.0.0",
-  "FS_PORT" : 7567,
-  "FS_IP" : "192.168.0.1",
-  "LOG_INFORMS" : true,
-  "DEBUG" : false
-}

config/ext-sample.js

@@ -1,34 +0,0 @@
-// This is an exmaple GenieACS extension to get the current latitude/longitude
-// of the International Space Station. Why, you ask? Because why not.
-// To install, copy this file to config/ext/iss.js.
-
-"use strict";
-
-const http = require("http");
-
-let cache = null;
-let cacheExpire = 0;
-
-function latlong(args, callback) {
-  if (Date.now() < cacheExpire)
-    return callback(null, cache);
-
-  http.get("http://api.open-notify.org/iss-now.json", (res) => {
-    if (res.statusCode !== 200)
-      return callback(new Error(`Request failed (status code: ${res.statusCode})`));
-
-    let rawData = "";
-    res.on("data", (chunk) => rawData += chunk);
-
-    res.on("end", () => {
-      let pos = JSON.parse(rawData)["iss_position"];
-      cache = [+pos["latitude"], +pos["longitude"]];
-      cacheExpire = Date.now() + 10000;
-      callback(null, cache);
-    });
-  }).on("error", (err) => {
-    callback(err);
-  });
-}
-
-exports.latlong = latlong;

docs/administration-faq.rst

@@ -0,0 +1,64 @@
+.. _administration-faq:
+
+Administration FAQ
+==================
+
+.. _administration-faq-duplicate-log-entries:
+
+Duplicate log entries when using :func:`log()` function
+-------------------------------------------------------
+
+Because GenieACS uses a full fledged scripting language for device
+configuration, the only way to guarantee that it has satisfied the 'desired
+state' is by repeatedly executing the script until there's no more
+discrepancies with the current device state. Though it may seem like this will
+cause duplicate requests going to the device, this isn't actually the case
+because device configuration are stated declaratively and that the scripts
+themselves are pure functions in the context of a session (e.g. Date.now()
+always returns the same value within the session).
+
+To illustrate with an example, consider the following script:
+
+.. code:: javascript
+
+  log("Executing script");
+  declare("Device.param", null, {value: 1});
+  commit();
+  declare("Device.param", null, {value: 2});
+
+This will set the value of the 'Device.param' to 1, then to 2. Then as the
+script is run again the value is set back to 1 and so on. A stable state will
+never be reached so GenieACS will execute the script a few times until it gives
+up and throws a fault. This is an edge case that should be avoided. A more
+typical case is where the script is run once or twice. Essentially if an
+execution doesn't result in any request to the CPE or a change in the data
+model then a stable state is deemed to have been reached.
+
+Configurations not pushed to device after factory reset
+---------------------------------------------------------
+
+After a device is reset to its factory default state, the cached data model in
+GenieACS's database needs to be invalidated to force rediscovery. Ensure the
+following lines are called on ``0 BOOTSTRAP`` event:
+
+.. code:: javascript
+
+  const now = Date.now();
+
+  // Clear cached data model to force a refresh
+  clear("Device", now);
+  clear("InternetGatewayDevice", now);
+
+
+Most device parameters are missing
+----------------------------------
+
+For performance reasons (server, client, and network), GenieACS by default only
+fetches parts of the data model that are necessary to satisfy the declarations
+in your provision scripts. Create declarations for any parameters you need
+fetched by default.
+
+If you're unsure and want to explore the available parameters exposed by the
+device, refresh the root parameter (e.g. ``InternetGatewayDevice``) from
+GenieACS's UI. You typically only need to do that one time for a given CPE
+model.

docs/api-reference.rst

@@ -0,0 +1,538 @@
+API Reference
+=============
+
+GenieACS exposes a rich RESTful API through its NBI component. This document
+serves as a reference for the available APIs.
+
+This API makes use of MongoDB's query language in some of its endpoints. Refer
+to MongoDB's documentation for details.
+
+.. note::
+
+  The examples below use ``curl`` command for simplicity and ease of testing.
+  Query parameters are URL-encoded, but the original pre-encoding values are
+  shown for reference. These examples assume genieacs-nbi is running locally
+  and listening on the default NBI port (7557).
+
+.. warning::
+
+  A common pitfall is not properly percent-encoding special characters in the
+  device ID or query in the URL.
+
+Endpoints
+---------
+
+GET /\<collection\>/?query=\<query\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Search for records in the database (e.g. devices, tasks, presets, files).
+Returns a JSON representation of all items in the given collection that match
+the search criteria.
+
+*collection*: The data collection to search. Could be one of: tasks, devices,
+presets, objects.
+
+*query*: Search query. Refer to MongoDB queries for reference.
+
+Examples
+^^^^^^^^
+
+- Find a device by its ID:
+
+.. code:: javascript
+
+  query = {"_id": "202BC1-BM632w-000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D'
+
+- Find a device by its MAC address:
+
+.. code:: javascript
+
+  query = {
+    "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress": "20:2B:C1:E0:06:65"
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress%22%3A%2220:2B:C1:E0:06:65%22%7D'
+
+- Search for devices that have not initiated an inform in the last 7 days.
+
+.. code:: javascript
+
+  query = {
+    "_lastInform": {
+      "$lt" : "2017-12-11 13:16:23 +0000"
+    }
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_lastInform%22%3A%7B%22%24lt%22%3A%222017-12-11%2013%3A16%3A23%20%2B0000%22%7D%7D'
+
+- Show pending tasks for a given device:
+
+.. code:: javascript
+
+  query = {"device": "202BC1-BM632w-000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-000000%22%7D'
+
+- Return specific parameters for a given device:
+
+.. code:: javascript
+
+  query = {"_id": "202BC1-BM632w-000000"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
+
+The ``projection`` URL param is a comma-separated list of the parameters to receive.
+
+POST /devices/\<device_id\>/tasks?[connection_request]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Enqueue task(s) and optionally trigger a connection request to the device.
+Refer to :ref:`tasks` section for information about the task object format.
+Returns status code 200 if the tasks have been successfully executed, and 202
+if the tasks have been queued to be executed at the next inform.
+
+*device_id*: The ID of the device.
+
+*connection_request*: Indicates that a connection request will be triggered to
+execute the tasks immediately. Otherwise, the tasks will be queued and be
+processed at the next inform.
+
+The response body is the task object as it is inserted in the database. The
+object will include ``_id`` property which you can use to look up the task
+later.
+
+Examples
+^^^^^^^^
+
+- Refresh all device parameters now:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
+  -X POST \
+  --data '{"name": "refreshObject", "objectName": ""}'
+
+- Change WiFi SSID and password:
+
+.. code:: javascript
+
+  {
+    "name": "setParameterValues",
+    "parameterValues": [
+      ["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],
+      ["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
+  -X POST \
+  --data '{"name":"setParameterValues", "parameterValues": [["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]]}'
+
+POST /tasks/\<task_id\>/retry
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Retry a faulty task at the next inform.
+
+*task_id*: The ID of the task as returned by 'GET /tasks' request.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/5403908ef28ea3a25c138adc/retry' -X POST
+
+DELETE /tasks/\<task_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given task.
+
+*task_id*: The ID of the task as returned by 'GET /tasks' request.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/tasks/5403908ef28ea3a25c138adc' -X DELETE
+
+DELETE /faults/\<fault_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given fault.
+
+*fault_id*: The ID of the fault as returned by 'GET /faults' request. The ID
+format is "\<device_id\>:\<channel\>".
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/faults/202BC1-BM632w-000000:default' -X DELETE
+
+DELETE /devices/\<device_id\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete the given device from the database.
+
+Example
+^^^^^^^
+
+.. code:: bash
+
+  curl -X DELETE -i 'http://localhost:7557/devices/202BC1-BM632w-000001'
+
+.. note::
+
+  Note that the device will be registered again when/if it contacts the ACS
+  again (e.g. on the next periodic inform).
+
+POST /devices/\<device_id\>/tags/\<tag\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Assign a tag to a device. Has no effect if such tag already exists.
+
+*device_id*: The ID of the device.
+
+*tag*: The tag to be assigned.
+
+Example
+^^^^^^^
+
+Assign the tag "testing" to a device:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X POST
+
+DELETE /devices/\<device_id\>/tags/\<tag\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Remove a tag from a device.
+
+*device_id*: The ID of the device.
+
+*tag*: The tag to be removed.
+
+Example
+^^^^^^^
+
+Remove the tag "testing" from a device:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X DELETE
+
+PUT /presets/\<preset_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Create or update a preset. Returns status code 200 if the preset has been
+added/updated successfully. The body of the request is a JSON representation of
+the preset. Refer to :ref:`presets` section below for details about its format.
+
+*preset_name*: The name of the preset.
+
+Example
+^^^^^^^
+
+Create a preset to set 5 minutes inform interval for all devices tagged with
+"test":
+
+.. code:: javascript
+
+  query = {
+    "weight": 0,
+    "precondition": "{\"_tags\": \"test\"}"
+    "configurations": [
+      {
+        "type": "value",
+        "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable",
+        "value": "true"
+      },
+      {
+        "type": "value",
+        "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval",
+        "value": "300"
+      }
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/presets/inform' \
+  -X PUT \
+  --data '{"weight": 0, "precondition": "{\"_tags\": \"test\"}", "configurations": [{"type": "value", "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable", "value": "true"}, {"type": "value", "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval", "value": "300"}]}'
+
+DELETE /presets/\<preset_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+	curl -i 'http://localhost:7557/presets/inform' -X DELETE
+
+PUT /files/\<file_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Upload a new file or overwrite an existing one. Returns status code 200 if the
+file has been added/updated successfully. The file content should be sent as
+the request body.
+
+*file_name*: The name of the uploaded file.
+
+The following file metadata may be sent as request headers:
+
+- ``fileType``: For firmware images it should be "1 Firmware Upgrade Image".
+  Other common types are "2 Web Content" and "3 Vendor Configuration File".
+
+- ``oui``: The OUI of the device model that this file belongs to.
+
+- ``productClass``: The product class of the device.
+
+- ``version``: In case of firmware images, this refer to the firmware version.
+
+Example
+^^^^^^^
+
+Upload a firmware image file:
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/files/new_firmware_v1.0.bin' \
+  -X PUT \
+  --data-binary @"./new_firmware_v1.0.bin" \
+  --header "fileType: 1 Firmware Upgrade Image" \
+  --header "oui: 123456" \
+  --header "productClass: ABC" \
+  --header "version: 1.0"
+
+DELETE /files/\<file_name\>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Delete a previously uploaded file:
+
+.. code:: bash
+
+	curl -i 'http://localhost:7557/files/new_firmware_v1.0.bin' -X DELETE
+
+GET /files/
+~~~~~~~~~~~
+
+Gets all previously uploaded files.
+
+GET /files/?query={"filename":"\<filename\>"}
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Find files using a query.
+
+.. _tasks:
+
+Tasks
+-----
+
+Find the different available tasks and their object structure.
+
+``getParameterValues``
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  query = {
+    "name": "getParameterValues",
+    "parameterNames": [
+      "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnectionNumberOfEntries",
+      "InternetGatewayDevice.Time.NTPServer1", "InternetGatewayDevice.Time.Status"
+    ]
+  }
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-96318REF-SR360NA0A4%252D0003196/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "getParameterValues", "parameterNames": ["InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnectionNumberOfEntries", "InternetGatewayDevice.Time.NTPServer1", "InternetGatewayDevice.Time.Status"] }'
+
+You may request a single or multiple parameters at once.
+
+After the task has been executed successfully you can then fetch the CPE object
+and read the parameters from the JSON object.
+
+.. code:: javascript
+
+  query = {"_id": "00236a-96318REF-SR360NA0A4%2D0003196"}
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%2200236a-96318REF-SR360NA0A4%252D0003196%22%7D'
+
+``refreshObject``
+~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "refreshObject", "objectName": "InternetGatewayDevice.WANDevice.1.WANConnectionDevice"}'
+
+``setParameterValues``
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "setParameterValues", "parameterValues": [["InternetGatewayDevice.ManagementServer.UpgradesManaged",false]]}'
+
+Multiple values can be set at once by adding multiple arrays to the
+parameterValues key. For example:
+
+.. code:: javascript
+
+  {
+    name: "setParameterValues",
+    parameterValues: [["InternetGatewayDevice.ManagementServer.UpgradesManaged", false], ["InternetGatewayDevice.Time.Enable", true], ["InternetGatewayDevice.Time.NTPServer1", "pool.ntp.org"]]
+  }
+
+``addObject``
+~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name":"addObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection"}'
+
+``deleteObject``
+~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name":"deleteObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection.1"}'
+
+``reboot``
+~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "reboot"}'
+
+``factoryReset``
+~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "factoryReset"}'
+
+``download``
+~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
+  -X POST \
+  --data '{"name": "download", "file": "mipsbe-6-42-lite.xml"}'
+
+.. _presets:
+
+Presets
+-------
+
+Presets assign a set of configuration or a Provision script to devices based on
+a precondition (search filter), schedule (cron expression), and events.
+
+Precondition
+~~~~~~~~~~~~
+
+The ``precondition`` property is a JSON string representation of the search
+filter to test if the preset applies to a given device. Examples preconditions
+are:
+
+- ``{"param": "value"}``
+- ``{"param": value", "param2": {"$ne": "value2"}}``
+
+Other operators that can be used are ``$gt``, ``$lt``, ``$gte`` and ``$lte``.
+
+Configuration
+~~~~~~~~~~~~~
+
+The configuration property is an array containing the different configurations
+to be applied to a device, as shown below:
+
+.. code:: javascript
+
+  [
+    {
+      "type": "value",
+      "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable",
+      "value": "true"
+    },
+    {
+      "type": "value",
+      "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval",
+      "value": "300"
+    },
+    {
+      "type": "delete_object",
+      "name": "object_parent",
+      "object": "object_name"
+    },
+    {
+      "type": "add_object",
+      "name": "object_parent",
+      "object": "object_name"
+    },
+    {
+      "type": "provision",
+      "name": "YourProvisionName"
+    },
+  ] 
+
+The configuration type ``provision`` triggers a Provision script. In the
+example above, the provision named "YourProvisionName" will be executed.
+
+Provisions
+----------
+
+Create a provision
+~~~~~~~~~~~~~~~~~~
+
+The Provision's JavaScript code is the body of the HTTP PUT request.
+
+.. code:: bash
+
+  curl -X PUT -i 'http://localhost:7557/provisions/mynewprovision' --data 'log("Provision started at " + now);'
+
+Delete a provision
+~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+  curl -X DELETE -i 'http://localhost:7557/provisions/mynewprovision'
+
+Get provisions
+~~~~~~~~~~~~~~
+
+Get all provisions:
+
+.. code:: bash
+
+  curl -X GET -i 'http://localhost:7557/provisions/'

docs/conf.py

@@ -0,0 +1,65 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+import json
+
+# -- Project information -----------------------------------------------------
+
+project = 'GenieACS Documentation'
+copyright = '2019, GenieACS Inc.'
+author = 'GenieACS Inc.'
+
+# The full version, including alpha/beta/rc tags
+release = json.load(open("../package.json"))["version"]
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+# html_theme = 'alabaster'
+html_theme = 'sphinx_rtd_theme'
+html_logo = "logo.svg"
+html_theme_options = {
+  "logo_only": True
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+master_doc = 'index'
+
+highlight_language = 'javascript'

docs/cpe-authentication.rst

@@ -0,0 +1,72 @@
+.. _cpe-authentication:
+
+CPE Authentication
+==================
+
+CPE to ACS
+----------
+
+.. note::
+
+  By default GenieACS will accept any incoming connection via HTTP/HTTPS and
+  respond to it.
+
+The following parameters are used to set and get (password is redacted but
+can be set) the username/password used to authenticate against the ACS:
+
+Username: ``Device.ManagementServer.Username`` or ``InternetGatewayDevice.ManagementServer.Username``
+
+Password: ``Device.ManagementServer.Password`` or ``InternetGatewayDevice.ManagementServer.Password``
+
+Enable CPE to ACS Authentication
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+CPE to ACS authentication can be configured in the web interface by using the
+`Config` option in the `Admin` tab.
+
+Go to the `Admin` -> `Config` page and click on `New config` button at the
+bottom of the page. This will open pop-up which requires you to fill in a key
+and value. The key should be ``cwmp.auth``. The value accepts a boolean.
+Setting the value to ``true`` makes it so that GenieACS accepts any incoming
+connection, setting it to ``false`` makes GenieACS deny all incoming
+connections. This can be further configured using the ``AUTH()`` and ``EXT()``
+functions.
+
+The ``AUTH()`` function
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``AUTH()`` function accepts two parameters, username and password. It
+checks the given username and password with the incoming request to determine
+whether to return true or false.
+
+Basic usage of the ``AUTH()`` function could be as follows:
+
+.. code:: sql
+
+   AUTH("fixed-username", "fixed-password")
+
+This will only accept incoming request who authenticate with
+"fixed-username" and "fixed-password".
+
+The various device parameters can be referenced from within the ``cwmp.auth``
+expression. For example:
+
+.. code:: sql
+
+   AUTH(Device.ManagementServer.Username, Device.ManagementServer.Password)
+
+The ``EXT()`` function
+~~~~~~~~~~~~~~~~~~~~~~
+
+The ``EXT()`` function makes it possible to call an :ref:`extension
+<extensions>` script from the auth expression. This can be used to fetch
+the credentials from an external source:
+
+.. code:: sql
+
+   AUTH(DeviceID.SerialNumber, EXT("authenticate", "getPassword", DeviceID.SerialNumber))
+
+ACS to CPE
+----------
+
+TODO

docs/environment-variables.rst

@@ -0,0 +1,217 @@
+.. _environment-variables:
+
+Environment Variables
+=====================
+
+Configuring GenieACS services can be done through the following environment
+variables:
+
+.. attention::
+
+  All GenieACS environment variables must be prefixed with ``GENIEACS_``.
+
+MONGODB_CONNECTION_URL
+  MongoDB connection string.
+
+  Default: ``mongodb://127.0.0.1/genieacs``
+
+EXT_DIR
+  The directory from which to look up extension scripts.
+
+  Default: ``<installation dir>/config/ext``
+
+EXT_TIMEOUT
+  Timeout (in milliseconds) to allow for calls to extensions to return a
+  response.
+
+  Default: ``3000``
+
+DEBUG_FILE
+  File to dump CPE debug log.
+
+  Default: unset
+
+DEBUG_FORMAT
+  Debug log format. Valid values are 'yaml' and 'json'.
+
+  Default: ``yaml``
+
+LOG_FORMAT
+  The format used for the log entries in ``CWMP_LOG_FILE``, ``NBI_LOG_FILE``,
+  ``FS_LOG_FILE``, and ``UI_LOG_FILE``. Possible values are ``simple`` and
+  ``json``.
+
+  Default: ``simple``
+
+ACCESS_LOG_FORMAT
+  The format used for the log entries in ``CWMP_ACCESS_LOG_FILE``,
+  ``NBI_ACCESS_LOG_FILE``, ``FS_ACCESS_LOG_FILE``, and ``UI_ACCESS_LOG_FILE``.
+  Possible values are ``simple`` and ``json``.
+
+  Default: ``simple``
+
+CWMP_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-cwmp. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+CWMP_PORT
+  The TCP port that genieacs-cwmp listens on.
+
+  Default: ``7547``
+
+CWMP_INTERFACE
+  The network interface that genieacs-cwmp binds to.
+
+  Default: ``0.0.0.0``
+
+CWMP_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+CWMP_LOG_FILE
+  File to log process related events for genieacs-cwmp. If omitted, logs will
+  go to stderr.
+
+  Default: unset
+
+CWMP_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-cwmp. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+NBI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-nbi. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+NBI_PORT
+  The TCP port that genieacs-nbi listens on.
+
+  Default: ``7557``
+
+NBI_INTERFACE
+  The network interface that genieacs-nbi binds to.
+
+  Default: ``0.0.0.0``
+
+NBI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+NBI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+NBI_LOG_FILE
+  File to log process related events for genieacs-nbi. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+NBI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-nbi. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-fs. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+FS_PORT
+  The TCP port that genieacs-fs listens on.
+
+  Default: ``7567``
+
+FS_INTERFACE
+  The network interface that genieacs-fs binds to.
+
+  Default: ``0.0.0.0``
+
+FS_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+FS_LOG_FILE
+  File to log process related events for genieacs-fs. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+FS_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-fs. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+FS_URL_PREFIX
+  The URL prefix (e.g. 'https://example.com:7657/') to use when generating the
+  file URL for TR-069 Download requests. Set this if genieacs-fs and
+  genieacs-cwmp are behind a proxy or running on different servers.
+
+  Default: auto generated based on the hostname from the ACS URL, FS_PORT
+  config, and whether or not SSL is enabled for genieacs-fs.
+
+UI_WORKER_PROCESSES
+  The number of worker processes to spawn for genieacs-ui. A value of 0 means
+  as many as there are CPU cores available.
+
+  Default: ``0``
+
+UI_PORT
+  The TCP port that genieacs-ui listens on.
+
+  Default: ``3000``
+
+UI_INTERFACE
+  The network interface that genieacs-ui binds to.
+
+  Default: ``0.0.0.0``
+
+UI_SSL_CERT
+  Path to certificate file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_SSL_KEY
+  Path to certificate key file. If omitted, non-secure HTTP will be used.
+
+  Default: unset
+
+UI_LOG_FILE
+  File to log process related events for genieacs-ui. If omitted, logs will go
+  to stderr.
+
+  Default: unset
+
+UI_ACCESS_LOG_FILE
+  File to log incoming requests for genieacs-ui. If omitted, logs will go to
+  stdout.
+
+  Default: unset
+
+UI_JWT_SECRET
+  The key used for signing JWT tokens that are stored in browser cookies. The
+  string can be up to 64 characters in length.
+
+  Default: unset

docs/ext-sample.js

@@ -0,0 +1,37 @@
+// This is an example GenieACS extension to get the current latitude/longitude
+// of the International Space Station. Why, you ask? Because why not.
+// To install, copy this file to config/ext/iss.js.
+
+"use strict";
+
+const http = require("http");
+
+let cache = null;
+let cacheExpire = 0;
+
+function latlong(args, callback) {
+  if (Date.now() < cacheExpire) return callback(null, cache);
+
+  http
+    .get("http://api.open-notify.org/iss-now.json", (res) => {
+      if (res.statusCode !== 200)
+        return callback(
+          new Error(`Request failed (status code: ${res.statusCode})`)
+        );
+
+      let rawData = "";
+      res.on("data", (chunk) => (rawData += chunk));
+
+      res.on("end", () => {
+        let pos = JSON.parse(rawData)["iss_position"];
+        cache = [+pos["latitude"], +pos["longitude"]];
+        cacheExpire = Date.now() + 10000;
+        callback(null, cache);
+      });
+    })
+    .on("error", (err) => {
+      callback(err);
+    });
+}
+
+exports.latlong = latlong;

docs/extensions.rst

@@ -0,0 +1,33 @@
+.. _extensions:
+
+Extensions
+==========
+
+Given that :ref:`provisions` and :ref:`virtual-parameters` are executed in a
+sandbox environment, it is not possible to interact with external sources or
+execute any action that requires OS, file system, or network access. Extensions
+exist to bridge that gap.
+
+Extensions are fully-privileged Node.js modules and as such have access to
+standard Node libraries and 3rd party packages. Functions exposed by the
+extension can be called from Provision scripts using the ``ext()`` function. A
+typical use case for extensions is fetching credentials from a database to have
+that pushed to the device during provisioning.
+
+By default, the extension JS code must be placed under ``config/ext``
+directory. You may need to create that directory if it doesn't already exist.
+
+The example extension below fetches data from an external REST API and returns
+that to the caller:
+
+.. literalinclude:: ext-sample.js
+  :language: javascript
+
+To call this extension from a Provision or a Virtual Parameter script:
+
+.. code:: javascript
+
+  // The arguments "arg1" and "arg2" are passed to the latlong. Though they are
+  // unused in this particular example.
+  const res = ext("ext-sample", "latlong", "arg1", "arg2");
+  log(JSON.stringify(res));

docs/https.rst

@@ -0,0 +1,6 @@
+.. _https:
+
+HTTPS
+=====
+
+TODO

docs/index.rst

@@ -0,0 +1,56 @@
+.. GenieACS documentation master file, created by
+   sphinx-quickstart on Wed Jun  5 13:47:06 2019.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to GenieACS's documentation!
+====================================
+
+.. toctree::
+  :caption: Table of Contents
+
+.. raw:: latex
+
+   \part{Installation}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Installation
+
+  installation-guide
+  environment-variables
+
+.. raw:: latex
+
+   \part{Administration}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Administration
+
+  provisions
+  virtual-parameters
+  administration-faq
+
+.. raw:: latex
+
+   \part{Integration}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Integration
+
+  extensions
+  api-reference
+
+.. raw:: latex
+
+   \part{Security}
+
+.. toctree::
+  :maxdepth: 1
+  :caption: Security
+
+  https
+  cpe-authentication
+  roles-and-permissions

docs/installation-guide.rst

@@ -0,0 +1,233 @@
+Installation Guide
+==================
+
+This guide is for installing GenieACS on a single server on any Linux distro
+that uses *systemd* as its init system.
+
+The various GenieACS services are independent of each other and may be
+installed on different servers. You may also run multiple instances of each in
+a load-balancing/failover setup.
+
+.. attention::
+
+  For production deployments make sure to configure TLS and change
+  ``UI_JWT_SECRET`` to a unique and secure string. Refer to :ref:`https`
+  section for how to enable TLS to encrypt traffic.
+
+Prerequisites
+-------------
+
+.. topic:: Node.js
+
+  GenieACS requires Node.js 10.x and up. Refer to https://nodejs.org/ for
+  instructions.
+
+.. topic:: MongoDB
+
+  GenieACS requires MongoDB 3.6 and up. Refer to https://www.mongodb.com/ for
+  instructions.
+
+Install GenieACS
+-------------------
+
+.. topic:: Installing from NPM:
+
+  .. parsed-literal::
+
+    sudo npm install -g genieacs@\ |release|
+
+.. topic:: Installing from source
+
+  If you prefer installing from source, such as when running a GenieACS copy
+  with custom patches, refer to README.md file in the source package. Adjust
+  the next steps below accordingly.
+
+Configure systemd
+-----------------
+
+.. topic:: Create a system user to run GenieACS daemons
+
+  .. code:: bash
+
+    sudo useradd --system --no-create-home --user-group genieacs
+
+.. topic:: Create directory to save extensions and environment file
+
+  We'll use :file:`/opt/genieacs/ext/` directory to store extension scripts (if any).
+
+  .. code:: bash
+    
+    mkdir /opt/genieacs
+    mkdir /opt/genieacs/ext
+    chown genieacs:genieacs /opt/genieacs/ext
+
+  Create the file :file:`/opt/genieacs/genieacs.env` to hold our configuration
+  options which we pass to GenieACS as environment variables. See
+  :ref:`environment-variables` section for a list of all available
+  configuration options.
+
+  .. code:: bash
+
+    GENIEACS_CWMP_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-cwmp-access.log
+    GENIEACS_NBI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-nbi-access.log
+    GENIEACS_FS_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-fs-access.log
+    GENIEACS_UI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-ui-access.log
+    GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
+    NODE_OPTIONS=--enable-source-maps
+    GENIEACS_EXT_DIR=/opt/genieacs/ext
+    GENIEACS_UI_JWT_SECRET=secret
+  
+  Set file ownership and permissions:
+
+  .. code:: bash
+
+    sudo chown genieacs:genieacs /opt/genieacs/genieacs.env
+    sudo chmod 600 /opt/genieacs/genieacs.env
+
+.. topic:: Create logs directory
+
+  .. code:: bash
+    
+    mkdir /var/log/genieacs
+    chown genieacs:genieacs /var/log/genieacs
+
+.. topic:: Create systemd unit files
+
+  Create a systemd unit file for each of the four GenieACS services. Note that
+  we're using EnvironmentFile directive to read the environment variables from
+  the file we created earlier.
+
+  Each service has two streams of logs: access log and process log. Access logs
+  are configured here to be dumped in a log file under
+  :file:`/var/log/genieacs/` while process logs go to *journald*. Use
+  ``journalctl`` command to view process logs.
+
+  .. attention::
+
+    If the command :command:`systemctl edit --force --full` fails, you can
+    create the unit file manually.
+
+  1. Run the following command to create ``genieacs-cwmp`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-cwmp
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS CWMP
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-cwmp
+
+      [Install]
+      WantedBy=default.target
+
+  2. Run the following command to create ``genieacs-nbi`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-nbi
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS NBI
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-nbi
+
+      [Install]
+      WantedBy=default.target
+
+  3. Run the following command to create ``genieacs-fs`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-fs
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS FS
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-fs
+
+      [Install]
+      WantedBy=default.target
+
+  4. Run the following command to create ``genieacs-ui`` service:
+  
+    .. code:: bash
+
+      sudo systemctl edit --force --full genieacs-ui
+    
+    Then paste the following in the editor and save:
+
+    .. code:: cfg
+
+      [Unit]
+      Description=GenieACS UI
+      After=network.target
+
+      [Service]
+      User=genieacs
+      EnvironmentFile=/opt/genieacs/genieacs.env
+      ExecStart=/usr/bin/genieacs-ui
+
+      [Install]
+      WantedBy=default.target
+
+.. topic:: Configure log file rotation using logrotate
+
+  Save the following as :file:`/etc/logrotate.d/genieacs`
+
+  .. code::
+  
+    /var/log/genieacs/*.log /var/log/genieacs/*.yaml {
+        daily
+        rotate 30
+        compress
+        delaycompress
+        dateext
+    }
+
+.. topic:: Enable and start services
+
+  .. code:: bash
+
+    sudo systemctl enable genieacs-cwmp
+    sudo systemctl start genieacs-cwmp
+    sudo systemctl status genieacs-cwmp
+
+    sudo systemctl enable genieacs-nbi
+    sudo systemctl start genieacs-nbi
+    sudo systemctl status genieacs-nbi
+
+    sudo systemctl enable genieacs-fs
+    sudo systemctl start genieacs-fs
+    sudo systemctl status genieacs-fs
+
+    sudo systemctl enable genieacs-ui
+    sudo systemctl start genieacs-ui
+    sudo systemctl status genieacs-ui
+
+  Review the status message for each to verify that the services are running
+  successfully.

docs/logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="310" height="64" viewBox="0 0 82.021 16.933"><path d="M7.341 9.892l.687 1.307c-.305.53-.718.99-1.238 1.378-.851.503-1.543.421-2.076-.244 1.03-.466 1.907-1.28 2.627-2.44M4.676 5.97a2.144 2.144 0 0 1 .237 1.228 8.358 8.358 0 0 1-.518 2.21 7.386 7.386 0 0 0-.424 1.708 2.38 2.38 0 0 0 .168 1.205c.163.393.457.73.883 1.01-.458.138-.859.175-1.202.112-.6-.114-.992-.476-1.175-1.084a3.571 3.571 0 0 1-.093-1.787 11.742 11.742 0 0 1 .435-1.644 11.606 11.606 0 0 0 .432-1.708c.037-.22.041-.45.01-.687a1.66 1.66 0 0 0-.273-.706 1.582 1.582 0 0 0-.6-.506c.245-.046.487-.083.728-.113a1.45 1.45 0 0 1 .728.123c.295.13.517.343.665.639m7.901-1.802l.222.218-1.847 1.678.555.555 1.678-1.846.218.22-1.843 1.679.521.521L13.76 5.35l.222.218-1.847 1.678.556.556 1.764-1.765.375.368-1.584 1.584h.004l-2.065 2.061-1.253.515-1.795-1.348-.007-.008-.008.008.004.007 1.194 1.592-.398.161-1.75-1.75.444-1.08 1.636 1.228.004.003.011-.007-.003-.004-1.39-1.85.225-.553 1.694-1.689h-.004l1.956-1.952.372.368-1.769 1.768.552.552 1.678-1.843m1.456 9.91c1.55-1.549 2.324-3.42 2.324-5.612 0-2.187-.774-4.058-2.323-5.612C12.48 1.304 10.61.53 8.423.53 6.23.53 4.36 1.305 2.81 2.854c-.95.953-1.609 2.025-1.974 3.213l.158-.086c.788-.415 1.486-.203 2.094.638-.55.113-.992.232-1.325.357a1.923 1.923 0 0 0-.503.289c-.337.268-.55.608-.638 1.02a3.98 3.98 0 0 0 .011 1.761c.285 1.527 1.011 2.87 2.178 4.032 1.549 1.552 3.42 2.328 5.612 2.328 2.19 0 4.06-.776 5.612-2.328" fill="white"/><path d="M47.104 1.767c-.456 0-.816.388-.816.849 0 .46.355.816.816.816a.82.82 0 0 0 .833-.816.85.85 0 0 0-.833-.849zM25.602 3.171c-.205 0-.382.046-.5.13-.118.085-.187.201-.187.4v.981l-.151-.031a9.89 9.89 0 0 0-1.872-.173c-1.288 0-2.148.066-2.657.374a1.264 1.264 0 0 0-.548.676c-.119.313-.177.73-.177 1.28v6.123c0 .982.25 1.508.787 1.824.539.315 1.407.41 2.628.41 1.308 0 2.178-.13 2.701-.484.524-.354.745-.932.745-1.942 0-.821-.147-1.37-.543-1.722-.395-.352-1.075-.528-2.186-.528h-2.787V9.255l.148.03c.478.092 1.094.11 1.89.11 1.292 0 2.148-.112 2.658-.453.51-.34.724-.915.724-1.958V3.7c0-.198-.07-.314-.186-.399a.843.843 0 0 0-.487-.13zm6.112 1.29c-1.226 0-2.038.137-2.526.495-.488.357-.697.945-.697 1.947v2.84c0 .935.23 1.54.743 1.932.512.39 1.337.571 2.544.571 1.392 0 2.17-.058 2.567-.196.199-.069.298-.15.355-.246.058-.097.078-.227.078-.405 0-.208-.04-.378-.117-.485-.076-.108-.186-.171-.396-.171-.794 0-1.057.19-2.408.19-.593 0-1.06-.078-1.395-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.717.717 0 0 0 .539-.213c.13-.138.213-.342.213-.62V6.903c0-1.037-.179-1.623-.654-1.97-.476-.348-1.3-.472-2.617-.472zm8.83 0c-1.303 0-2.117.086-2.59.49-.472.403-.665 1.168-.665 2.623v4.002c0 .215.074.338.197.426.123.088.31.134.524.134.223 0 .412-.046.537-.134.125-.088.198-.211.198-.426v-3.81c0-.424.006-.754.04-1.019.036-.265.102-.47.236-.623.135-.152.328-.236.574-.28.246-.044.552-.053.949-.053.4 0 .712.01.963.053.25.044.447.126.587.278.139.151.211.358.25.623.038.265.046.597.046 1.02v3.81c0 .217.07.34.19.427.122.088.308.134.53.134.215 0 .401-.046.525-.134.123-.088.196-.211.196-.426V7.574c0-1.455-.201-2.219-.682-2.623-.48-.404-1.303-.49-2.605-.49zm12.996 0c-1.227 0-2.038.137-2.527.495-.488.357-.696.945-.696 1.947v2.84c0 .935.23 1.54.742 1.932.513.39 1.338.571 2.545.571 1.392 0 2.17-.058 2.567-.196.2-.069.298-.15.355-.246.058-.097.079-.227.079-.405 0-.208-.04-.378-.117-.485-.077-.108-.188-.171-.398-.171-.793 0-1.056.19-2.406.19-.594 0-1.062-.078-1.396-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.716.716 0 0 0 .539-.213c.13-.138.213-.342.213-.62V6.903c0-1.037-.18-1.623-.655-1.97-.475-.348-1.3-.472-2.616-.472zm-6.436.13c-.212 0-.398.047-.522.14-.124.09-.198.22-.198.435v6.41c0 .215.073.338.196.426.123.088.31.134.524.134.223 0 .414-.046.54-.134.124-.088.198-.211.198-.426v-6.41c0-.215-.075-.342-.201-.434-.126-.092-.316-.142-.537-.142zM22.892 5.726c.662 0 1.4.065 1.918.146l.105.016v.92c0 .246-.01.453-.058.632a.757.757 0 0 1-.292.441c-.305.216-.798.263-1.673.263h-.123v-.013c-.761 0-1.26-.002-1.55-.16a.637.637 0 0 1-.307-.395c-.049-.17-.057-.37-.057-.625v-.062c0-.227.03-.418.108-.58a.811.811 0 0 1 .373-.373c.337-.171.826-.21 1.556-.21zm8.839.049c.677 0 1.116.054 1.412.237a.775.775 0 0 1 .316.373c.063.154.085.33.085.535v.811h-3.611V6.92c0-.437.135-.755.439-.928.304-.174.739-.217 1.359-.217zm21.825 0c.677 0 1.117.054 1.413.237a.779.779 0 0 1 .318.373c.062.154.084.33.084.535v.811h-3.613V6.92c0-.437.136-.755.44-.928.304-.174.738-.217 1.358-.217zm-32.701 5.867h2.388c.613 0 1.05.045 1.355.226.306.18.443.505.443.95 0 .429-.179.743-.533.907-.354.164-.861.206-1.568.206-.652 0-1.151-.045-1.51-.186a.952.952 0 0 1-.425-.3.789.789 0 0 1-.15-.483v-1.195z" fill="white"/><path d="M62.339 4.462c-1.399 0-2.18.078-2.58.225-.2.073-.3.157-.357.254-.058.097-.079.22-.079.386 0 .199.043.362.122.469.079.107.19.172.392.172.38 0 .616-.048.947-.096a9.457 9.457 0 0 1 1.475-.098c.58 0 .966.043 1.233.192a.685.685 0 0 1 .292.312c.06.13.082.281.082.452v.857h-2.484c-1.134 0-1.763.19-2.11.559-.346.367-.443.95-.443 1.802 0Line truncated

docs/provisions.rst

@@ -0,0 +1,279 @@
+.. _provisions:
+
+Provisions
+==========
+
+A Provision is a piece of JavaScript code that is executed on the server on a
+per-device basis. It enables implementing complex provisioning scenarios and
+other operations such as automated firmware upgrade rollout. Apart from a few
+special functions, the script is essentially a standard ES6 code executed in
+strict mode.
+
+Provisions are mapped to devices using presets. Note that the added performance
+overhead when using Provisions as opposed to simple preset configuration
+entries is relatively small. Anything that can be done via preset
+configurations can be done using a Provision script. In fact, the now
+deprecated configuration format is still supported primarily for backward
+compatibility and it is recommended to use Provision scripts for all
+configuration.
+
+When assigning a Provision script to a preset, you may pass arguments to the
+script. The arguments can be accessed from the script through the global
+``args`` variable.
+
+.. note::
+
+  Provision scripts may get executed multiple times in a given session.
+  Although all data model-mutating operations are idempotent, a script as a
+  whole may not be. It is, therefore, necessary to repeatedly run the script
+  until there are no more side effects and a stable state is reached.
+
+Built-in functions
+------------------
+
+``declare(path, timestamps, values)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This function is for declaring parameter values to be set, as well as specify
+constraints on how recent you'd like the parameter value (or other attributes)
+to have been refreshed from the device. If the given timestamp is lower than
+the timestamp of the last refresh from the device, then this function will
+return the last known value. Otherwise, the value will be fetched from the
+device before being returned to the caller.
+
+The timestamp argument is an object where the key is the attribute name (e.g.
+``value``, ``object``, ``writable``, ``path``) and the value is an integer
+representing a Unix timestamp.
+
+The values argument is an object similar to the timestamp argument but its
+property values being the parameter values to be set.
+
+The possible attributes in 'timestamps' and 'values' arguments are:
+
+- ``value``: a [<value>, <type>] pair
+
+This attribute is not available for objects or object instances. If the value
+is not a [<value>, <type>] array then it'll assumed to be a value without a
+type and therefore the type will be inferred from the parameter's type.
+
+- ``writable``: boolean
+
+The meaning of this attribute can vary depending on the type of the parameter.
+In the case of regular parameters, it indicates if its value is writable. In
+the case of objects, it's whether or not it's possible to add new object
+instances. In the case of object instances, it indicates whether or not this
+instance can be deleted.
+
+- ``object``: boolean
+
+True if this is an object or object instance, false otherwise.
+
+- ``path``: string
+
+This attribute is special in that it's not a parameter attribute per se, but it
+refers to the presence of parameters matching the given path. For example,
+given the following wildcard path:
+
+``InternetGatewayDevice.LANDevice.1.Hosts.Host.*.MACAddress``
+
+Using a recent timestamp for path in ``declare()`` will result in a sync with
+the device to rediscover all Host instances (``Host.*``). The path attribute
+can also be used to create or delete object instances as described in
+:ref:`path-format` section.
+
+The return value of ``declare()`` is an iterator to access parameters that
+match the given path. Each item in the iterator has the attribute 'path' in
+addition to any other attribute given in the ``declare()`` call. The iterator
+object itself has convenience attribute accessors which come in handy when
+you're expecting a single parameter (e.g. when path does not contain wildcards
+or aliases).
+
+.. code:: javascript
+
+  // Example: Setting the SSID as the last 6 characters of the serial number
+  let serial = declare("Device.DeviceInfo.SerialNumber", {value: 1});
+  declare("Device.LANDevice.1.WLANConfiguration.1.SSID", null, {value: serial.value[0]});
+
+``clear(path, timestamp)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This function invalidates the database copy of parameters (and their child
+parameters) that match the given path and have a last refresh timestamp that is
+less than the given timestamp. The most obvious use for this function is to
+invalidate the database copy of the entire data model after the device has been
+factory reset:
+
+.. code:: javascript
+
+  // Example: Clear cached device data model Note
+  // Make sure to apply only on "0 BOOTSTRAP" event
+  clear("Device", Date.now());
+  clear("InternetGatewayDevice", Date.now());
+
+``commit()``
+~~~~~~~~~~~~
+
+This function commits the pending declarations and performs any necessary sync
+with the device. It's usually not required to call this function as it called
+implicitly at the end of the script and when accessing any property of the
+promise-like object returned by the ``declare()`` function. Calling this
+explicitly is only necessary if you want to control the order in which
+parameters are configured.
+
+``ext(file, function, arg1, arg2, ...)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Execute an extension script and return the result. The first argument is the
+script filename while second argument is the function name within that script.
+Any remaining arguments will be passed to that function. See :ref:`extensions`
+for more details.
+
+``log(message)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Prints out a string in genieacs-cwmp's access log. It's meant to be used for
+debugging. Note that you may see multiple log entries as the script can be
+executed multiple times in a session. See :ref:`this FAQ
+<administration-faq-duplicate-log-entries>`.
+
+.. _path-format:
+
+Path format
+-----------
+
+A parameter path may contain a wildcard (``*``) or an alias filter
+(``[name:value]``). A wildcard segment in a parameter path will apply the
+declared configuration to zero or more parameters that match the given path
+where the wildcard segment can be anything.
+
+An alias filter is like a wildcard, but additionally performs filtering on the
+child parameters based on the key-value pairs provided. For example, the
+following path:
+
+``Device.WANDevice.1.WANConnectionDevice.1.WANIPConnection.[AddressingType:DHCP].ExternalIPAddress``
+
+will return a list of ExternalIPAddress parameters (0 or more) where the
+sibling parameter AddressingType is assigned the value "DHCP".
+
+This can be useful when the exact instance numbers may be different from one
+device to another. It is possible to use more than one key-value pair in the
+alias filter. It's also possible to use multiple filters or use a combination
+of filters and wildcards.
+
+Creating/deleting object instances
+----------------------------------
+
+Given the declarative nature of provisions, we cannot explicitly tell the
+device to create or delete an instance under a given object. Instead, we
+specify the number of instances we want there to be, and based on that GenieACS
+will determine whether or not it needs to create or delete instances. For
+example, the following declaration will ensure we have one and only one
+WANIPConnection object:
+
+.. code:: javascript
+
+  // Example: Ensure we have one and only one WANIPConnection object
+  declare("InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.*", null, {path: 1});
+
+Note the wildcard at the end of the parameter path.
+
+It is also possible to use alias filters as the last path segment which will
+ensure that the declared number of instances is satisfied given the alias
+filter:
+
+.. code:: javascript
+
+  // Ensure that *all* other instances are deleted
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[]", null, {path: 0});
+
+  // Add the two entries we care about
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[SourceIPAddress:192.168.1.0,SourceNetMask:255.255.255.0]",  {path: now}, {path: 1});
+  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[SourceIPAddress:172.16.12.0,SourceNetMask:255.255.0.0]", {path: now}, {path: 1});
+
+Special GenieACS parameters
+---------------------------
+
+In addition to the parameters exposed in the device's data model through
+TR-069, GenieACS has its own set of special parameters:
+
+``DeviceID``
+~~~~~~~~~~~~
+
+This parameter sub-tree includes the following read-only parameters:
+
+- ``DeviceID.ID``
+- ``DeviceID.SerialNumber``
+- ``DeviceID.ProductClass``
+- ``DeviceID.OUI``
+- ``DeviceID.Manufacturer``
+
+``Tags``
+~~~~~~~~
+
+The ``Tags`` root parameter is used to expose device tags in the data model.
+Tags appear as child parameters that are writable and have boolean value.
+Setting a tag to ``false`` will delete that tag, and setting the value of a
+non-existing tag parameter to ``true`` will create it.
+
+.. code:: javascript
+
+  // Example: Remove "tag1", add "tag2", and read "tag3"
+  declare("Tags.tag1", null, {value: false});
+  declare("Tags.tag2", null, {value: true});
+  let tag3 = declare("Tags.tag3", {value: 1});
+
+``Reboot``
+~~~~~~~~~~
+
+The ``Reboot`` root parameter hold the timestamp of the last reboot command.
+The parameter value is writable and declaring a timestamp value that is larger
+than the current value will trigger a reboot.
+
+.. code:: javascript
+
+  // Example: Reboot the device only if it hasn't been rebooted in the past 300 seconds
+  declare("Reboot", null, {value: Date.now() - (300 * 1000)});
+
+``FactoryReset``
+~~~~~~~~~~~~~~~~
+
+Works like ``Reboot`` parameter but for factory reset.
+
+.. code:: javascript
+
+  // Example: Default the device to factory settings
+  declare("FactoryReset", null, {value: Date.now()});
+
+``Downloads``
+~~~~~~~~~~~~~
+
+The ``Downloads`` sub-tree holds information about the last download
+command(s). A download command is represented as an instance (e.g.
+``Downloads.1``) containing parameters such as ``Download`` (timestamp),
+``LastFileType``, ``LastFileName``. The parameters ``FileType``, ``FileName``,
+``TargetFileName`` and ``Download`` are writable and can be used to trigger a
+new download.
+
+.. code:: javascript
+
+  declare("Downloads.[FileType:1 Firmware Upgrade Image]", {path: 1}, {path: 1});
+  declare("Downloads.[FileType:1 Firmware Upgrade Image].FileName", {value: 1}, {value: "firmware-2017.01.tar"});
+  declare("Downloads.[FileType:1 Firmware Upgrade Image].Download", {value: 1}, {value: Date.now()});
+
+Common file types are:
+
+- ``1 Firmware Upgrade Image``
+- ``2 Web Content``
+- ``3 Vendor Configuration File``
+- ``4 Tone File``
+- ``5 Ringer File``
+
+.. warning::
+
+  Pushing a file to the device is often a service-interrupting operation. It's
+  recommended to only trigger it on certain events such as ``1 BOOT`` or during
+  a predetermined maintenance window).
+
+After the CPE had finished downloading and applying the config file, it will
+send a ``7 TRANSFER COMPLETE`` event. You may use that to trigger a reboot
+after the firmware image or configuration file had been applied.

docs/roles-and-permissions.rst

@@ -0,0 +1,6 @@
+.. _roles-and-permissions:
+
+Roles and Permissions
+=====================
+
+TODO

docs/virtual-parameters.rst

@@ -0,0 +1,105 @@
+.. _virtual-parameters:
+
+Virtual Parameters
+==================
+
+Virtual parameters are user-defined parameters whose values are generated using
+a custom Javascript code. Virtual parameters behave just like regular
+parameters and appear in the data model under ``VirtualParameters.`` path.
+Virtual parameter names cannot contain a period (``.``).
+
+The execution environment for virtual parameters is almost identical to that of
+provisions. See :ref:`provisions` for more details and examples. The only
+differences between the scripts of provisions and virtual parameters are:
+
+- You can't pass custom arguments to virtual parameter scripts. Instead, the
+  variable ``args`` will hold the current vparam timestamps and values as well
+  as the declared timestamps and values. Like this:
+
+.. code:: javascript
+
+  // [<declared attr timestamps, declared attr values>, <current attr timestamps>, <current attr values>]
+  [{path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}, {path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}]
+
+- Virtual parameter scripts must return an object containing the attributes of
+  this parameter.
+
+.. note::
+
+  Just like a regular parameter, creating a virtual parameter does not
+  automatically add it to the parameter list for a device. It needs to fetched
+  (manually or via a preset) before you can see it in the data model.
+
+Examples
+--------
+
+Unified MAC parameter across different device models
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Unified MAC parameter across different device models
+  let m = "00:00:00:00:00:00";
+  let d = declare("Device.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress", {value: Date.now()});
+  let igd = declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANPPPConnection.*.MACAddress", {value: Date.now()});
+
+  if (d.size) {
+    for (let p of d) {
+      if (p.value[0]) {
+        m = p.value[0];
+        break;
+      }
+    }  
+  }
+  else if (igd.size) {
+    for (let p of igd) {
+      if (p.value[0]) {
+        m = p.value[0];
+        break;
+      }
+    }  
+  }
+
+  return {writable: false, value: [m, "xsd:string"]};
+
+Expose an external value as a virtual parameter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Expose an external value as a virtual parameter
+  let serial = declare("DeviceID.SerialNumber", {value: 1});
+  if (args[1].value) {
+    ext("example-ext", "set", serial.value[0], args[1].value[0]);
+    return {writable: true, value: [args[1].value[0], "xsd:string"]};
+  }
+  else {
+    let v = ext("example-ext", "get", serial.value[0]);
+    return {writable: true, value: [v, "xsd:string"]};
+  }
+
+Create an editable virtual parameter for WPA passphrase
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: javascript
+
+  // Example: Create an editable virtual parameter for WPA passphrase
+  let m = "";
+  if (args[1].value) {
+    m = args[1].value[0];
+    declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", null, {value: m});
+    declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", null, {value: m});
+  }
+  else {
+    let d = declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", {value: Date.now()});
+    let igd = declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", {value: Date.now()});
+
+    if (d.size) {
+      m = d.value[0];
+    }
+    else if (igd.size) {
+      m = igd.value[0];  
+    }
+  }
+
+  return {writable: true, value: [m, "xsd:string"]};

lib/api-functions.coffee

@@ -1,254 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-crypto = require 'crypto'
-dgram = require 'dgram'
-
-config = require './config'
-db = require './db'
-http = require 'http'
-common = require './common'
-util = require 'util'
-URL = require 'url'
-auth = require './auth'
-
-
-udpConReq = (address, un, key, callback) ->
-  return false if not address
-  [host, port] = address.split(':', 2)
-  if port is undefined
-    port = 80
-  else
-    port = parseInt(port)
-    return false if isNaN(port)
-
-  ts = Math.trunc(Date.now() / 1000)
-  id = Math.trunc(Math.random() * 4294967295)
-  cn = crypto.randomBytes(8).toString('hex')
-  sig = crypto.createHmac('sha1', key).update("#{ts}#{id}#{un}#{cn}").digest('hex')
-  uri = "http://#{address}?ts=#{ts}&id=#{id}&un=#{un}&cn=#{cn}&sig=#{sig}"
-
-  message = Buffer.from("GET #{uri} HTTP/1.1\r\nHost: #{address}\r\n\r\n")
-
-  client = dgram.createSocket('udp4')
-
-  count = 3
-  client.send(message, 0, message.length, port, host, f = (err) ->
-    if err or -- count <= 0
-      client.close()
-      return callback(err)
-    client.send(message, 0, message.length, port, host, f)
-  )
-  return true
-
-
-httpConReq = (url, username, password, timeout, callback) ->
-  options = URL.parse(url)
-  # Ensure socket is reused in case of digest authentication
-  options.agent = new http.Agent({maxSockets : 1})
-
-  statusToError = (statusCode) ->
-    switch statusCode
-      when 200, 204
-        null
-      when 401
-        new Error('Incorrect connection request credentials')
-      when 0
-        new Error('Device is offline')
-      else
-        new Error("Unexpected response code from device: #{statusCode}")
-
-  request = http.get(options, (res) ->
-    if res.statusCode == 401 and res.headers['www-authenticate']?
-      authHeader = auth.parseAuthHeader(res.headers['www-authenticate'])
-      if authHeader.method is 'Basic'
-        options.headers = {'Authorization' : auth.basic(username or '', password or '')}
-      else if authHeader.method is 'Digest'
-        options.headers = {
-          'Authorization' : auth.digest(username or '', password or '', options.path, 'GET', null, authHeader)
-        }
-
-      request = http.get(options, (res) ->
-        if res.statusCode == 0
-          # Workaround for some devices unexpectedly closing the connection
-          request = http.get(options, (res) ->
-            callback(statusToError(res.statusCode))
-            res.resume()
-          ).on('error', (err) ->
-            request.abort()
-            callback(statusToError(0))
-          ).on('socket', (socket) ->
-            socket.setTimeout(timeout)
-            socket.on('timeout', () -> request.abort())
-          )
-        else
-          callback(statusToError(res.statusCode))
-        res.resume()
-      ).on('error', (err) ->
-        request.abort()
-        callback(statusToError(0))
-      ).on('socket', (socket) ->
-        socket.setTimeout(timeout)
-        socket.on('timeout', () -> request.abort())
-      )
-    else
-      callback(statusToError(res.statusCode))
-
-    # No listener for data so emit resume
-    res.resume()
-  ).on('error', (err) ->
-    request.abort()
-    callback(statusToError(0))
-  ).on('socket', (socket) ->
-    socket.setTimeout(timeout)
-    socket.on('timeout', () -> request.abort())
-  )
-
-
-connectionRequest = (deviceId, callback) ->
-  CONNECTION_REQUEST_TIMEOUT = config.get('CONNECTION_REQUEST_TIMEOUT', deviceId)
-  proj = {
-    'Device.ManagementServer.ConnectionRequestURL._value' : 1,
-    'Device.ManagementServer.UDPConnectionRequestAddress._value' : 1,
-    'Device.ManagementServer.ConnectionRequestUsername._value' : 1,
-    'Device.ManagementServer.ConnectionRequestPassword._value' : 1,
-    'InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value' : 1,
-    'InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress._value' : 1,
-    'InternetGatewayDevice.ManagementServer.ConnectionRequestUsername._value' : 1,
-    'InternetGatewayDevice.ManagementServer.ConnectionRequestPassword._value' : 1
-  }
-
-  db.devicesCollection.findOne({_id : deviceId}, proj, (err, device)->
-    return callback(err) if err
-
-    if device.Device? # TR-181 data model
-      connectionRequestUrl = device.Device.ManagementServer.ConnectionRequestURL._value
-      udpConnectionRequestAddress = device.Device.ManagementServer?.UDPConnectionRequestAddress?._value
-      username = device.Device.ManagementServer.ConnectionRequestUsername?._value
-      password = device.Device.ManagementServer.ConnectionRequestPassword?._value
-    else # TR-098 data model
-      connectionRequestUrl = device.InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value
-      udpConnectionRequestAddress = device.InternetGatewayDevice.ManagementServer?.UDPConnectionRequestAddress?._value
-      username = device.InternetGatewayDevice.ManagementServer.ConnectionRequestUsername?._value
-      password = device.InternetGatewayDevice.ManagementServer.ConnectionRequestPassword?._value
-
-    conReq = () ->
-      udpSent = udpConReq(udpConnectionRequestAddress, username, password, (err) -> throw err if err)
-
-      httpConReq(connectionRequestUrl, username, password, CONNECTION_REQUEST_TIMEOUT, (err) ->
-        if udpSent
-          return callback()
-        callback(err)
-      )
-
-    if config.auth?.connectionRequest?
-      # Callback is optional for backward compatibility
-      if config.auth.connectionRequest.length > 4
-        return config.auth.connectionRequest(deviceId, connectionRequestUrl, username, password, (u, p) ->
-          username = u
-          password = p
-          conReq()
-        )
-      [username, password] = config.auth.connectionRequest(deviceId, connectionRequestUrl, username, password)
-    conReq()
-  )
-
-
-watchTask = (deviceId, taskId, timeout, callback) ->
-  setTimeout(() ->
-    db.tasksCollection.findOne({_id : taskId}, {'_id' : 1}, (err, task) ->
-      return callback(err) if err
-
-      if not task
-        return callback(null, 'completed')
-
-      db.faultsCollection.findOne({_id : "#{deviceId}:task_#{taskId}"}, {'_id' : 1}, (err, fault) ->
-        return callback(err) if err
-
-        if fault
-          return callback(null, 'fault')
-
-        if (timeout -= 500) <= 0
-          return callback(null, 'timeout')
-
-        watchTask(deviceId, taskId, timeout, callback)
-      )
-    )
-  , 500)
-
-
-sanitizeTask = (task, callback) ->
-  task.timestamp = new Date(task.timestamp ? Date.now())
-  if task.expiry?
-    if common.typeOf(task.expiry) is common.DATE_TYPE or isNaN(task.expiry)
-      task.expiry = new Date(task.expiry)
-    else
-      task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000)
-
-  callback(task)
-
-
-insertTasks = (tasks, callback) ->
-  if tasks? and common.typeOf(tasks) isnt common.ARRAY_TYPE
-    tasks = [tasks]
-  else if not tasks? or tasks.length == 0
-    return callback(tasks)
-
-  counter = tasks.length
-
-  for task in tasks
-    sanitizeTask(task, (t) ->
-      if t.uniqueKey?
-        db.tasksCollection.remove({device : t.device, uniqueKey : t.uniqueKey}, (err) ->
-        )
-
-      --counter
-      if counter == 0
-        db.tasksCollection.insert(tasks, (err, _tasks) ->
-          #util.log("#{_task.device}: Added task #{_task.name}(#{_task._id})") for _task in _tasks
-          callback(err, _tasks)
-        )
-    )
-
-
-deleteDevice = (deviceId, callback) ->
-  db.tasksCollection.remove({'device' : deviceId}, (err) ->
-    return callback(err) if err
-    db.devicesCollection.remove({'_id' : deviceId}, (err) ->
-      return callback(err) if err
-      db.faultsCollection.remove({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}, (err) ->
-        return callback(err) if err
-        db.operationsCollection.remove({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}, (err) ->
-          return callback(err) if err
-          db.redisClient.del("#{deviceId}_presets_hash",
-            "#{deviceId}_inform_hash",
-            "#{deviceId}_faults",
-            "#{deviceId}_tasks",
-            "#{deviceId}_operations",
-            callback)
-        )
-      )
-    )
-  )
-
-
-exports.connectionRequest = connectionRequest
-exports.watchTask = watchTask
-exports.insertTasks = insertTasks
-exports.deleteDevice = deleteDevice

lib/api-functions.ts

@@ -0,0 +1,352 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { parse } from "url";
+import * as db from "./db";
+import * as common from "./common";
+import * as cache from "./cache";
+import {
+  getCurrentSnapshot,
+  getConfig,
+  getConfigExpression,
+} from "./local-cache";
+import {
+  httpConnectionRequest,
+  udpConnectionRequest,
+} from "./connection-request";
+import { Expression, Task } from "./types";
+import { flattenDevice } from "./mongodb-functions";
+import { evaluate } from "./common/expression";
+
+export async function connectionRequest(
+  deviceId: string,
+  device?: Record<string, { value?: [boolean | number | string, string] }>
+): Promise<string> {
+  if (!device) {
+    const res = await db.devicesCollection.findOne({ _id: deviceId });
+    if (!res) throw new Error("No such device");
+    device = flattenDevice(res);
+  }
+
+  let connectionRequestUrl, udpConnectionRequestAddress, username, password;
+
+  if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
+    connectionRequestUrl = (device[
+      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL"
+    ].value || [""])[0];
+    udpConnectionRequestAddress = ((
+      device[
+        "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
+      ] || {}
+    ).value || [""])[0];
+    username = ((
+      device[
+        "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
+      ] || {}
+    ).value || [""])[0];
+    password = ((
+      device[
+        "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword"
+      ] || {}
+    ).value || [""])[0];
+  } else {
+    connectionRequestUrl = (device[
+      "Device.ManagementServer.ConnectionRequestURL"
+    ].value || [""])[0];
+    udpConnectionRequestAddress = ((
+      device["Device.ManagementServer.UDPConnectionRequestAddress"] || {}
+    ).value || [""])[0];
+    username = ((
+      device["Device.ManagementServer.ConnectionRequestUsername"] || {}
+    ).value || [""])[0];
+    password = ((
+      device["Device.ManagementServer.ConnectionRequestPassword"] || {}
+    ).value || [""])[0];
+  }
+
+  const remoteAddress = parse(connectionRequestUrl).host;
+
+  const evalCallback = (exp): Expression => {
+    if (!Array.isArray(exp)) return exp;
+    if (exp[0] === "PARAM" && typeof exp[1] === "string") {
+      let name = exp[1];
+      if (name === "id") name = "DeviceID.ID";
+      else if (name === "serialNumber") name = "DeviceID.SerialNumber";
+      else if (name === "productClass") name = "DeviceID.ProductClass";
+      else if (name === "oui") name = "DeviceID.OUI";
+      else if (name === "remoteAddress") return remoteAddress;
+      else if (name === "username") return username;
+      else if (name === "password") return password;
+
+      const p = device[name];
+      if (p && p.value) return p.value[0];
+    } else if (exp[0] === "FUNC") {
+      if (exp[1] === "REMOTE_ADDRESS") return remoteAddress;
+      else if (exp[1] === "USERNAME") return username;
+      else if (exp[1] === "PASSWORD") return password;
+    }
+    return exp;
+  };
+
+  const snapshot = await getCurrentSnapshot();
+  const now = Date.now();
+  const UDP_CONNECTION_REQUEST_PORT = +getConfig(
+    snapshot,
+    "cwmp.udpConnectionRequestPort",
+    {},
+    now,
+    evalCallback
+  );
+  const CONNECTION_REQUEST_TIMEOUT = +getConfig(
+    snapshot,
+    "cwmp.connectionRequestTimeout",
+    {},
+    now,
+    evalCallback
+  );
+  const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = !!getConfig(
+    snapshot,
+    "cwmp.connectionRequestAllowBasicAuth",
+    {},
+    now,
+    evalCallback
+  );
+  let authExp: Expression = getConfigExpression(
+    snapshot,
+    "cwmp.connectionRequestAuth"
+  );
+
+  if (authExp === undefined) {
+    authExp = [
+      "FUNC",
+      "AUTH",
+      ["PARAM", "username"],
+      ["PARAM", "password"],
+    ] as Expression;
+  }
+
+  authExp = evaluate(authExp, {}, now, evalCallback);
+
+  const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
+
+  let udpProm = Promise.resolve(false);
+  if (udpConnectionRequestAddress) {
+    try {
+      const u = new URL("udp://" + udpConnectionRequestAddress);
+      udpProm = udpConnectionRequest(
+        u.hostname,
+        parseInt(u.port || "80"),
+        authExp,
+        UDP_CONNECTION_REQUEST_PORT,
+        debug,
+        deviceId
+      ).then(
+        () => true,
+        () => false
+      );
+    } catch (err) {
+      // Ignore invalid address
+    }
+  }
+
+  const status = await httpConnectionRequest(
+    connectionRequestUrl,
+    authExp,
+    CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
+    CONNECTION_REQUEST_TIMEOUT,
+    debug,
+    deviceId
+  );
+
+  if (await udpProm) return "";
+
+  return status;
+}
+
+export async function awaitSessionStart(
+  deviceId: string,
+  lastInform: number,
+  timeout: number
+): Promise<boolean> {
+  const now = Date.now();
+  const device = await db.devicesCollection.findOne(
+    { _id: deviceId },
+    { projection: { _lastInform: 1 } }
+  );
+  const li = (device["_lastInform"] as Date).getTime();
+  if (li > lastInform) return true;
+  const token = await cache.get(`cwmp_session_${deviceId}`);
+  if (token) return true;
+  if (timeout < 500) return false;
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionStart(deviceId, lastInform, timeout);
+}
+
+export async function awaitSessionEnd(
+  deviceId: string,
+  timeout: number
+): Promise<boolean> {
+  const now = Date.now();
+  const token = await cache.get(`cwmp_session_${deviceId}`);
+  if (!token) return true;
+  if (timeout < 500) return false;
+  await new Promise((resolve) => setTimeout(resolve, 500));
+  timeout -= Date.now() - now;
+  return awaitSessionEnd(deviceId, timeout);
+}
+
+function sanitizeTask(task): void {
+  task.timestamp = new Date(task.timestamp || Date.now());
+  if (task.expiry) {
+    if (task.expiry instanceof Date || isNaN(task.expiry))
+      task.expiry = new Date(task.expiry);
+    else task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000);
+  }
+
+  const validParamValue = (p): boolean => {
+    if (
+      !Array.isArray(p) ||
+      p.length < 2 ||
+      typeof p[0] !== "string" ||
+      !p[0].length ||
+      !["string", "boolean", "number"].includes(typeof p[1]) ||
+      (p[2] != null && typeof p[2] !== "string")
+    )
+      return false;
+    return true;
+  };
+
+  switch (task.name) {
+    case "getParameterValues":
+      if (!Array.isArray(task.parameterNames) || !task.parameterNames.length)
+        throw new Error("Missing 'parameterNames' property");
+      for (const p of task.parameterNames) {
+        if (typeof p !== "string" || !p.length)
+          throw new Error(`Invalid parameter name '${p}'`);
+      }
+      break;
+
+    case "setParameterValues":
+      if (!Array.isArray(task.parameterValues) || !task.parameterValues.length)
+        throw new Error("Missing 'parameterValues' property");
+      for (const p of task.parameterValues) {
+        if (!validParamValue(p))
+          throw new Error(`Invalid parameter value '${p}'`);
+      }
+      break;
+
+    case "refreshObject":
+      if (typeof task.objectName !== "string")
+        throw new Error("Missing 'objectName' property");
+      break;
+
+    case "deleteObject":
+      if (typeof task.objectName !== "string" || !task.objectName.length)
+        throw new Error("Missing 'objectName' property");
+      break;
+
+    case "addObject":
+      if (task.parameterValues != null) {
+        if (!Array.isArray(task.parameterValues))
+          throw new Error("Invalid 'parameterValues' property");
+        for (const p of task.parameterValues) {
+          if (!validParamValue(p))
+            throw new Error(`Invalid parameter value '${p}'`);
+        }
+      }
+      break;
+
+    case "download":
+      // genieacs-gui sends file ID instead of fileName and fileType
+      if (!task.file) {
+        if (typeof task.fileType !== "string" || !task.fileType.length)
+          throw new Error("Missing 'fileType' property");
+
+        if (typeof task.fileName !== "string" || !task.fileName.length)
+          throw new Error("Missing 'fileName' property");
+      }
+
+      if (
+        task.targetFileName != null &&
+        typeof task.targetFileName !== "string"
+      )
+        throw new Error("Invalid 'targetFileName' property");
+      break;
+
+    case "provisions":
+      if (
+        !Array.isArray(task.provisions) ||
+        !task.provisions.every((arr) =>
+          arr.every(
+            (s) =>
+              s == null || ["boolean", "number", "string"].includes(typeof s)
+          )
+        )
+      )
+        throw new Error("Invalid 'provisions' property");
+      break;
+
+    case "reboot":
+      break;
+
+    case "factoryReset":
+      break;
+
+    default:
+      throw new Error("Invalid task name");
+  }
+
+  return task;
+}
+
+export async function insertTasks(tasks: any[]): Promise<Task[]> {
+  if (tasks && !Array.isArray(tasks)) tasks = [tasks];
+  else if (!tasks || tasks.length === 0) return tasks || [];
+
+  for (const task of tasks) {
+    sanitizeTask(task);
+    if (task.uniqueKey) {
+      await db.tasksCollection.deleteOne({
+        device: task.device,
+        uniqueKey: task.uniqueKey,
+      });
+    }
+  }
+  await db.tasksCollection.insertMany(tasks);
+  return tasks;
+}
+
+export async function deleteDevice(deviceId: string): Promise<void> {
+  await Promise.all([
+    db.tasksCollection.deleteMany({ device: deviceId }),
+    db.devicesCollection.deleteOne({ _id: deviceId }),
+    db.faultsCollection.deleteMany({
+      _id: {
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+      },
+    }),
+    db.operationsCollection.deleteMany({
+      _id: {
+        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
+      },
+    }),
+    cache.del(`${deviceId}_tasks_faults_operations`),
+  ]);
+}

lib/auth.coffee

@@ -1,72 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-crypto = require 'crypto'
-
-
-exports.parseAuthHeader = (authHeader) ->
-  res = {method: authHeader.slice(0, authHeader.indexOf(' '))}
-  re = /([a-z0-9_-]+)=(?:"([^"]+)"|([a-z0-9_-]+))/gi
-  while match = re.exec(authHeader)
-    res[match[1]] = match[2] or match[3]
-  return res
-
-
-exports.basic = (username, password) ->
-  "Basic #{new Buffer("#{username}:#{password}").toString('base64')}"
-
-
-exports.digest = (username, password, uri, httpMethod, body, authHeader) ->
-  cnonce = '0a4f113b'
-  nc = '00000001'
-
-  if authHeader.qop?
-    if authHeader.qop.indexOf(',') != -1
-      qop = 'auth' # either auth or auth-int. prefer auth
-    else
-      qop = authHeader.qop
-
-  ha1 = crypto.createHash('md5')
-  ha1.update(username).update(':').update(authHeader.realm).update(':').update(password)
-  # TODO support "MD5-sess" algorithm directive
-  ha1 = ha1.digest('hex')
-
-  ha2 = crypto.createHash('md5')
-  ha2.update(httpMethod).update(':').update(uri)
-  if qop == 'auth-int'
-    ha2.update(':').update(body)
-  ha2 = ha2.digest('hex')
-
-  response = crypto.createHash('md5')
-  response.update(ha1).update(':').update(authHeader.nonce)
-
-  if qop?
-    response.update(':').update(nc).update(':').update(cnonce).update(':').update(qop)
-  response.update(':').update(ha2)
-  response = response.digest('hex')
-
-  authString = "Digest username=\"#{username}\""
-  authString += ",realm=\"#{authHeader.realm}\""
-  authString += ",nonce=\"#{authHeader.nonce}\""
-  authString += ",uri=\"#{uri}\""
-  authString += ",algorithm=#{authHeader.algorithm}" if authHeader.algorithm?
-  authString += ",qop=#{qop},nc=#{nc},cnonce=\"#{cnonce}\"" if qop?
-  authString += ",response=\"#{response}\""
-  authString += ",opaque=\"#{authHeader.opaque}\"" if authHeader.opaque?
-  return authString

lib/auth.ts

@@ -0,0 +1,197 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { createHash, randomBytes, pbkdf2 } from "crypto";
+
+function parseHeaderFeilds(str: string): Record<string, string> {
+  const res = {};
+  const parts = str.split(",");
+
+  let part: string;
+  while ((part = parts.shift()) != null) {
+    const name = part.split("=", 1)[0];
+    if (name.length === part.length) {
+      if (!part.trim()) continue;
+      throw new Error("Unable to parse auth header");
+    }
+
+    let value = part.slice(name.length + 1);
+    if (!/^\s*"/.test(value)) {
+      value = value.trim();
+    } else {
+      while (!/[^\\]"\s*$/.test(value)) {
+        const p = parts.shift();
+        if (p == null) throw new Error("Unable to parse auth header");
+        value += "," + p;
+      }
+
+      try {
+        value = JSON.parse(value);
+      } catch (error) {
+        throw new Error("Unable to parse auth header");
+      }
+    }
+    res[name.trim()] = value;
+  }
+  return res;
+}
+
+export function parseAuthorizationHeader(
+  authHeader: string
+): { method: string } {
+  authHeader = authHeader.trim();
+  const method = authHeader.split(" ", 1)[0];
+  const res = { method: method };
+
+  if (method === "Basic") {
+    // Inspired by https://github.com/jshttp/basic-auth
+    const USER_PASS_REGEX = /^([^:]*):(.*)$/;
+    const creds = USER_PASS_REGEX.exec(
+      Buffer.from(authHeader.slice(method.length + 1), "base64").toString()
+    );
+
+    if (!creds) throw new Error("Unable to parse auth header");
+    res["username"] = creds[1];
+    res["password"] = creds[2];
+  } else if (method === "Digest") {
+    Object.assign(res, parseHeaderFeilds(authHeader.slice(method.length + 1)));
+  }
+
+  return res;
+}
+
+export function parseWwwAuthenticateHeader(
+  authHeader: string
+): Record<string, string> {
+  authHeader = authHeader.trim();
+  const method = authHeader.split(" ", 1)[0];
+  const res = { method: method };
+  Object.assign(res, parseHeaderFeilds(authHeader.slice(method.length + 1)));
+  return res;
+}
+
+export function basic(username: string, password: string): string {
+  return "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
+}
+
+export function digest(
+  username: string | Buffer,
+  realm: string | Buffer,
+  password: string | Buffer,
+  nonce: string | Buffer,
+  httpMethod: string | Buffer,
+  uri: string | Buffer,
+  qop?: string | Buffer,
+  body?: string | Buffer,
+  cnonce?: string | Buffer,
+  nc?: string | Buffer
+): string {
+  const ha1 = createHash("md5");
+  ha1.update(username).update(":").update(realm).update(":").update(password);
+  // TODO support "MD5-sess" algorithm directive
+  const ha1d = ha1.digest("hex");
+
+  const ha2 = createHash("md5");
+  ha2.update(httpMethod).update(":").update(uri);
+
+  if (qop === "auth-int") {
+    const bodyHash = createHash("md5")
+      .update(body || "")
+      .digest("hex");
+    ha2.update(":").update(bodyHash);
+  }
+
+  const ha2d = ha2.digest("hex");
+
+  const hash = createHash("md5");
+  hash.update(ha1d).update(":").update(nonce);
+  if (qop) {
+    hash
+      .update(":")
+      .update(nc)
+      .update(":")
+      .update(cnonce)
+      .update(":")
+      .update(qop);
+  }
+  hash.update(":").update(ha2d);
+
+  return hash.digest("hex");
+}
+
+export function solveDigest(
+  username: string | Buffer,
+  password: string | Buffer,
+  uri: string | Buffer,
+  httpMethod: string | Buffer,
+  body: string | Buffer,
+  authHeader: Record<string, string>
+): string {
+  const cnonce = randomBytes(8).toString("hex");
+  const nc = "00000001";
+
+  let qop;
+  if (authHeader.qop) {
+    if (authHeader.qop.indexOf(",") !== -1) qop = "auth";
+    // Either auth or auth-int, prefer auth
+    else qop = authHeader.qop;
+  }
+
+  const hash = digest(
+    username,
+    authHeader.realm,
+    password,
+    authHeader.nonce,
+    httpMethod,
+    uri,
+    qop,
+    body,
+    cnonce,
+    nc
+  );
+
+  let authString = `Digest username="${username}"`;
+  authString += `,realm="${authHeader.realm}"`;
+  authString += `,nonce="${authHeader.nonce}"`;
+  authString += `,uri="${uri}"`;
+  if (authHeader.algorithm) authString += `,algorithm=${authHeader.algorithm}`;
+  if (qop) authString += `,qop=${qop},nc=${nc},cnonce="${cnonce}"`;
+  authString += `,response="${hash}"`;
+  if (authHeader.opaque) authString += `,opaque="${authHeader.opaque}"`;
+
+  return authString;
+}
+
+export function generateSalt(length: number): Promise<string> {
+  return new Promise((resolve, reject) => {
+    randomBytes(length, (err, rand) => {
+      if (err) return void reject(err);
+      resolve(rand.toString("hex"));
+    });
+  });
+}
+
+export function hashPassword(pass: string, salt: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    pbkdf2(pass, salt, 10000, 128, "sha512", (err, hash) => {
+      if (err) return void reject(err);
+      resolve(hash.toString("hex"));
+    });
+  });
+}

lib/cache.coffee

@@ -1,310 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-vm = require 'vm'
-crypto = require 'crypto'
-
-later = require 'later'
-
-db = require './db'
-query = require './query'
-
-
-REFRESH = 3000
-
-nextRefresh = 0
-hash = null
-presets = null
-provisions = null
-virtualParameters = null
-files = null
-
-
-UNLOCK_SCRIPT = 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end'
-EXTEND_SCRIPT = 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end'
-
-lock = (lockName, ttl, callback) ->
-  token = Math.random().toString(36).slice(2)
-  unlockOrExtend = (ttl) ->
-    if not ttl
-      db.redisClient.eval(UNLOCK_SCRIPT, 1, lockName, token, (err, res) ->
-        throw err or new Error('Lock expired') if err or not res
-      )
-    else
-      db.redisClient.eval(EXTEND_SCRIPT, 1, lockName, token, ttl, (err, res) ->
-        throw err or new Error('Lock expired') if err or not res
-      )
-    return
-
-  db.redisClient.set(lockName, token, 'NX', 'PX', ttl, (err, res) ->
-    if err or res
-      return callback(err, unlockOrExtend)
-
-    setTimeout(() ->
-      return lock(lockName, ttl, callback)
-    , 200)
-  )
-
-
-computeHash = () ->
-  # MD5 hash for presets, provisions, virtual parameters for detecting changes
-  h = crypto.createHash('md5')
-  for p in presets
-    h.update(JSON.stringify(p.name))
-    h.update(JSON.stringify(p.channel))
-    h.update(JSON.stringify(p.schedule))
-    h.update(JSON.stringify(p.events))
-    h.update(JSON.stringify(p.precondition))
-    h.update(JSON.stringify(p.provisions))
-
-  keys = Object.keys(provisions).sort()
-  h.update(JSON.stringify(keys))
-  h.update(provisions[k].md5) for k in keys
-
-  keys = Object.keys(virtualParameters).sort()
-  h.update(JSON.stringify(keys))
-  h.update(virtualParameters[k].md5) for k in keys
-
-  hash = h.digest('hex')
-
-
-refresh = (callback) ->
-  now = Date.now()
-  db.redisClient.get('presets_hash', (err, res) ->
-    return callback(err) if err
-
-    if hash? and res == hash
-      nextRefresh = now + REFRESH
-      return callback()
-
-    lock('presets_hash_lock', 3000, (err, unlockOrExtend) ->
-      return callback(err) if err
-      counter = 3
-
-      counter += 2
-      db.presetsCollection.find().toArray((err, res) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        db.objectsCollection.find().toArray((err, objects) ->
-          if err
-            callback(err) if (counter & 1)
-            return counter = 0
-
-          res.sort((a, b) ->
-            if a.weight == b.weight
-              return a._id > b._id
-            else
-              return a.weight - b.weight
-          )
-
-          presets = []
-
-          for preset in res
-            schedule = null
-            if preset.schedule
-              parts = preset.schedule.trim().split(/\s+/)
-              schedule = {
-                md5: crypto.createHash('md5').update(preset.schedule).digest('hex')
-              }
-
-              try
-                schedule.duration = +(parts.shift()) * 1000
-                parts.unshift('0') if parts.length == 5
-                # TODO later.js doesn't throw erorr if expression is invalid!
-                schedule.schedule = later.schedule(later.parse.cron(parts.join(' '), true))
-              catch err
-                # TODO show a warning
-                schedule.schedule = false
-
-            events = preset.events ? {}
-
-            precondition = query.convertMongoQueryToFilters(JSON.parse(preset.precondition))
-
-            _provisions = preset.provisions or []
-
-            # Generate provisions from the old configuration format
-            for c in preset.configurations
-              switch c.type
-                when 'age'
-                  _provisions.push(['refresh', c.name, c.age])
-                when 'value'
-                  _provisions.push(['value', c.name, c.value])
-                when 'add_tag'
-                  _provisions.push(['tag', c.tag, true])
-                when 'delete_tag'
-                  _provisions.push(['tag', c.tag, false])
-                when 'provision'
-                  _provisions.push([c.name].concat(c.args or []))
-                when 'add_object'
-                  for obj in objects
-                    if obj['_id'] == c.object
-                      alias = ("#{k}:#{JSON.stringify(obj[k])}" for k in obj['_keys']).join(',')
-                      p = "#{c.name}.[#{alias}]"
-                      _provisions.push(['instances', p, 1])
-                      for k of obj
-                        if k[0] != '_' and k not in obj['_keys']
-                          _provisions.push(['value', "#{p}.#{k}", obj[k]])
-                when 'delete_object'
-                  for obj in objects
-                    if obj['_id'] == c.object
-                      alias = ("#{k}:#{JSON.stringify(obj[k])}" for k in obj['_keys']).join(',')
-                      p = "#{c.name}.[#{alias}]"
-                      _provisions.push(['instances', p, 0])
-                else
-                  callback(new Error("Unknown configuration type #{c.type}")) if counter & 1
-                  return counter = 0
-
-            presets.push({name: preset._id, channel: preset.channel or 'default', schedule: schedule, events: events, precondition: precondition, provisions: _provisions})
-
-          if (counter -= 2) == 1
-            computeHash()
-            db.redisClient.setex("presets_hash", 300, hash, (err) ->
-              unlockOrExtend(0)
-              nextRefresh = now + REFRESH
-              return callback()
-            )
-        )
-      )
-
-      counter += 2
-      db.provisionsCollection.find().toArray((err, res) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        provisions = {}
-        for r in res
-          provisions[r._id] = {}
-          provisions[r._id].md5 = crypto.createHash('md5').update(r.script).digest('hex')
-          provisions[r._id].script = new vm.Script("\"use strict\";(function(){\n#{r.script}\n})();", {filename: r._id, lineOffset: -1, timeout: 50})
-
-        if (counter -= 2) == 1
-          computeHash()
-          db.redisClient.setex("presets_hash", 300, hash, (err) ->
-            unlockOrExtend(0)
-            nextRefresh = now + REFRESH
-            return callback()
-          )
-      )
-
-      counter += 2
-      db.virtualParametersCollection.find().toArray((err, res) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        virtualParameters = {}
-        for r in res
-          virtualParameters[r._id] = {}
-          virtualParameters[r._id].md5 = crypto.createHash('md5').update(r.script).digest('hex')
-          virtualParameters[r._id].script = new vm.Script("\"use strict\";(function(){\n#{r.script}\n})();", {filename: r._id, lineOffset: -1, timeout: 50})
-
-        if (counter -= 2) == 1
-          computeHash()
-          db.redisClient.setex("presets_hash", 300, hash, (err) ->
-            unlockOrExtend(0)
-            nextRefresh = now + REFRESH
-            return callback()
-          )
-      )
-
-      counter += 2
-      db.filesCollection.find().toArray((err, res) ->
-        if err
-          callback(err) if counter & 1
-          return counter = 0
-
-        files = {}
-        for r in res
-          id = r.filename or r._id.toString()
-          files[id] = {}
-          files[id].length = r.length
-          files[id].md5 = r.md5
-          files[id].contentType = r.contentType
-
-        if (counter -= 2) == 1
-          computeHash()
-          db.redisClient.setex("presets_hash", 300, hash, (err) ->
-            unlockOrExtend(0)
-            nextRefresh = now + REFRESH
-            return callback()
-          )
-      )
-
-      if (counter -= 2) == 1
-        computeHash()
-        db.redisClient.setex("presets_hash", 300, hash, (err) ->
-          unlockOrExtend(0)
-          nextRefresh = now + REFRESH
-          return callback()
-        )
-    )
-  )
-
-
-getPresets = (callback) ->
-  if Date.now() < nextRefresh
-    return callback(null, hash, presets)
-
-  refresh((err) ->
-    return callback(err, hash, presets)
-  )
-
-
-getProvisions = (callback) ->
-  if Date.now() < nextRefresh
-    return callback(null, hash, provisions)
-
-  refresh((err) ->
-    return callback(err, hash, provisions)
-  )
-
-
-getVirtualParameters = (callback) ->
-  if Date.now() < nextRefresh
-    return callback(null, hash, virtualParameters)
-
-  refresh((err) ->
-    return callback(err, hash, virtualParameters)
-  )
-
-
-getFiles = (callback) ->
-  if Date.now() < nextRefresh
-    return callback(null, hash, files)
-
-  refresh((err) ->
-    return callback(err, hash, files)
-  )
-
-
-getProvisionsAndVirtualParameters = (callback) ->
-  if Date.now() < nextRefresh
-    return callback(null, hash, provisions, virtualParameters)
-
-  refresh((err) ->
-    return callback(err, hash, provisions, virtualParameters)
-  )
-
-exports.getPresets = getPresets
-exports.getProvisions = getProvisions
-exports.getFiles = getFiles
-exports.getProvisionsAndVirtualParameters = getProvisionsAndVirtualParameters

lib/cache.ts

@@ -0,0 +1,118 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Collection } from "mongodb";
+import { onConnect } from "./db";
+import * as config from "./config";
+
+const CLOCK_SKEW_TOLERANCE = 30000;
+const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
+
+let cacheCollection: Collection;
+
+onConnect(async (db) => {
+  cacheCollection = db.collection("cache");
+  await cacheCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
+});
+
+export async function get(key: string): Promise<string> {
+  const res = await cacheCollection.findOne({ _id: key });
+  if (res) return res["value"];
+  return null;
+}
+
+export async function del(key: string): Promise<void> {
+  await cacheCollection.deleteOne({ _id: key });
+}
+
+export async function set(
+  key: string,
+  value: string,
+  ttl: number = MAX_CACHE_TTL
+): Promise<void> {
+  const timestamp = new Date();
+  const expire = new Date(
+    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000
+  );
+  await cacheCollection.replaceOne(
+    { _id: key },
+    { _id: key, value, expire, timestamp },
+    { upsert: true }
+  );
+}
+
+export async function pop(key: string): Promise<any> {
+  const res = await cacheCollection.findOneAndDelete({ _id: key });
+  if (res && res["value"]) return res["value"]["value"];
+  return null;
+}
+
+export async function acquireLock(
+  lockName: string,
+  ttl: number,
+  timeout = 0,
+  token = Math.random().toString(36).slice(2)
+): Promise<string> {
+  try {
+    const now = Date.now();
+    const r = await cacheCollection.findOneAndUpdate(
+      { _id: lockName, token },
+      {
+        $set: {
+          value: token,
+          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
+        },
+        $currentDate: { timestamp: true },
+      },
+      { upsert: true, returnOriginal: false }
+    );
+    const v = r.value;
+    if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
+      throw new Error("Database clock skew too great");
+  } catch (err) {
+    if (err.code === 11000) {
+      if (timeout > 0) {
+        return new Promise((resolve, reject) => {
+          const w = 50 + Math.random() * 50;
+          setTimeout(() => {
+            acquireLock(lockName, ttl, timeout - w, token).then(
+              resolve,
+              reject
+            );
+          }, w);
+        });
+      }
+      throw new Error("Failed to acquire lock");
+    }
+    throw err;
+  }
+
+  return token;
+}
+
+export async function releaseLock(
+  lockName: string,
+  token: string
+): Promise<void> {
+  const res = await cacheCollection.deleteOne({
+    _id: lockName,
+    value: token,
+  });
+  if (res["result"]["n"] !== 1) throw new Error("Lock expired");
+}

lib/cluster.js

@@ -1,139 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const cluster = require("cluster");
-const logger = require("./logger");
-
-const VERSION = require("../package.json").version;
-
-
-function getDependencyVersions() {
-  let dependencies = [`node@${process.versions.node}`];
-  try {
-    let npmls = JSON.parse(require("child_process").execSync("npm ls --prod --json"));
-
-    for (let k in npmls.dependencies)
-      dependencies.push(`${k}@${npmls.dependencies[k].version}`);
-  } catch (err) {}
-
-  return dependencies.join(",");
-}
-
-
-function getConfig() {
-  let conf = [];
-  const keys = Object.keys(process.env);
-  for (let k of keys)
-    if (k.startsWith("GENIEACS_"))
-      conf.push(`${k.slice(9)}=${process.env[k]}`);
-
-  return conf.join(",");
-}
-
-
-function restartWorker(worker, code, signal) {
-  let msg = {
-    message: "Worker died",
-    pid: worker.process.pid
-  };
-
-  if (code != null)
-    msg.exitCode = code;
-
-  if (signal != null)
-    msg.signal = signal;
-
-  logger.error(msg);
-  cluster.fork();
-}
-
-
-function start(service) {
-  logger.init(service, VERSION);
-  const config = require("./config");
-
-  logger.info({
-    message: `GenieACS (genieacs-${service}) starting`,
-    pid: process.pid,
-    version: VERSION,
-    dependencies: getDependencyVersions(),
-    config: getConfig()
-  });
-
-  cluster.on("listening", function(worker, address) {
-    logger.info({
-      message: "Worker listening",
-      pid: worker.process.pid,
-      address: address.address,
-      port: address.port
-    });
-  });
-
-  cluster.on("exit", restartWorker);
-
-  process.on("SIGINT", function() {
-    logger.info({
-      message: "Received signal SIGINT, exiting",
-      pid: process.pid
-    });
-
-    cluster.removeListener("exit", restartWorker);
-  });
-
-  process.on("SIGTERM", function() {
-    logger.info({
-      message: "Received signal SIGTERM, exiting",
-      pid: process.pid
-    });
-
-    cluster.removeListener("exit", restartWorker);
-    for (let pid in cluster.workers)
-      cluster.workers[pid].kill();
-  });
-
-  cluster.setupMaster({
-    exec: "lib/server",
-    args: [service]
-  });
-
-  let workerCount = config.get(`${service.toUpperCase()}_WORKER_PROCESSES`);
-
-  if (!workerCount)
-    workerCount = Math.max(2, require("os").cpus().length);
-
-  for (let i = 0; i < workerCount; ++ i)
-    cluster.fork();
-}
-
-
-process.on("uncaughtException", function(err) {
-  logger.error({
-    message: "Uncaught exception in master process, exiting",
-    exception: err
-  });
-  cluster.removeListener("exit", restartWorker);
-  for (let pid in cluster.workers)
-    cluster.workers[pid].kill();
-
-  logger.close();
-});
-
-
-exports.start = start;

lib/cluster.ts

@@ -0,0 +1,124 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import cluster from "cluster";
+import { cpus } from "os";
+import * as logger from "./logger";
+
+let respawnTimestamp = 0;
+let crashes: number[] = [];
+
+function fork(): cluster.Worker {
+  const w = cluster.fork();
+  w.on("error", (err: NodeJS.ErrnoException) => {
+    // Avoid exception when attempting to kill the process just as it's exiting
+    if (err.code !== "EPIPE") throw err;
+    setTimeout(() => {
+      if (!w.isDead()) throw err;
+    }, 50);
+  });
+  return w;
+}
+
+function restartWorker(worker, code, signal): void {
+  const msg = {
+    message: "Worker died",
+    pid: worker.process.pid,
+    exitCode: null,
+    signal: null,
+  };
+
+  if (code != null) msg.exitCode = code;
+
+  if (signal != null) msg.signal = signal;
+
+  logger.error(msg);
+
+  const now = Date.now();
+  crashes.push(now);
+
+  let min1 = 0,
+    min2 = 0,
+    min3 = 0;
+
+  crashes = crashes.filter((n) => {
+    if (n > now - 60000) ++min1;
+    else if (n > now - 120000) ++min2;
+    else if (n > now - 180000) ++min3;
+    else return false;
+    return true;
+  });
+
+  if (min1 > 5 && min2 > 5 && min3 > 5) {
+    process.exitCode = 1;
+    cluster.removeListener("exit", restartWorker);
+    for (const pid in cluster.workers) cluster.workers[pid].kill();
+
+    logger.error({
+      message: "Too many crashes, exiting",
+      pid: process.pid,
+    });
+    return;
+  }
+
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    fork();
+    return;
+  }
+
+  setTimeout(() => {
+    if (process.exitCode) return;
+    fork();
+  }, respawnTimestamp - now);
+}
+
+export function start(
+  workerCount: number,
+  servicePort: number,
+  serviceAddress: string
+): void {
+  cluster.on("listening", (worker, address) => {
+    if (
+      (address.addressType === 4 || address.addressType === 6) &&
+      address.address === serviceAddress &&
+      address.port === servicePort
+    ) {
+      logger.info({
+        message: "Worker listening",
+        pid: worker.process.pid,
+        address: address.address,
+        port: address.port,
+      });
+    }
+  });
+
+  cluster.on("exit", restartWorker);
+
+  if (!workerCount) workerCount = Math.max(2, cpus().length);
+
+  for (let i = 0; i < workerCount; ++i) fork();
+}
+
+export function stop(): void {
+  cluster.removeListener("exit", restartWorker);
+  for (const pid in cluster.workers) cluster.workers[pid].kill();
+}
+
+export const worker = cluster.worker;

lib/common.coffee

@@ -1,249 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-buffer = require 'buffer'
-querystring = require 'querystring'
-
-UNDEFINED_TYPE = '[object Undefined]'
-NULL_TYPE = '[object Null]'
-BOOLEAN_TYPE = '[object Boolean]'
-NUMBER_TYPE = '[object Number]'
-STRING_TYPE = '[object String]'
-ARRAY_TYPE = '[object Array]'
-OBJECT_TYPE = '[object Object]'
-REGEXP_TYPE = '[object RegExp]'
-DATE_TYPE = '[object Date]'
-
-
-typeOf = (obj) ->
-  Object.prototype.toString.call(obj)
-
-
-exports.generateDeviceId = (deviceIdStruct) ->
-  # Percent escaping function. Escapes everything except alphanumerics and underscore
-  esc = (str) ->
-    str.replace(/[^A-Za-z0-9_]/g, (chr) ->
-      buf = new buffer.Buffer(chr, 'utf8')
-      rep = ''
-      rep += "%#{b.toString(16).toUpperCase()}" for b in buf
-      return rep
-    )
-
-  # Guaranteeing globally unique id as defined in TR-069
-  if deviceIdStruct['ProductClass']
-    return "#{esc(deviceIdStruct['OUI'])}-#{esc(deviceIdStruct['ProductClass'])}-#{esc(deviceIdStruct['SerialNumber'])}"
-
-  return "#{esc(deviceIdStruct['OUI'])}-#{esc(deviceIdStruct['SerialNumber'])}"
-
-
-exports.getParamValueFromPath = (obj, path) ->
-  pp = path.split('.')
-  ref = obj
-  try
-    for p in pp
-      ref = ref[p]
-    return ref
-  catch err
-    return undefined
-
-
-pathOverlap = (a, b, start) ->
-  if a.length == b.length
-    res = 3
-  else if a.length > b.length
-    res = 2
-  else
-    res = 1
-
-  for i in [(start ? 0)...Math.min(a.length, b.length)] by 1
-    if a[i]?
-      if not b[i]?
-        res &= 2
-      else if a[i] != b[i]
-        return 0
-    else if b[i]?
-      res &= 1
-
-    return 0 if not res
-
-  return res
-
-
-parseAlias = (pattern, start, res) ->
-  aliases = []
-  i = start
-  while i < pattern.length and pattern[i] != ']'
-    alias = []
-    i = j = parsePath(pattern, i, alias) + 1
-
-    while pattern[j] not in [']', ',']
-      if pattern[j] == '"' and i == j
-        ++ j
-        while pattern[j] != '"' or pattern[j - 1] == '\\'
-          if ++ j >= pattern.length
-            throw new Error('Invalid alias expression')
-      if ++ j >= pattern.length
-        throw new Error('Invalid alias expression')
-    value = pattern[i...j].trim()
-    i = j
-    if value[0] == '"'
-      try
-        value = JSON.parse(value)
-      catch err
-        throw new Error('Invalid alias expression')
-    alias.push(value)
-    aliases.push(alias)
-    ++ i if pattern[i] == ','
-
-  # Need to sort to ensure identical expressions have idential string representation
-  srt = (a, b) ->
-    for j in [0...Math.min(a.length, b.length)] by 2
-      for k in [0...Math.min(a[j].length, b[j].length)] by 1
-        if Array.isArray(a[j][k])
-          if Array.isArray(b[j][k])
-            return srt(a[j][k], b[j][k])
-          else if not b[j][k]?
-            return -1
-          else
-            return 1
-        else if not a[j][k]?
-          if not b[j][k]?
-            return 0
-          else
-            return 1
-        else if not b[j][k]? or Array.isArray(b[j][k])
-            return -1
-        else if a[j][k] > b[j][k]
-          return 1
-        else if a[j][k] < b[j][k]
-          return -1
-
-      if a[j].length > b[j].length
-        return -1
-      else if a[j].length < b[j].length
-        return 1
-
-      if a[j + 1] > b[j + 1]
-        return -1
-      else if a[j + 1] < b[j + 1]
-        return 1
-
-    if a.length > b.length
-      return -1
-    else if a.length < b.length
-      return 1
-
-    return 0
-
-  aliases.sort(srt)
-  res.push([].concat.apply([], aliases))
-  return i
-
-
-parsePath = (pattern, start, res) ->
-  path = []
-  path.wildcard = 0
-  path.alias = 0
-  i = start ? 0
-
-  # Colon separator is needed for parseAlias
-  if i < pattern.length and pattern[i] != ':'
-    while true
-      if pattern[i] == '['
-        path.alias |= 1 << path.length
-        i = parseAlias(pattern, i + 1, path) + 1
-      else
-        j = i
-        while i < pattern.length and pattern[i] != ':' and pattern[i] != '.'
-          ++ i
-        n = pattern.slice(j, i)
-        path.wildcard |= 1 << path.length if n == '*'
-        path.push(n)
-
-      if i >= pattern.length or pattern[i] == ':'
-        break
-      else if pattern[i] != '.'
-        throw new Error('Invalid alias expression')
-
-      ++ i
-
-  Object.freeze(path)
-
-  if not res?
-    return path
-
-  res.push(path)
-  return i
-
-
-addPathMeta = (path) ->
-  return path if path.alias? or path.wildcard?
-
-  path.alias = 0
-  path.wildcard = 0
-
-  for p, i in path
-    if typeOf(p) is ARRAY_TYPE
-      path.alias |= 1 << i
-      for j in [0...p.length] by 2
-        addPathMeta(p[j])
-    else if p == '*'
-      path.wildcard |= 1 << i
-
-  Object.freeze(path)
-  return path
-
-
-hammingWeight = (flags) ->
-  flags -= ((flags >> 1) & 0x55555555)
-  flags = (((flags >> 2) & 0x33333333) + (flags & 0x33333333))
-  flags = (((flags >> 4) + flags) & 0x0f0f0f0f)
-  flags += (flags >> 8)
-  flags += (flags >> 16)
-  return flags & 0x0000003f
-
-
-# Source: http://stackoverflow.com/a/6969486
-escapeRegExp = (str) ->
-  return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&")
-
-
-# djb2 hashing algorithm adapted from http://www.cse.yorku.ca/~oz/hash.html
-djb2 = (str) ->
-  hash = 5381
-  for i in [0...str.length]
-    hash = ((hash << 5) + hash) ^ str.charCodeAt(i)
-  return hash
-
-
-exports.UNDEFINED_TYPE = UNDEFINED_TYPE
-exports.NULL_TYPE = NULL_TYPE
-exports.NUMBER_TYPE = NUMBER_TYPE
-exports.STRING_TYPE = STRING_TYPE
-exports.ARRAY_TYPE = ARRAY_TYPE
-exports.OBJECT_TYPE = OBJECT_TYPE
-exports.REGEXP_TYPE = REGEXP_TYPE
-
-exports.typeOf = typeOf
-exports.pathOverlap = pathOverlap
-exports.parsePath = parsePath
-exports.addPathMeta = addPathMeta
-exports.hammingWeight = hammingWeight
-exports.escapeRegExp = escapeRegExp
-exports.djb2 = djb2

lib/common.ts

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+export function generateDeviceId(
+  deviceIdStruct: Record<string, string>
+): string {
+  // Escapes everything except alphanumerics and underscore
+  function esc(str): string {
+    return str.replace(/[^A-Za-z0-9_]/g, (chr) => {
+      const buf = Buffer.from(chr, "utf8");
+      let rep = "";
+      for (const b of buf) rep += "%" + b.toString(16).toUpperCase();
+      return rep;
+    });
+  }
+
+  // Guaranteeing globally unique id as defined in TR-069
+  if (deviceIdStruct["ProductClass"]) {
+    return (
+      esc(deviceIdStruct["OUI"]) +
+      "-" +
+      esc(deviceIdStruct["ProductClass"]) +
+      "-" +
+      esc(deviceIdStruct["SerialNumber"])
+    );
+  }
+  return esc(deviceIdStruct["OUI"]) + "-" + esc(deviceIdStruct["SerialNumber"]);
+}
+
+// Source: http://stackoverflow.com/a/6969486
+export function escapeRegExp(str: string): string {
+  return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
+}
+
+export function encodeTag(tag: string): string {
+  return encodeURIComponent(tag)
+    .replace(
+      /[!~*'().]/g,
+      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
+    )
+    .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")
+    .replace(/%/g, "0x");
+}
+
+export function decodeTag(tag: string): string {
+  return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%"));
+}

lib/common/authorizer.ts

@@ -0,0 +1,149 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { PermissionSet, Expression } from "../types";
+import { evaluate, or } from "./expression";
+
+export default class Authorizer {
+  private permissionSets: PermissionSet[];
+  private validatorCache: WeakMap<
+    any,
+    (mutationType, mutation, any) => boolean
+  >;
+  private hasAccessCache: Map<string, boolean>;
+  private getFilterCache: Map<string, Expression>;
+
+  public constructor(permissionSets: PermissionSet[]) {
+    this.permissionSets = permissionSets;
+    this.validatorCache = new WeakMap();
+    this.hasAccessCache = new Map();
+    this.getFilterCache = new Map();
+  }
+
+  public hasAccess(resourceType: string, access: number): boolean {
+    const cacheKey = `${resourceType}-${access}`;
+    if (this.hasAccessCache.has(cacheKey))
+      return this.hasAccessCache.get(cacheKey);
+
+    let has = false;
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (perm[resourceType]) {
+          if (perm[resourceType].access >= access) {
+            has = true;
+            break;
+          }
+        }
+      }
+    }
+
+    this.hasAccessCache.set(cacheKey, has);
+    return has;
+  }
+
+  public getFilter(resourceType: string, access: number): Expression {
+    const cacheKey = `${resourceType}-${access}`;
+    if (this.getFilterCache.has(cacheKey))
+      return this.getFilterCache.get(cacheKey);
+
+    let filter: Expression = null;
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (perm[resourceType]) {
+          if (perm[resourceType].access >= access)
+            filter = or(filter, perm[resourceType].filter);
+        }
+      }
+    }
+
+    this.getFilterCache.set(cacheKey, filter);
+    return filter;
+  }
+
+  public getValidator(
+    resourceType: string,
+    resource: unknown
+  ): (mutationType: string, mutation?: any, args?: any) => boolean {
+    if (this.validatorCache.has(resource))
+      return this.validatorCache.get(resource);
+
+    const validators: Expression[] = [];
+
+    for (const permissionSet of this.permissionSets) {
+      for (const perm of permissionSet) {
+        if (
+          perm[resourceType] &&
+          perm[resourceType].access >= 3 &&
+          perm[resourceType].validate
+        )
+          validators.push(perm[resourceType].validate);
+      }
+    }
+
+    const validator = (
+      mutationType: string,
+      mutation: any,
+      any: any
+    ): boolean => {
+      if (!validators.length) return false;
+
+      const object = {
+        mutationType,
+        mutation,
+        resourceType,
+        object: resource,
+        options: any,
+      };
+
+      const valueFunction = (paramName): any => {
+        const entry = paramName.split(".", 1)[0];
+        paramName = paramName.slice(entry.length + 1);
+        let value = null;
+        if (["mutation", "options"].includes(entry)) {
+          value = object[entry];
+          for (const seg of paramName.split(".")) {
+            // typeof null is "object"
+            if (value != null && typeof value !== "object") value = null;
+            else value = value[seg];
+            if (value == null) break;
+          }
+        } else if (object[entry]) {
+          if (paramName) value = object[entry][paramName];
+          else value = object[entry];
+        }
+
+        return value;
+      };
+
+      const res = evaluate(
+        validators.length > 1 ? ["OR", validators] : validators[0],
+        valueFunction,
+        Date.now()
+      );
+      return !Array.isArray(res) && !!res;
+    };
+
+    this.validatorCache.set(resource, validator);
+    return validator;
+  }
+
+  public getPermissionSets(): PermissionSet[] {
+    return this.permissionSets;
+  }
+}

lib/common/bigint.ts

@@ -0,0 +1,106 @@
+/**
+ * Copyright 2013-2020  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+export function and(a: bigint, b: bigint): bigint {
+  return a & b;
+}
+
+export function or(a: bigint, b: bigint): bigint {
+  return a | b;
+}
+
+export function xor(a: bigint, b: bigint): bigint {
+  return a ^ b;
+}
+
+export function not(a: bigint): bigint {
+  return ~a;
+}
+
+export function lshift(a: bigint, b: bigint): bigint {
+  return a << b;
+}
+
+export function rshift(a: bigint, b: bigint): bigint {
+  return a >> b;
+}
+
+export function add(a: bigint, b: bigint): bigint {
+  return a + b;
+}
+
+export function sub(a: bigint, b: bigint): bigint {
+  return a - b;
+}
+
+export function mul(a: bigint, b: bigint): bigint {
+  return a * b;
+}
+
+export function div(a: bigint, b: bigint): bigint {
+  return a / b;
+}
+
+export function exp(a: bigint, b: bigint): bigint {
+  return a ** b;
+}
+
+export function rem(a: bigint, b: bigint): bigint {
+  return a % b;
+}
+
+export function toNumber(a: bigint): number {
+  return Number(a);
+}
+
+export function eq(a: bigint, b: bigint): boolean {
+  return a === b;
+}
+
+export function ne(a: bigint, b: bigint): boolean {
+  return a !== b;
+}
+
+export function lt(a: bigint, b: bigint): boolean {
+  return a < b;
+}
+
+export function lte(a: bigint, b: bigint): boolean {
+  return a <= b;
+}
+
+export function gt(a: bigint, b: bigint): boolean {
+  return a > b;
+}
+
+export function gte(a: bigint, b: bigint): boolean {
+  return a >= b;
+}
+
+export function asUintN(a: number, b: bigint): bigint {
+  return BigInt.asUintN(a, b);
+}
+
+export function asIntN(a: number, b: bigint): bigint {
+  return BigInt.asIntN(a, b);
+}
+
+const _BigInt = BigInt;
+type _bigint = bigint;
+export { _BigInt as BigInt, _bigint as bigint };

lib/common/expression-parser.ts

@@ -0,0 +1,478 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import parsimmon from "parsimmon";
+import { Expression } from "../types";
+
+// Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
+function interpretEscapes(str): string {
+  const escapes = {
+    b: "\b",
+    f: "\f",
+    n: "\n",
+    r: "\r",
+    t: "\t",
+  };
+  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
+    const type = escape.charAt(0);
+    const hex = escape.slice(1);
+    if (type === "u") return String.fromCharCode(parseInt(hex, 16));
+
+    if (escapes.hasOwnProperty(type)) return escapes[type];
+
+    return type;
+  });
+}
+
+export function map(
+  exp: Expression,
+  callback: (e: Expression) => Expression
+): Expression {
+  if (!Array.isArray(exp)) return callback(exp);
+
+  let clone;
+  for (let i = 1; i < exp.length; ++i) {
+    const sub = map(exp[i], callback);
+    if (sub !== exp[i]) {
+      clone = clone || exp.slice();
+      clone[i] = sub;
+    }
+  }
+
+  return callback(clone || exp);
+}
+
+export async function mapAsync(
+  exp: Expression,
+  callback: (e: Expression) => Promise<Expression>
+): Promise<Expression> {
+  if (!Array.isArray(exp)) return callback(exp);
+
+  let clone;
+  for (let i = 1; i < exp.length; ++i) {
+    const sub = await mapAsync(exp[i], callback);
+    if (sub !== exp[i]) {
+      clone = clone || exp.slice();
+      clone[i] = sub;
+    }
+  }
+
+  return callback(clone || exp);
+}
+
+function binaryLeft(
+  operatorsParser: parsimmon.Parser<string>,
+  nextParser: parsimmon.Parser<Expression>
+): parsimmon.Parser<Expression> {
+  return parsimmon.seqMap(
+    nextParser,
+    parsimmon.seq(operatorsParser, nextParser).many(),
+    (first, rest) =>
+      rest.reduce((acc, ch) => {
+        const [op, another] = ch;
+        if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
+        if (Array.isArray(another) && op === another[0])
+          return [op, acc].concat(another.slice(1));
+        return [op, acc, another];
+      }, first)
+  );
+}
+
+const lang = parsimmon.createLanguage({
+  ComparisonOperator: function () {
+    return parsimmon
+      .alt(
+        parsimmon.string(">="),
+        parsimmon.string("<>"),
+        parsimmon.string("<="),
+        parsimmon.string("="),
+        parsimmon.string(">"),
+        parsimmon.string("<")
+      )
+      .skip(parsimmon.optWhitespace);
+  },
+  LikeOperator: function () {
+    return parsimmon
+      .alt(
+        parsimmon.regexp(/like/i).result("LIKE").desc("LIKE"),
+        parsimmon
+          .regexp(/not\s+like/i)
+          .result("NOT LIKE")
+          .desc("NOT LIKE")
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  IsNullOperator: function () {
+    return parsimmon
+      .alt(
+        parsimmon
+          .regexp(/is\s+null/i)
+          .result("IS NULL")
+          .desc("IS NULL"),
+        parsimmon
+          .regexp(/is\s+not\s+null/i)
+          .result("IS NOT NULL")
+          .desc("IS NOT NULL")
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  NotOperator: function () {
+    return parsimmon
+      .regexp(/not/i)
+      .result("NOT")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("NOT");
+  },
+  AndOperator: function () {
+    return parsimmon
+      .regexp(/and/i)
+      .result("AND")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("AND");
+  },
+  OrOperator: function () {
+    return parsimmon
+      .regexp(/or/i)
+      .result("OR")
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .desc("OR");
+  },
+  Parameter: function (r) {
+    return parsimmon
+      .alt(
+        parsimmon.regexp(/[a-zA-Z0-9_.*-]+/),
+        r.Expression.wrap(
+          parsimmon.string("{").skip(parsimmon.optWhitespace),
+          parsimmon.string("}")
+        )
+      )
+      .atLeast(1)
+      .map((x) => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
+      .skip(parsimmon.optWhitespace)
+      .desc("parameter");
+  },
+  StringValueSql: function () {
+    return parsimmon
+      .regexp(/'([^']*)'/, 1)
+      .atLeast(1)
+      .skip(parsimmon.optWhitespace)
+      .map((s) => s.join("'"))
+      .desc("string");
+  },
+  StringValueJs: function () {
+    return parsimmon
+      .regexp(/"((?:\\.|.)*?)"/, 1)
+      .skip(parsimmon.optWhitespace)
+      .map(interpretEscapes)
+      .desc("string");
+  },
+  NumberValue: function () {
+    return parsimmon
+      .regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/)
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .map(Number)
+      .desc("number");
+  },
+  BooleanValue: function () {
+    return parsimmon
+      .alt(
+        parsimmon.regexp(/true/i).result(true).desc("TRUE"),
+        parsimmon.regexp(/false/i).result(false).desc("FALSE")
+      )
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace);
+  },
+  NullValue: function () {
+    return parsimmon
+      .regexp(/null/i)
+      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+      .skip(parsimmon.optWhitespace)
+      .result(null)
+      .desc("NULL");
+  },
+  FuncValue: function (r) {
+    return parsimmon.seqMap(
+      parsimmon
+        .regexp(/([a-zA-Z0-9_]+)/, 1)
+        .skip(parsimmon.optWhitespace)
+        .desc("function"),
+      r.ExpressionList.wrap(
+        parsimmon.string("(").skip(parsimmon.optWhitespace),
+        parsimmon.string(")").skip(parsimmon.optWhitespace)
+      ),
+      (f, args) => ["FUNC", f.toUpperCase()].concat(args)
+    );
+  },
+  WhenPair: function (r) {
+    return parsimmon.seq(
+      parsimmon
+        .regexp(/when/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("WHEN")
+        .then(r.Expression),
+      parsimmon
+        .regexp(/then/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("THEN")
+        .then(r.Expression)
+    );
+  },
+  CaseStatement: function (r) {
+    return parsimmon.seqMap(
+      parsimmon
+        .regexp(/case/i)
+        .result("CASE")
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("CASE"),
+      r.WhenPair.many(),
+      parsimmon
+        .regexp(/else/i)
+        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        .skip(parsimmon.optWhitespace)
+        .desc("ELSE")
+        .then(r.Expression)
+        .map((e) => [[true, e]])
+        .fallback(null)
+        .skip(
+          parsimmon
+            .regex(/end/i)
+            .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
+        )
+        .skip(parsimmon.optWhitespace),
+      (...arr) => arr.flat(2)
+    );
+  },
+  Value: function (r) {
+    return parsimmon.alt(
+      r.NullValue,
+      r.BooleanValue,
+      r.NumberValue,
+      r.StringValueSql,
+      r.StringValueJs,
+      r.FuncValue,
+      r.CaseStatement
+    );
+  },
+  ValueExpression: function (r) {
+    return binaryLeft(
+      parsimmon.string("||").skip(parsimmon.optWhitespace),
+      binaryLeft(
+        parsimmon
+          .alt(parsimmon.string("+"), parsimmon.string("-"))
+          .skip(parsimmon.optWhitespace),
+        binaryLeft(
+          parsimmon
+            .alt(parsimmon.string("*"), parsimmon.string("/"))
+            .skip(parsimmon.optWhitespace),
+          parsimmon.alt(
+            r.Value,
+            r.Parameter,
+            r.Expression.wrap(
+              parsimmon.string("(").skip(parsimmon.optWhitespace),
+              parsimmon.string(")").skip(parsimmon.optWhitespace)
+            )
+          )
+        )
+      )
+    );
+  },
+  Comparison: function (r) {
+    return parsimmon.alt(
+      parsimmon.seqMap(r.ValueExpression, r.IsNullOperator, (p, o) => [o, p]),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.ComparisonOperator,
+        r.ValueExpression,
+        (p, o, v) => [o, p, v]
+      ),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.LikeOperator,
+        r.ValueExpression.skip(
+          parsimmon
+            .regexp(/escape/i)
+            .result("ESCAPE")
+            .skip(parsimmon.whitespace)
+            .desc("ESCAPE")
+        ),
+        r.ValueExpression,
+        (a, b, c, d) => [b, a, c, d]
+      ),
+      parsimmon.seqMap(
+        r.ValueExpression,
+        r.LikeOperator,
+        r.ValueExpression,
+        (a, b, c) => [b, a, c]
+      )
+    );
+  },
+  ExpressionList: function (r) {
+    return r.Expression.sepBy(
+      parsimmon.string(",").skip(parsimmon.optWhitespace)
+    );
+  },
+  Expression: function (r) {
+    function unary(
+      operatorsParser: parsimmon.Parser<string>,
+      nextParser: parsimmon.Parser<Expression>
+    ): parsimmon.Parser<Expression> {
+      return parsimmon.seq(operatorsParser, nextParser).or(nextParser);
+    }
+
+    return binaryLeft(
+      r.OrOperator,
+      binaryLeft(
+        r.AndOperator,
+        unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
+      )
+    ).trim(parsimmon.optWhitespace);
+  },
+});
+
+export function parse(str: string): Expression {
+  if (!str) return null;
+  return lang.Expression.tryParse(str);
+}
+
+export function parseList(str: string): Expression[] {
+  if (!str) return [];
+  return lang.ExpressionList.tryParse(str);
+}
+
+export function stringify(exp: Expression, level = 0): string {
+  if (!Array.isArray(exp)) return JSON.stringify(exp);
+
+  const opLevels = {
+    OR: 10,
+    AND: 11,
+    NOT: 12,
+    "=": 20,
+    "<>": 20,
+    ">": 20,
+    ">=": 20,
+    "<": 20,
+    "<=": 20,
+    LIKE: 20,
+    "NOT LIKE": 20,
+    "IS NULL": 20,
+    "IS NOT NULL": 20,
+    "||": 30,
+    "+": 31,
+    "-": 31,
+    "*": 32,
+    "/": 32,
+  };
+
+  const op = exp[0].toUpperCase();
+
+  function wrap(e): string {
+    if (opLevels[op] <= level) return `(${e})`;
+    else return e;
+  }
+
+  if (op === "FUNC") {
+    return wrap(
+      `${exp[1]}(${exp
+        .slice(2)
+        .map((e) => stringify(e))
+        .join(", ")})`
+    );
+  } else if (op === "PARAM") {
+    if (typeof exp[1] === "string") {
+      return wrap(exp[1]);
+    } else if (Array.isArray(exp[1]) && exp[1][0] === "||") {
+      return wrap(
+        exp[1]
+          .slice(1)
+          .map((p) => {
+            if (typeof p === "string") return p;
+            else return `{${stringify(p)}}`;
+          })
+          .join("")
+      );
+    } else {
+      return wrap(`{${stringify(exp[1])}}`);
+    }
+  } else if (op === "IS NULL" || op === "IS NOT NULL") {
+    return wrap(`${stringify(exp[1], opLevels[op])} ${op}`);
+  } else if (op === "LIKE" || op === "NOT LIKE") {
+    if (exp[3]) {
+      return wrap(
+        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
+          exp[2],
+          opLevels[op]
+        )} ESCAPE ${stringify(exp[3], opLevels[op])}`
+      );
+    } else {
+      return wrap(
+        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
+          exp[2],
+          opLevels[op]
+        )}`
+      );
+    }
+  } else if (op === "CASE") {
+    const parts: string[] = ["CASE"];
+    for (let i = 1; i < exp.length - 1; i += 2) {
+      if (!Array.isArray(exp[i]) && exp[i]) {
+        if (exp[i + 1] != null) parts.push("ELSE", stringify(exp[i + 1]));
+        break;
+      }
+      parts.push("WHEN", stringify(exp[i]), "THEN", stringify(exp[i + 1]));
+    }
+    parts.push("END");
+    return parts.join(" ");
+  } else if (op in opLevels) {
+    const parts = exp.slice(1).map((e, i) => {
+      return stringify(e, opLevels[exp[0]] + Math.min(i - 1, 0));
+    });
+
+    if (op === "NOT") return wrap(`${op} ${parts[0]}`);
+    else return wrap(parts.join(` ${op} `));
+  } else {
+    throw new Error(`Unrecognized operator ${exp[0]}`);
+  }
+}
+
+export function parseLikePattern(pat: string, esc: string): string[] {
+  const chars = pat.split("");
+
+  for (let i = 0; i < chars.length; ++i) {
+    const c = chars[i];
+    if (c === esc) {
+      chars[i] = chars[i + 1] || "";
+      chars[i + 1] = "";
+    } else if (c === "_") {
+      chars[i] = "\\_";
+    } else if (c === "%") {
+      chars[i] = "\\%";
+      while (chars[i + 1] === "%") chars[++i] = "";
+    }
+  }
+  return chars.filter((c) => c);
+}

lib/common/expression.ts

@@ -0,0 +1,408 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { map, mapAsync, parseLikePattern } from "./expression-parser";
+import { Expression } from "../types";
+
+const isArray = Array.isArray;
+
+const regExpCache = new WeakMap<any, RegExp>();
+
+const REDUCE_SKIP = {};
+function reduce(exp, callback): Expression {
+  let loop = true;
+  while (loop) {
+    loop = false;
+    for (let i = 2; i < exp.length; ++i) {
+      const res = callback(exp[i - 1], exp[i], i - 2);
+      if (res !== REDUCE_SKIP) {
+        loop = true;
+        exp = exp.slice();
+        exp.splice(i - 1, 2, res);
+      }
+    }
+  }
+  if (exp.length === 2) return exp[1];
+  return exp;
+}
+
+export function likePatternToRegExp(pat: string, esc = "", flags = ""): RegExp {
+  const convChars = {
+    "-": "\\-",
+    "/": "\\/",
+    "\\": "\\/",
+    "^": "\\^",
+    $: "\\$",
+    "*": "\\*",
+    "+": "\\+",
+    "?": "\\?",
+    ".": "\\.",
+    "(": "\\(",
+    ")": "\\)",
+    "|": "\\|",
+    "[": "\\[",
+    "]": "\\]",
+    "{": "\\{",
+    "}": "\\}",
+    "\\%": ".*",
+    "\\_": ".",
+  };
+  let chars = parseLikePattern(pat, esc);
+  if (!chars.length) return new RegExp("^$", flags);
+  chars = chars.map((c) => convChars[c] || c);
+  chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
+  const l = chars.length - 1;
+  chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
+  return new RegExp(chars.join(""), flags);
+}
+
+function compare(
+  a: boolean | number | string,
+  b: boolean | number | string
+): number {
+  if (typeof a === "boolean") a = +a;
+  if (typeof b === "boolean") b = +b;
+  if (typeof a !== typeof b) return typeof a === "string" ? 1 : -1;
+  return a > b ? 1 : a < b ? -1 : 0;
+}
+
+function toNumber(a: boolean | number | string): number {
+  switch (typeof a) {
+    case "number":
+      return a;
+    case "boolean":
+      return +a;
+    case "string":
+      return parseFloat(a) || 0;
+  }
+}
+
+function toString(a: boolean | number | string): string {
+  switch (typeof a) {
+    case "string":
+      return a;
+    case "number":
+      return a.toString();
+    case "boolean":
+      return (+a).toString();
+  }
+}
+
+export function evaluateCallback(exp: Expression): Expression {
+  if (!Array.isArray(exp)) return exp;
+  if (exp[0] === "CASE") {
+    for (let i = 1; i < exp.length; i += 2) {
+      if (Array.isArray(exp[i])) return exp;
+      if (exp[i]) return exp[i + 1];
+    }
+    return null;
+  } else if (exp[0] === "FUNC") {
+    if (exp[1] === "COALESCE") {
+      const args: Expression[] = [];
+      for (let i = 2; i < exp.length; ++i) {
+        const e = exp[i];
+        if (e == null) continue;
+        args.push(e);
+        if (!Array.isArray(e)) break;
+      }
+      if (!args.length) return null;
+      if (args.length === 1) return args[0];
+      return ["FUNC", "COALESCE", ...args];
+    } else if (exp[1] === "UPPER") {
+      if (exp[2] == null) return null;
+      if (!isArray(exp[2])) return toString(exp[2]).toUpperCase();
+    } else if (exp[1] === "LOWER") {
+      if (exp[2] == null) return null;
+      if (!isArray(exp[2])) return toString(exp[2]).toLowerCase();
+    } else if (exp[1] === "ROUND") {
+      const n = exp[2];
+      const p = exp.length > 3 ? exp[3] : 0;
+      if (n == null || p == null) return null;
+      if (!isArray(n) && !isArray(p)) {
+        const d = 10 ** p;
+        const m = n * d * (1 + Number.EPSILON);
+        return Math.round(m) / d;
+      }
+    }
+  } else if (exp[0] === "PARAM") {
+    if (exp[1] == null) return null;
+  } else if (exp[0] === "AND") {
+    for (let i = 1; i < exp.length; ++i)
+      if (!Array.isArray(exp[i]) && exp[i] != null && !exp[i]) return false;
+    const args: Expression[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const ee = exp[i];
+      if (ee == null) return null;
+      if (Array.isArray(ee)) {
+        if (ee[0] === "AND") args.push(...ee.slice(1));
+        else args.push(ee);
+      }
+    }
+    if (!args.length) return true;
+    if (args.length === 1) args.push(true);
+    return ["AND", ...args];
+  } else if (exp[0] === "OR") {
+    const args: Expression[] = [];
+    for (let i = 1; i < exp.length; ++i) {
+      const ee = exp[i];
+      if (Array.isArray(ee)) {
+        if (ee[0] === "OR") args.push(...ee.slice(1));
+        else args.push(ee);
+      } else if (ee) {
+        return true;
+      }
+    }
+    if (!args.length) return exp.some((ee) => ee == null) ? null : false;
+    if (args.length === 1) args.push(false);
+    return ["OR", ...args];
+  } else if (exp[0] === "NOT") {
+    if (exp[1] == null) return null;
+    if (!isArray(exp[1])) return !exp[1];
+    else if (exp[1][0] === "NOT") return exp[1][1];
+  } else if (exp[0] === "IS NULL") {
+    if (isArray(exp[1])) return exp;
+    else if (exp[1] == null) return true;
+    else return null;
+  } else if (exp[0] === "IS NOT NULL") {
+    if (isArray(exp[1])) return exp;
+    else if (exp[1] != null) return true;
+    else return null;
+  } else if (exp[0] === "LIKE") {
+    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
+    else if (
+      exp[1] == null ||
+      exp[2] == null ||
+      ((exp as any[]).length >= 4 && exp[3] == null)
+    )
+      return null;
+    let r = regExpCache.get(exp as any[]);
+    if (!r) {
+      r = likePatternToRegExp(exp[2], exp[3]);
+      regExpCache.set(exp as any[], r);
+    }
+    return r.test(exp[1]);
+  } else if (exp[0] === "NOT LIKE") {
+    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
+    else if (
+      exp[1] == null ||
+      exp[2] == null ||
+      ((exp as any[]).length >= 4 && exp[3] == null)
+    )
+      return null;
+    let r = regExpCache.get(exp as any[]);
+    if (!r) {
+      r = likePatternToRegExp(exp[2], exp[3]);
+      regExpCache.set(exp as any[], r);
+    }
+    return !r.test(exp[1]);
+  } else if (exp[0] === "=") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) === 0;
+  } else if (exp[0] === "<>") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) !== 0;
+  } else if (exp[0] === ">") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) > 0;
+  } else if (exp[0] === ">=") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) >= 0;
+  } else if (exp[0] === "<") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) < 0;
+  } else if (exp[0] === "<=") {
+    if (exp[1] == null || exp[2] == null) return null;
+    if (isArray(exp[1]) || isArray(exp[2])) return exp;
+    return compare(exp[1], exp[2]) <= 0;
+  } else if (exp[0] === "*") {
+    return reduce(exp, (a, b) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b)) return toNumber(a) * toNumber(b);
+      return REDUCE_SKIP;
+    });
+  } else if (exp[0] === "/") {
+    return reduce(exp, (a, b, i) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b))
+        return i === 0 ? toNumber(a) / toNumber(b) : toNumber(a) * toNumber(b);
+      return REDUCE_SKIP;
+    });
+  } else if (exp[0] === "+") {
+    return reduce(exp, (a, b) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b)) return toNumber(a) + toNumber(b);
+      return REDUCE_SKIP;
+    });
+  } else if (exp[0] === "-") {
+    return reduce(exp, (a, b, i) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b))
+        return i === 0 ? toNumber(a) - toNumber(b) : toNumber(a) + toNumber(b);
+      return REDUCE_SKIP;
+    });
+  } else if (exp[0] === "||") {
+    return reduce(exp, (a, b) => {
+      if (a == null || b == null) return null;
+      if (!isArray(a) && !isArray(b)) return toString(a) + toString(b);
+      return REDUCE_SKIP;
+    });
+  }
+  return exp;
+}
+
+export function evaluate(
+  exp: Expression,
+  obj: Record<string, unknown> | ((e: string) => Expression),
+  now: number,
+  cb?: (e: Expression) => Expression
+): string | number | boolean | null;
+export function evaluate(
+  exp: Expression,
+  obj?: Record<string, unknown> | ((e: string) => Expression),
+  now?: number,
+  cb?: (e: Expression) => Expression
+): Expression;
+export function evaluate(
+  exp: Expression,
+  obj?: Record<string, unknown> | ((e: string) => Expression),
+  now?: number,
+  cb?: (e: Expression) => Expression
+): Expression {
+  return map(exp, (e) => {
+    if (cb) e = cb(e);
+    if (!isArray(e)) return e;
+
+    if (e[0] === "FUNC" && e[1] === "NOW") {
+      if (now) return now;
+    } else if (e[0] === "PARAM") {
+      if (e[1] == null) return null;
+      if (obj && !isArray(e[1])) {
+        let v;
+        if (typeof obj === "function") v = obj(e[1]);
+        else v = obj[e[1]];
+
+        if (v == null) return null;
+        if (typeof v === "object") v = v.value ? v.value[0] : null;
+        return v;
+      }
+    }
+    return evaluateCallback(e);
+  });
+}
+
+export async function evaluateAsync(
+  exp: Expression,
+  obj: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Promise<Expression>
+): Promise<string | number | boolean | null>;
+export async function evaluateAsync(
+  exp: Expression,
+  obj?: Record<string, unknown>,
+  now?: number,
+  cb?: (e: Expression) => Promise<Expression>
+): Promise<Expression>;
+export async function evaluateAsync(
+  exp: Expression,
+  obj?: Record<string, unknown>,
+  now?: number,
+  cb?: (e: Expression) => Promise<Expression>
+): Promise<Expression> {
+  return mapAsync(exp, async (e) => {
+    if (cb) e = await cb(e);
+    if (!isArray(e)) return e;
+
+    if (e[0] === "FUNC") {
+      if (e[1] === "NOW") {
+        if (now) return now;
+      } else if (e[1] === "UPPER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
+      } else if (e[1] === "LOWER") {
+        if (e[2] == null) return null;
+        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
+      }
+    } else if (e[0] === "PARAM") {
+      if (e[1] == null) return null;
+      if (obj && !isArray(e[1])) {
+        let v = obj[e[1]];
+        if (v == null) return null;
+        if (typeof v === "object") v = v["value"] ? v["value"][0] : null;
+        return v as Expression;
+      }
+    }
+    return evaluateCallback(e);
+  });
+}
+
+export function and(exp1: Expression, exp2: Expression): Expression {
+  if (exp1 != null && !exp1) return false;
+  if (exp2 != null && !exp2) return false;
+  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
+    if (exp1 == null || exp2 == null) return null;
+    return true;
+  }
+  if (!Array.isArray(exp2) && exp1[0] === "AND") return exp1;
+  if (!Array.isArray(exp1) && exp2[0] === "AND") return exp2;
+
+  const res: Expression = ["AND"];
+
+  if (Array.isArray(exp1) && exp1[0] === "AND") res.push(...exp1.slice(1));
+  else res.push(exp1);
+
+  if (Array.isArray(exp2) && exp2[0] === "AND") res.push(...exp2.slice(1));
+  else res.push(exp2);
+
+  return res;
+}
+
+export function or(exp1: Expression, exp2: Expression): Expression {
+  if (!Array.isArray(exp1) && exp1) return true;
+  if (!Array.isArray(exp2) && exp2) return true;
+  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
+    if (exp1 == null || exp2 == null) return null;
+    return false;
+  }
+  if (!Array.isArray(exp2) && exp1[0] === "OR") return exp1;
+  if (!Array.isArray(exp1) && exp2[0] === "OR") return exp2;
+
+  const res: Expression = ["OR"];
+
+  if (Array.isArray(exp1) && exp1[0] === "OR") res.push(...exp1.slice(1));
+  else res.push(exp1);
+
+  if (Array.isArray(exp2) && exp2[0] === "OR") res.push(...exp2.slice(1));
+  else res.push(exp2);
+
+  return res;
+}
+
+export function extractParams(exp: Expression): Expression[] {
+  const params = new Set<Expression>();
+  map(exp, (e) => {
+    if (isArray(e) && e[0] === "PARAM") params.add(e[1]);
+    return e;
+  });
+  return Array.from(params);
+}

lib/common/memoize.ts

@@ -0,0 +1,72 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+let cache1 = new Map();
+let cache2 = new Map();
+const keys = new WeakMap();
+
+function getKey(obj): string {
+  if (obj === null) return "null";
+  else if (obj === undefined) return "undefined";
+
+  const t = typeof obj;
+  if (t === "number" || t === "boolean" || t === "string") return `${t}:${obj}`;
+  if (t !== "function" && t !== "object")
+    throw new Error(`Cannot memoize ${t} arguments`);
+
+  let k = keys.get(obj);
+  if (!k) {
+    const rnd = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER);
+    k = `${t}:${rnd.toString(36)}`;
+    keys.set(obj, k);
+  }
+  return k;
+}
+
+export default function memoize<T extends (...args: any[]) => any>(func: T): T {
+  const funcKey = getKey(func);
+  return ((...args) => {
+    const key = JSON.stringify(args.map(getKey)) + funcKey;
+
+    if (cache1.has(key)) return cache1.get(key);
+
+    let r;
+    if (cache2.has(key)) {
+      cache1.set(key, (r = cache2.get(key)));
+    } else {
+      cache1.set(key, (r = func(...args)));
+      // Evict rejected promises
+      if (r instanceof Promise) {
+        r.catch(() => {
+          cache1.delete(key);
+          cache2.delete(key);
+        });
+      }
+    }
+    return r;
+  }) as any;
+}
+
+const interval = setInterval(() => {
+  cache2 = cache1;
+  cache1 = new Map();
+}, 120000);
+
+// Don't hold Node.js process
+if (interval.unref) interval.unref();

lib/common/path-set.ts

@@ -0,0 +1,130 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./path";
+
+export default class PathSet {
+  private lengthIndex: Set<Path>[];
+  private fragmentIndex: Map<string, Set<Path>>[];
+  private stringIndex: Map<string, Path>;
+
+  public constructor() {
+    this.lengthIndex = [];
+    this.fragmentIndex = [];
+    this.stringIndex = new Map();
+  }
+
+  public get depth(): number {
+    return this.lengthIndex.length;
+  }
+
+  public add(path: Path): Path {
+    if (path.alias) throw new Error("PathSet does not support aliased paths");
+    const p = this.get(path);
+
+    if (p) return p;
+
+    this.stringIndex.set(path.toString(), path);
+
+    while (this.lengthIndex.length <= path.length) {
+      this.lengthIndex.push(new Set());
+      // fragmentIndex is one less than lengthIndex
+      if (this.lengthIndex.length > 1) this.fragmentIndex.push(new Map());
+    }
+
+    const lengthIndex = this.lengthIndex[path.length];
+    lengthIndex.add(path);
+
+    for (let i = 0; i < path.length; ++i) {
+      const fragment = path.segments[i] as string;
+      const fragmentIndex = this.fragmentIndex[i];
+
+      let fragmentIndexSet = fragmentIndex.get(fragment);
+      if (!fragmentIndexSet) {
+        fragmentIndexSet = new Set<Path>();
+        fragmentIndex.set(fragment, fragmentIndexSet);
+      }
+
+      fragmentIndexSet.add(path);
+    }
+
+    return path;
+  }
+
+  public get(path: Path): Path {
+    return this.stringIndex.get(path.toString()) || null;
+  }
+
+  public find(
+    path: Path,
+    superset = false,
+    subset = false,
+    depth: number = path.length
+  ): Path[] {
+    if (path.alias) throw new Error("PathSet does not support aliased paths");
+
+    const len = path.length;
+
+    if (!superset && depth === len && (!subset || !path.wildcard)) {
+      const p = this.get(path);
+      return p ? [p] : [];
+    }
+
+    const lengthIndex = this.lengthIndex.slice(len, depth + 1);
+    if (!lengthIndex.length) return [];
+
+    let res;
+    for (let i = len - 1; i >= 0; --i) {
+      let fragmentIndexSet2;
+      const fragmentIndex = this.fragmentIndex[i];
+
+      if ((path.wildcard >> i) & 1) {
+        if (subset) continue;
+      } else if (superset) {
+        fragmentIndexSet2 = fragmentIndex.get("*");
+      }
+
+      const fragment = path.segments[i] as string;
+      const fragmentIndexSet1 = fragmentIndex.get(fragment);
+
+      if (!fragmentIndexSet1) {
+        if (!fragmentIndexSet2) return [];
+        if (!res) res = [...fragmentIndexSet2];
+        else res = res.filter((r) => fragmentIndexSet2.has(r));
+      } else if (!fragmentIndexSet2) {
+        if (!res) res = [...fragmentIndexSet1];
+        else res = res.filter((r) => fragmentIndexSet1.has(r));
+      } else {
+        if (!res) {
+          res = [...fragmentIndexSet1, ...fragmentIndexSet2];
+        } else {
+          res = res.filter(
+            (r) => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
+          );
+        }
+      }
+      if (!res.length) return res;
+    }
+
+    if (!res) res = [].concat(...lengthIndex.map((a) => [...a]));
+    else res = res.filter((r) => lengthIndex.some((a) => a.has(r)));
+
+    return res;
+  }
+}

lib/common/path.ts

@@ -0,0 +1,293 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+type Segments = (string | Alias)[];
+type Alias = [Path, string][];
+
+let cache1 = new Map<string, Path>();
+let cache2 = new Map<string, Path>();
+
+const CHAR_SPACE = 32;
+const CHAR_BACKSLASH = 92;
+const CHAR_COLON = 58;
+const CHAR_QUOTE = 34;
+const CHAR_OPEN_BRACKET = 91;
+const CHAR_CLOSE_BRACKET = 93;
+const CHAR_ASTERISK = 42;
+const CHAR_COMMA = 44;
+const CHAR_DOT = 46;
+
+function charCodeAt(str: string, index: number): number {
+  if (index >= str.length) return 0;
+  return str.charCodeAt(index);
+}
+
+function legalChar(c: number): boolean {
+  return (
+    (c >= 97 && c <= 122) ||
+    (c >= 65 && c <= 90) ||
+    (c >= 48 && c <= 57) ||
+    c === 95 ||
+    c === 45
+  );
+}
+
+export default class Path {
+  public readonly segments: Segments;
+  public readonly wildcard: number;
+  public readonly alias: number;
+  protected _string: string;
+  protected _stringIndex: number[];
+
+  protected static parseAliasValue(
+    pattern: string,
+    index: number
+  ): { index: number; value: string } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) === CHAR_QUOTE) {
+      for (let j = i + 1; j < pattern.length; ++j) {
+        if (
+          pattern.charCodeAt(j) === CHAR_QUOTE &&
+          pattern.charCodeAt(j - 1) !== CHAR_BACKSLASH
+        ) {
+          try {
+            ++j;
+            const v = JSON.parse(pattern.slice(i, j));
+            return { index: j, value: v };
+          } catch {
+            return { index, value: null };
+          }
+        }
+      }
+      return { index, value: null };
+    }
+
+    for (; i < pattern.length; ++i) {
+      const c = pattern.charCodeAt(i);
+      if (c === CHAR_CLOSE_BRACKET || c === CHAR_COMMA) break;
+    }
+
+    return { index: i, value: pattern.slice(index, i).trim() };
+  }
+
+  protected static parseAliasPath(
+    pattern: string,
+    index: number
+  ): { index: number; path: Path } {
+    let i = index;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    const { index: idx, segments } = Path.parsePath(pattern, i);
+    if (idx === i) return { index, path: null };
+    i = idx;
+    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
+    if (charCodeAt(pattern, i) !== CHAR_COLON) return { index, path: null };
+    return { index: i + 1, path: new Path(segments) };
+  }
+
+  protected static parseAlias(
+    pattern: string,
+    index: number
+  ): { index: number; alias: Alias } {
+    const alias: Alias = [];
+    let i = index;
+    for (;;) {
+      let path: Path;
+      let value: string;
+      ({ index: i, path } = Path.parseAliasPath(pattern, i));
+      if (!path) break;
+      ({ index: i, value } = Path.parseAliasValue(pattern, i));
+      if (value == null) break;
+      alias.push([path, value]);
+      index = i;
+
+      if (charCodeAt(pattern, index) !== CHAR_COMMA) break;
+      ++i;
+    }
+
+    // Ensure identical expressions have idential string representation
+    alias.sort((a, b) => {
+      if (a[0].toString() > b[0].toString()) return 1;
+      else if (a[0].toString() < b[0].toString()) return -1;
+      else if (a[1] > b[1]) return 1;
+      else if (a[1] < b[1]) return -1;
+      else return 0;
+    });
+
+    Object.freeze(alias);
+    return { index, alias };
+  }
+
+  protected static parsePath(
+    pattern: string,
+    index: number
+  ): { index: number; segments: Segments } {
+    const segments = [];
+
+    let segStart = index;
+    let wildcard = -1;
+    for (let i = index; i <= pattern.length; ++i) {
+      const c = charCodeAt(pattern, i);
+      if (c === CHAR_OPEN_BRACKET && i === segStart) {
+        const { index: idx, alias } = Path.parseAlias(pattern, i + 1);
+        if (charCodeAt(pattern, idx) !== CHAR_CLOSE_BRACKET) break;
+        segments.push(alias);
+        index = i = idx + 1;
+        segStart = i + 1;
+      } else if (c === CHAR_ASTERISK && i === segStart) {
+        wildcard = i + 1;
+      } else if (!legalChar(c) || i === wildcard) {
+        if (i === segStart) break;
+        const s = pattern.slice(segStart, i);
+        segments.push(s);
+        index = i;
+        if (c !== CHAR_DOT) break;
+        segStart = i + 1;
+      }
+    }
+
+    Object.freeze(segments);
+    return { index, segments };
+  }
+
+  protected constructor(segments: Segments) {
+    let alias = 0;
+    let wildcard = 0;
+    const arr = segments.map((s, i) => {
+      if (Array.isArray(s)) {
+        alias |= 1 << i;
+        const parts = s.map(
+          (al) => `${al[0].toString()}:${JSON.stringify(al[1])}`
+        );
+        return `[${parts.join(",")}]`;
+      } else if (s === "*") {
+        wildcard |= 1 << i;
+      }
+      return s;
+    });
+
+    let offset = 0;
+    const stringIndex = arr.map((s, i) => (offset += s.length) + i);
+
+    this.segments = segments;
+    this.wildcard = wildcard;
+    this.alias = alias;
+    this._string = arr.join(".");
+    this._stringIndex = stringIndex;
+  }
+
+  public static parse(str: string): Path {
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const { index, segments } = Path.parsePath(str, 0);
+        if (index < str.length) throw new Error("Invalid parameter path");
+        path = new Path(segments);
+        if (path.toString() !== str) cache1.set(path.toString(), path);
+      }
+      cache1.set(str, path);
+    }
+    return path;
+  }
+
+  public get length(): number {
+    return this.segments.length;
+  }
+
+  public toString(): string {
+    return this._string;
+  }
+
+  public slice(start = 0, end: number = this.segments.length): Path {
+    if (start < 0) start = Math.max(0, this.segments.length + start);
+    if (end < 0) end = Math.max(0, this.segments.length + end);
+
+    let str;
+    if (start >= end) {
+      str = "";
+    } else {
+      const i1 = start > 0 ? this._stringIndex[start - 1] + 1 : 0;
+      const i2 =
+        end <= this.segments.length
+          ? this._stringIndex[end - 1]
+          : this._string.length;
+      str = this._string.slice(i1, i2);
+    }
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const segments = this.segments.slice(start, end);
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+
+  public concat(path2: Path): Path {
+    if (!path2._string) return this;
+    else if (!this._string) return path2;
+
+    const str = `${this._string}.${path2._string}`;
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        const segments = this.segments.concat(path2.segments);
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+
+  public stripAlias(): Path {
+    if (!this.alias) return this;
+    const segments = this.segments.map((s) => (Array.isArray(s) ? "*" : s));
+    const str = segments.join(".");
+
+    let path = cache1.get(str);
+    if (!path) {
+      path = cache2.get(str);
+      if (!path) {
+        Object.freeze(segments);
+        path = new Path(segments);
+      }
+      cache1.set(str, path);
+    }
+
+    return path;
+  }
+}
+
+const interval = setInterval(() => {
+  cache2 = cache1;
+  cache1 = new Map();
+}, 120000);
+
+// Don't hold Node.js process
+if (interval.unref) interval.unref();

lib/common/yaml.ts

@@ -0,0 +1,218 @@
+const LINE_WIDTH = 80;
+const INDENTATION = "  ";
+
+const STRING_RESERVED = new Set([
+  "true",
+  "True",
+  "TRUE",
+  "false",
+  "False",
+  "FALSE",
+  "null",
+  "Null",
+  "NULL",
+]);
+
+function isPrintable(str: string): boolean {
+  return !/[^\t\n\x20-\x7e\x85\u{a0}-\u{d7ff}\u{e000}-\u{fffd}\u{10000}-\u{10ffff}]/u.test(
+    str
+  );
+}
+
+function stringifyKey(str: string): string {
+  if (!str || !isPrintable(str)) return JSON.stringify(str);
+  if (/^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|[\n,[\]{}]|\s$/.test(str))
+    return JSON.stringify(str);
+  return str;
+}
+
+function foldString(str: string): string[] {
+  if (str.length <= LINE_WIDTH) return [str];
+  if (str.startsWith(" ")) return [str];
+  const lines: string[] = [];
+
+  let idx = 0;
+  let cand = 0;
+  for (let i = 1; i < str.length - 1; ++i) {
+    if (str[i] !== " ") continue;
+
+    if (str[i + 1] === " ") {
+      i += 2;
+      while (str[i] === " ") ++i;
+      continue;
+    }
+
+    if (i <= idx + LINE_WIDTH) {
+      cand = i;
+      continue;
+    }
+
+    const c = cand > idx ? cand : i;
+    lines.push(str.slice(idx, c));
+    idx = c + 1;
+    cand = i;
+  }
+
+  if (cand > idx && str.length > idx + LINE_WIDTH) {
+    lines.push(str.slice(idx, cand));
+    idx = cand + 1;
+  }
+
+  lines.push(str.slice(idx));
+
+  return lines;
+}
+
+function stringifyString(str: string, res: string[], prefix1, prefix2): void {
+  if (/^\s*$/.test(str) || STRING_RESERVED.has(str) || !isPrintable(str)) {
+    res.push(prefix1 + JSON.stringify(str));
+    return;
+  }
+
+  if (!prefix2) prefix2 = INDENTATION;
+
+  const lines = str.split("\n");
+  if (lines.length > 1) {
+    let idt = "";
+    let chmp = "-";
+    if ((lines.find((l) => l) || "").startsWith(" "))
+      idt = `${INDENTATION.length}`;
+
+    if (!lines[lines.length - 1]) {
+      lines.pop();
+      if (lines[lines.length - 1]) chmp = "";
+      else chmp = "+";
+    }
+
+    if (/^\s+$/.test(lines[lines.length - 1])) {
+      res.push(prefix1 + JSON.stringify(str));
+      return;
+    }
+
+    let isFolded = false;
+    const folded = lines.map((l) => {
+      const ls = foldString(l);
+      if (ls.length > 1) isFolded = true;
+      return ls;
+    });
+
+    if (!isFolded) {
+      res.push(
+        `${prefix1}|${idt}${chmp}`,
+        ...lines.map((l) => (l ? prefix2 + l : l))
+      );
+      return;
+    }
+
+    res.push(`${prefix1}>${idt}${chmp}`);
+    res.push(...folded[0].map((f) => prefix2 + f));
+    for (let i = 1; i < folded.length; ++i) {
+      const prevLine = folded[i - 1][0];
+      if (prevLine && !folded[i - 1][0].startsWith(" ")) res.push("");
+      res.push(...folded[i].map((f) => prefix2 + f));
+    }
+    return;
+  }
+
+  if (
+    /^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|\s$/.test(str) ||
+    parseFloat(str) === +str
+  ) {
+    res.push(prefix1 + JSON.stringify(str));
+    return;
+  }
+
+  res.push(prefix1 + str);
+}
+
+function stringifyAny(
+  obj: unknown,
+  res: string[],
+  prefix1 = "",
+  prefix2 = ""
+): void {
+  if (obj == null) {
+    res.push(`${prefix1}null`);
+    return;
+  }
+  if (typeof obj === "number" || typeof obj === "boolean") {
+    res.push(`${prefix1}${JSON.stringify(obj)}`);
+    return;
+  }
+  if (obj instanceof Date) {
+    res.push(`${prefix1}${obj.toJSON()}`);
+    return;
+  }
+
+  if (typeof obj === "string") {
+    stringifyString(obj, res, prefix1, prefix2);
+    return;
+  }
+
+  if (Array.isArray(obj)) {
+    if (!obj.length) {
+      res.push(prefix1 + "[]");
+      return;
+    }
+
+    if (!prefix1 || prefix1.endsWith("- ")) {
+      stringifyAny(obj[0], res, prefix1 + "- ", prefix2 + INDENTATION);
+      prefix1 = prefix2 + "- ";
+      prefix2 = prefix2 + INDENTATION;
+      for (let i = 1; i < obj.length; ++i)
+        stringifyAny(obj[i], res, prefix1, prefix2);
+    } else {
+      res.push(prefix1);
+      prefix1 = prefix2 + "- ";
+      prefix2 = prefix2 + INDENTATION;
+      for (let i = 0; i < obj.length; ++i)
+        stringifyAny(obj[i], res, prefix1, prefix2);
+    }
+    return;
+  }
+
+  const entries = Object.entries(obj).filter((e) => e[1] !== undefined);
+
+  if (!entries.length) {
+    res.push(prefix1 + "{}");
+    return;
+  }
+
+  if (!prefix1 || prefix1.endsWith("- ")) {
+    stringifyAny(
+      entries[0][1],
+      res,
+      prefix1 + `${stringifyKey(entries[0][0])}: `,
+      prefix2 + INDENTATION
+    );
+    prefix1 = prefix2;
+    prefix2 = prefix2 + INDENTATION;
+    for (let i = 1; i < entries.length; ++i) {
+      stringifyAny(
+        entries[i][1],
+        res,
+        prefix1 + `${stringifyKey(entries[i][0])}: `,
+        prefix2
+      );
+    }
+  } else {
+    res.push(prefix1);
+    prefix1 = prefix2;
+    prefix2 = prefix2 + INDENTATION;
+    for (let i = 0; i < entries.length; ++i) {
+      stringifyAny(
+        entries[i][1],
+        res,
+        prefix1 + `${stringifyKey(entries[i][0])}: `,
+        prefix2
+      );
+    }
+  }
+}
+
+export function stringify(obj: unknown): string {
+  if (obj === undefined) return undefined;
+  const lines: string[] = [];
+  stringifyAny(obj, lines);
+  return lines.join("\n") + "\n";
+}

lib/config.coffee

@@ -1,192 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-path = require 'path'
-common = require './common'
-
-options = {
-  CONFIG_DIR : {type : 'path', default : 'config'},
-  MONGODB_CONNECTION_URL : {type : 'string', default : 'mongodb://127.0.0.1/genieacs'},
-  REDIS_PORT : {type : 'int', default : 6379},
-  REDIS_HOST : {type : 'string', default : '127.0.0.1'},
-  REDIS_DB : {type : 'int', default : 0},
-
-  CWMP_WORKER_PROCESSES : {type : 'int', default : 0},
-  CWMP_PORT : {type : 'int', default : 7547},
-  CWMP_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  CWMP_SSL : {type : 'bool', default : false},
-  CWMP_LOG_FILE : {type: 'path', default : ''},
-  CWMP_ACCESS_LOG_FILE : {type : 'path', default : ''},
-
-  NBI_WORKER_PROCESSES : {type : 'int', default : 0},
-  NBI_PORT : {type : 'int', default : 7557},
-  NBI_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  NBI_SSL : {type : 'bool', default : false},
-  NBI_LOG_FILE : {type: 'path', default : ''},
-  NBI_ACCESS_LOG_FILE : {type : 'path', default : ''},
-
-  FS_WORKER_PROCESSES : {type : 'int', default : 0},
-  FS_PORT : {type : 'int', default : 7567},
-  FS_INTERFACE : {type : 'string', default : '0.0.0.0'},
-  FS_SSL : {type : 'bool', default : false},
-  FS_HOSTNAME : {type : 'string', default : 'acs.example.com'},
-  FS_LOG_FILE : {type: 'path', default : ''},
-  FS_ACCESS_LOG_FILE : {type : 'path', default : ''},
-
-  DOWNLOAD_TIMEOUT: {type : 'int', default : 3600},
-  EXT_TIMEOUT: {type: 'int', default: 3000},
-  MAX_CACHE_TTL : {type : 'int', default : 86400},
-  DEBUG : {type : 'bool', default : false},
-  RETRY_DELAY : {type : 'int', default : 300},
-  SESSION_TIMEOUT : {type : 'int', default : 30},
-  CONNECTION_REQUEST_TIMEOUT : {type : 'int', default: 2000},
-  GPN_NEXT_LEVEL : {type : 'int', default : 0},
-  GPV_BATCH_SIZE : {type : 'int', default : 32},
-  MAX_DEPTH : {type: 'int', default : 16},
-  COOKIES_PATH : {type : 'string'},
-  LOG_FORMAT : {type : 'string', default : 'simple'},
-  ACCESS_LOG_FORMAT : {type : 'string', default : ''},
-
-  # XML configuration
-  XML_RECOVER : {type : 'bool', default : false},
-  XML_IGNORE_ENC : {type : 'bool', default : false},
-  XML_FORMAT : {type : 'bool', default : false},
-  XML_NO_DECL : {type : 'bool', default : false},
-  XML_NO_EMPTY : {type : 'bool', default : false},
-  XML_IGNORE_NAMESPACE : {type : 'bool', default : false},
-
-  # Should probably never be changed
-  DEVICE_ONLINE_THRESHOLD : {type : 'int', default : 4000}
-}
-
-allConfig = {}
-
-
-setConfig = (name, value, commandLineArgument) ->
-  return true if allConfig[name]?
-
-  # For compatibility with v1.0
-  if name in ['PRESETS_CACHE_DURATION', 'presets-cache-duration']
-    setConfig('MAX_CACHE_TTL', value)
-
-  if name in ['GET_PARAMETER_NAMES_DEPTH_THRESHOLD', 'get-parameter-names-depth-threshold']
-    setConfig('GPN_NEXT_LEVEL', value)
-
-  if name in ['TASK_PARAMETERS_BATCH_SIZE', 'task-parameters-batch-size']
-    setConfig('GPV_BATCH_SIZE', value)
-
-  if name in ['IGNORE_XML_NAMESPACES', 'ignore-xml-namespaces']
-    setConfig('XML_IGNORE_NAMESPACE', value)
-
-  if name in ['XML_PARSE_IGNORE_ENC', 'xml-parse-ignore-enc']
-    setConfig('XML_IGNORE_ENC', value)
-
-  if name in ['XML_PARSE_RECOVER', 'xml-parse-recover']
-    setConfig('XML_RECOVER', value)
-
-  if name in ['FS_IP', 'fs-ip']
-    setConfig('FS_HOSTNAME', value)
-
-  cast = (val, type) ->
-    switch type
-      when 'int'
-        Number(val)
-      when 'bool'
-        String(val).trim().toLowerCase() in ['true', 'on', 'yes', '1']
-      when 'string'
-        String(val)
-      when 'path'
-        if val then path.resolve(val) else ''
-      else
-        null
-
-  _value = null
-
-  for optionName, optionDetails of options
-    n = optionName
-    if commandLineArgument
-      n = n.toLowerCase().replace(/_/g, '-')
-
-    if name == n
-      _value = cast(value, optionDetails.type)
-      n = optionName
-    else if name.startsWith("#{n}-")
-      _value = cast(value, optionDetails.type)
-      n = "#{optionName}-#{name[optionName.length+1..]}"
-
-    if _value?
-      allConfig[n] = _value
-      # Save as environmnet variable to pass on to any child processes
-      process.env["GENIEACS_#{n}"] = _value
-      return true
-
-  return false
-
-
-# Command line arguments
-argv = process.argv[2..]
-while argv.length
-  arg = argv.shift()
-  if arg[0] == '-'
-    v = argv.shift()
-    setConfig(arg[2..], v, true)
-
-
-# Environment variable
-for k, v of process.env
-  continue if k.lastIndexOf('GENIEACS_', 0) != 0
-  k = k[9..] # remove "GENIEACS_" prefix
-  setConfig(k, v)
-
-
-# Use default config dir if none defined
-setConfig('CONFIG_DIR', options['CONFIG_DIR'].default)
-
-# Configuration file
-for k, v of require(path.resolve(allConfig.CONFIG_DIR, 'config'))
-  if not setConfig(k, v)
-    # Pass as environment variable to be accessable by extensions
-    process.env["GENIEACS_#{k}"] = v
-
-
-# Defaults
-for k, v of options
-  setConfig(k, v.default) if v.default?
-
-
-get = (option, deviceId) ->
-  if deviceId?
-    name = "#{option}-#{deviceId}"
-    v = allConfig[name]
-    return v if v?
-    i = name.lastIndexOf('-')
-    v = allConfig[name[0...i]]
-    return v if v?
-    i = name.lastIndexOf('-', i-1)
-    v = allConfig[name[0...i]]
-    return v if v? or i == -1
-
-  return allConfig[option]
-
-
-# load authentication scripts
-try
-  exports.auth = require(path.resolve(allConfig.CONFIG_DIR, 'auth'))
-
-exports.get = get

lib/config.ts

@@ -0,0 +1,295 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { resolve } from "path";
+import { readFileSync, existsSync } from "fs";
+
+// Find project root directory
+export let ROOT_DIR = resolve(__dirname, "..");
+while (!existsSync(`${ROOT_DIR}/package.json`)) {
+  const d = resolve(ROOT_DIR, "..");
+  if (d === ROOT_DIR) {
+    ROOT_DIR = process.cwd();
+    break;
+  }
+  ROOT_DIR = d;
+}
+
+// For compatibility with v1.1
+let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl, fsHostname;
+
+const options = {
+  EXT_DIR: { type: "path", default: resolve(ROOT_DIR, "config/ext") },
+  MONGODB_CONNECTION_URL: {
+    type: "string",
+    default: "mongodb://127.0.0.1/genieacs",
+  },
+
+  CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
+  CWMP_PORT: { type: "int", default: 7547 },
+  CWMP_INTERFACE: { type: "string", default: "0.0.0.0" },
+  CWMP_SSL_CERT: { type: "string", default: "" },
+  CWMP_SSL_KEY: { type: "string", default: "" },
+  CWMP_LOG_FILE: { type: "path", default: "" },
+  CWMP_ACCESS_LOG_FILE: { type: "path", default: "" },
+
+  NBI_WORKER_PROCESSES: { type: "int", default: 0 },
+  NBI_PORT: { type: "int", default: 7557 },
+  NBI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  NBI_SSL_CERT: { type: "string", default: "" },
+  NBI_SSL_KEY: { type: "string", default: "" },
+  NBI_LOG_FILE: { type: "path", default: "" },
+  NBI_ACCESS_LOG_FILE: { type: "path", default: "" },
+
+  FS_WORKER_PROCESSES: { type: "int", default: 0 },
+  FS_PORT: { type: "int", default: 7567 },
+  FS_INTERFACE: { type: "string", default: "0.0.0.0" },
+  FS_SSL_CERT: { type: "string", default: "" },
+  FS_SSL_KEY: { type: "string", default: "" },
+  FS_URL_PREFIX: { type: "string", default: "" },
+  FS_LOG_FILE: { type: "path", default: "" },
+  FS_ACCESS_LOG_FILE: { type: "path", default: "" },
+
+  UI_WORKER_PROCESSES: { type: "int", default: 0 },
+  UI_PORT: { type: "int", default: 3000 },
+  UI_INTERFACE: { type: "string", default: "0.0.0.0" },
+  UI_SSL_CERT: { type: "string", default: "" },
+  UI_SSL_KEY: { type: "string", default: "" },
+  UI_LOG_FILE: { type: "path", default: "" },
+  UI_ACCESS_LOG_FILE: { type: "path", default: "" },
+  UI_JWT_SECRET: { type: "string", default: "" },
+
+  UDP_CONNECTION_REQUEST_PORT: { type: "int", default: 0 },
+  FORWARDED_HEADER: { type: "string", default: "" },
+
+  DOWNLOAD_TIMEOUT: { type: "int", default: 3600 },
+  EXT_TIMEOUT: { type: "int", default: 3000 },
+  MAX_CACHE_TTL: { type: "int", default: 86400 },
+  DEBUG_FILE: { type: "path", default: "" },
+  DEBUG_FORMAT: { type: "string", default: "yaml" },
+  DEBUG: { type: "bool", default: false },
+  RETRY_DELAY: { type: "int", default: 300 },
+  SESSION_TIMEOUT: { type: "int", default: 30 },
+  CONNECTION_REQUEST_TIMEOUT: { type: "int", default: 2000 },
+  GPN_NEXT_LEVEL: { type: "int", default: 0 },
+  GPV_BATCH_SIZE: { type: "int", default: 32 },
+  MAX_DEPTH: { type: "int", default: 16 },
+  COOKIES_PATH: { type: "string" },
+  LOG_FORMAT: { type: "string", default: "simple" },
+  ACCESS_LOG_FORMAT: { type: "string", default: "" },
+  MAX_CONCURRENT_REQUESTS: { type: "int", default: 20 },
+  DATETIME_MILLISECONDS: { type: "bool", default: true },
+  BOOLEAN_LITERAL: { type: "bool", default: true },
+  CONNECTION_REQUEST_ALLOW_BASIC_AUTH: { type: "bool", default: false },
+  MAX_COMMIT_ITERATIONS: { type: "int", default: 32 },
+
+  // Should probably never be changed
+  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
+};
+
+const allConfig: { [name: string]: string | number } = {};
+
+function setConfig(name, value, commandLineArgument = false): boolean {
+  if (allConfig[name] != null) return true;
+
+  // For compatibility with v1.1
+  if (name === "CONFIG_DIR" || name === "config-dir")
+    configDir = configDir || resolve(ROOT_DIR, value);
+
+  if (name === "CWMP_SSL" || name === "cwmp-ssl")
+    cwmpSsl = cwmpSsl || String(value).toLowerCase().trim();
+
+  if (name === "NBI_SSL" || name === "nbi-ssl")
+    nbiSsl = nbiSsl || String(value).toLowerCase().trim();
+
+  if (name === "FS_SSL" || name === "fs-ssl")
+    fsSsl = fsSsl || String(value).toLowerCase().trim();
+
+  if (name === "UI_SSL" || name === "ui-ssl")
+    uiSsl = uiSsl || String(value).toLowerCase().trim();
+
+  if (name === "FS_HOSTNAME" || name === "fs-hostname")
+    fsHostname = fsHostname || String(value).trim();
+
+  // For compatibility with v1.0
+  if (name === "PRESETS_CACHE_DURATION" || name === "presets-cache-duration")
+    setConfig("MAX_CACHE_TTL", value);
+
+  if (
+    name === "GET_PARAMETER_NAMES_DEPTH_THRESHOLD" ||
+    name === "get-parameter-names-depth-threshold"
+  )
+    setConfig("GPN_NEXT_LEVEL", value);
+
+  if (
+    name === "TASK_PARAMETERS_BATCH_SIZE" ||
+    name === "task-parameters-batch-size"
+  )
+    setConfig("GPV_BATCH_SIZE", value);
+
+  if (name === "FS_IP" || name === "fs-ip") setConfig("FS_HOSTNAME", value);
+
+  function cast(val, type): string | number | boolean {
+    switch (type) {
+      case "int":
+        return Number(val);
+      case "bool":
+        return ["true", "1"].includes(String(val).trim().toLowerCase());
+      case "string":
+        return String(val);
+      case "path":
+        if (!val) return "";
+        return resolve(val);
+      default:
+        return null;
+    }
+  }
+
+  let _value = null;
+  for (const [optionName, optionDetails] of Object.entries(options)) {
+    let n = optionName;
+    if (commandLineArgument) n = n.toLowerCase().replace(/_/g, "-");
+
+    if (name === n) {
+      _value = cast(value, optionDetails.type);
+      n = optionName;
+    } else if (name.startsWith(`${n}-`)) {
+      _value = cast(value, optionDetails.type);
+      n = `${optionName}-${name.slice(optionName.length + 1)}`;
+    }
+
+    if (_value != null) {
+      allConfig[n] = _value;
+      // Save as environmnet variable to pass on to any child process
+      process.env[`GENIEACS_${n}`] = _value;
+      return true;
+    }
+  }
+
+  return false;
+}
+
+// Command line arguments
+const argv = process.argv.slice(2);
+while (argv.length) {
+  const arg = argv.shift();
+  if (arg[0] === "-") {
+    const v = argv.shift();
+    setConfig(arg.slice(2), v, true);
+  }
+}
+
+// Environment variable
+for (const [k, v] of Object.entries(process.env))
+  if (k.startsWith("GENIEACS_")) setConfig(k.slice(9), v);
+
+// Configuration file
+const configFilename = configDir
+  ? `${configDir}/config.json`
+  : `${ROOT_DIR}/config/config.json`;
+
+if (existsSync(configFilename)) {
+  const configFile = JSON.parse(readFileSync(configFilename).toString());
+
+  for (const [k, v] of Object.entries(configFile)) {
+    if (!setConfig(k, v))
+      // Pass as environment variable to be accessable by extensions
+      process.env[`GENIEACS_${k}`] = `${v}`;
+  }
+}
+
+if (configDir) setConfig("EXT_DIR", `${configDir}/ext`);
+
+if (["true", "1"].includes(cwmpSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("CWMP_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("CWMP_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(nbiSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("NBI_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("NBI_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(fsSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("FS_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("FS_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (["true", "1"].includes(uiSsl)) {
+  const d = configDir || `${ROOT_DIR}/config`;
+  setConfig("UI_SSL_CERT", `${d}/cwmp.crt`);
+  setConfig("UI_SSL_KEY", `${d}/cwmp.key`);
+}
+
+if (fsHostname) {
+  const FS_PORT = allConfig["FS_PORT"] || 7567;
+  const FS_SSL = !!allConfig["FS_SSL_CERT"];
+  setConfig(
+    "FS_URL_PREFIX",
+    (FS_SSL ? "https" : "http") + `://${fsHostname}:${FS_PORT}/`
+  );
+}
+
+// Defaults
+for (const [k, v] of Object.entries(options))
+  if (v["default"] != null) setConfig(k, v["default"]);
+
+export function get(
+  optionName: string,
+  deviceId?: string
+): string | number | boolean {
+  if (!deviceId) return allConfig[optionName];
+
+  optionName = `${optionName}-${deviceId}`;
+  let v = allConfig[optionName];
+  if (v != null) return v;
+
+  let i = optionName.lastIndexOf("-");
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  v = allConfig[optionName.slice(0, i)];
+  if (v != null) return v;
+
+  i = optionName.lastIndexOf("-", i - 1);
+  if (i > 0) {
+    v = allConfig[optionName.slice(0, i)];
+    if (v != null) return v;
+  }
+
+  return null;
+}
+
+export function getDefault(optionName: string): string | number | boolean {
+  const option = options[optionName];
+  if (!option) return null;
+
+  let val = option["default"];
+  if (val && option.type === "path") val = resolve(val);
+
+  return val;
+}

lib/connection-request.ts

@@ -0,0 +1,239 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as crypto from "crypto";
+import * as dgram from "dgram";
+import { parse } from "url";
+import * as http from "http";
+import { evaluateAsync } from "./common/expression";
+import { Expression } from "./types";
+import * as auth from "./auth";
+import * as extensions from "./extensions";
+import * as debug from "./debug";
+
+async function extractAuth(
+  exp: Expression,
+  dflt: any
+): Promise<[string, string, Expression]> {
+  let username, password;
+  const _exp = await evaluateAsync(
+    exp,
+    {},
+    0,
+    async (e: Expression): Promise<Expression> => {
+      if (!username && Array.isArray(e) && e[0] === "FUNC") {
+        if (e[1] === "EXT") {
+          if (typeof e[2] !== "string" || typeof e[3] !== "string") return null;
+
+          for (let i = 4; i < e.length; i++)
+            if (Array.isArray(e[i])) return null;
+
+          const { fault, value } = await extensions.run(e.slice(2));
+          return fault ? null : value;
+        } else if (e[1] === "AUTH") {
+          if (!Array.isArray(e[2]) && !Array.isArray(e[3])) {
+            username = e[2] || "";
+            password = e[3] || "";
+          }
+          return dflt;
+        }
+      }
+      return e;
+    }
+  );
+  return [username, password, _exp];
+}
+
+function httpGet(
+  options: http.RequestOptions,
+  _debug: boolean,
+  deviceId: string
+): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
+  return new Promise((resolve, reject) => {
+    const req = http
+      .get(options, (res) => {
+        res.resume();
+        resolve({ statusCode: res.statusCode, headers: res.headers });
+        if (_debug) {
+          debug.outgoingHttpRequest(req, deviceId, options, null);
+          debug.incomingHttpResponse(res, deviceId, null);
+        }
+      })
+      .on("error", (err) => {
+        req.destroy();
+        reject(err);
+        if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
+      })
+      .on("timeout", () => {
+        req.destroy();
+      });
+  });
+}
+
+export async function httpConnectionRequest(
+  address: string,
+  authExp: Expression,
+  allowBasicAuth: boolean,
+  timeout: number,
+  _debug: boolean,
+  deviceId: string
+): Promise<string> {
+  const options: http.RequestOptions = parse(address);
+  if (options.protocol !== "http:")
+    return "Invalid connection request URL or protocol";
+
+  options.agent = new http.Agent({
+    maxSockets: 1,
+    keepAlive: true,
+    timeout: timeout,
+  });
+
+  let authHeader: Record<string, string>;
+  let username: string;
+  let password: string;
+
+  while (!authHeader || (username != null && password != null)) {
+    let opts = options;
+    if (authHeader) {
+      if (authHeader["method"] === "Basic") {
+        if (!allowBasicAuth) return "Basic HTTP authentication not allowed";
+
+        opts = Object.assign(
+          {
+            headers: {
+              Authorization: auth.basic(username || "", password || ""),
+            },
+          },
+          options
+        );
+      } else if (authHeader["method"] === "Digest") {
+        opts = Object.assign(
+          {
+            headers: {
+              Authorization: auth.solveDigest(
+                username,
+                password,
+                options.path,
+                "GET",
+                null,
+                authHeader
+              ),
+            },
+          },
+          options
+        );
+      } else {
+        return "Unrecognized auth method";
+      }
+    }
+
+    let res: { statusCode: number; headers: http.IncomingHttpHeaders };
+    try {
+      res = await httpGet(opts, _debug, deviceId);
+    } catch (err) {
+      // Workaround for some devices unexpectedly closing the connection
+      if (authHeader) {
+        try {
+          res = await httpGet(opts, _debug, deviceId);
+        } catch (err) {
+          return `Connection request error: ${err.message}`;
+        }
+      }
+
+      if (err["code"] === "ECONNRESET" || err["code"] === "ECONNREFUSED")
+        return "Device is offline";
+
+      return `Connection request error: ${err.message}`;
+    }
+
+    if (res.statusCode === 200 || res.statusCode === 204) return "";
+
+    // When a Connection Request is received for the Virtual CWMP Device and the Proxied
+    // Device is offline the CPE Proxier MUST respond with an HTTP 503 failure
+    if (res.statusCode === 503) return "Device is offline";
+
+    if (res.statusCode === 401 && res.headers["www-authenticate"]) {
+      try {
+        authHeader = auth.parseWwwAuthenticateHeader(
+          res.headers["www-authenticate"]
+        );
+      } catch (err) {
+        return "Connection request error: Error parsing www-authenticate header";
+      }
+      [username, password, authExp] = await extractAuth(authExp, false);
+    } else {
+      return `Connection request error: Unexpected status code ${res.statusCode}`;
+    }
+  }
+
+  return "Connection request error: Incorrect connection request credentials";
+}
+
+export async function udpConnectionRequest(
+  host: string,
+  port: number,
+  authExp: Expression,
+  sourcePort = 0,
+  _debug: boolean,
+  deviceId: string
+): Promise<void> {
+  const now = Date.now();
+
+  const client = dgram.createSocket({ type: "udp4", reuseAddr: true });
+  // When a device is NAT'ed, the UDP Connection Request must originate from
+  // the same address and port used by the STUN server, in order to traverse
+  // the firewall. This does require that the Genieacs NBI and STUN server
+  // are allowed to bind to the same address and port. The STUN server needs
+  // to open its UDP port with the SO_REUSEADDR option, allowing the NBI to
+  // also bind to the same port.
+  if (sourcePort) client.bind({ port: sourcePort, exclusive: true });
+
+  let username: string;
+  let password: string;
+
+  [username, password, authExp] = await extractAuth(authExp, null);
+
+  if (username == null) username = "";
+  if (password == null) password = "";
+  while (username != null && password != null) {
+    const ts = Math.trunc(now / 1000);
+    const id = Math.trunc(Math.random() * 4294967295);
+    const cn = crypto.randomBytes(8).toString("hex");
+    const sig = crypto
+      .createHmac("sha1", password)
+      .update(`${ts}${id}${username}${cn}`)
+      .digest("hex");
+    const uri = `http://${host}:${port}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
+    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${host}:${port}\r\n\r\n`;
+    const message = Buffer.from(msg);
+
+    for (let i = 0; i < 3; ++i) {
+      await new Promise<void>((resolve, reject) => {
+        client.send(message, 0, message.length, port, host, (err: Error) => {
+          if (err) reject(err);
+          else resolve();
+          if (_debug) debug.outgoingUdpMessage(host, deviceId, port, msg);
+        });
+      });
+    }
+
+    [username, password, authExp] = await extractAuth(authExp, null);
+  }
+  client.close();
+}

lib/cwmp.coffee

@@ -1,725 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-util = require 'util'
-zlib = require 'zlib'
-crypto = require 'crypto'
-
-config = require './config'
-common = require './common'
-soap = require './soap'
-session = require './session'
-query = require './query'
-device = require './device'
-cache = require './cache'
-db = require './db'
-logger = require './logger'
-
-MAX_CYCLES = 4
-
-
-throwError = (err, httpResponse) ->
-  if httpResponse
-    httpResponse.writeHead(500, {'Connection' : 'close'})
-    httpResponse.end("#{err.name}: #{err.message}")
-
-  throw err
-
-
-writeResponse = (sessionContext, res) ->
-  if config.get('DEBUG', sessionContext.deviceId)
-    dump = "# RESPONSE #{new Date(Date.now())}\n" + JSON.stringify(res.headers) + "\n#{res.data}\n\n"
-    require('fs').appendFile("./debug/#{sessionContext.deviceId}.dump", dump, (err) ->
-      throwError(err) if err
-    )
-
-  # respond using the same content-encoding as the request
-  if sessionContext.httpRequest.headers['content-encoding']? and res.data.length > 0
-    switch sessionContext.httpRequest.headers['content-encoding']
-      when 'gzip'
-        res.headers['Content-Encoding'] = 'gzip'
-        compress = zlib.gzip
-      when 'deflate'
-        res.headers['Content-Encoding'] = 'deflate'
-        compress = zlib.deflate
-
-  if compress?
-    compress(res.data, (err, data) ->
-      return throwError(err, sessionContext.httpResponse) if err
-      res.headers['Content-Length'] = data.length
-      sessionContext.httpResponse.writeHead(res.code, res.headers)
-      sessionContext.httpResponse.end(data)
-    )
-  else
-    res.headers['Content-Length'] = res.data.length
-    sessionContext.httpResponse.writeHead(res.code, res.headers)
-    sessionContext.httpResponse.end(res.data)
-
-  delete sessionContext.httpRequest
-  delete sessionContext.httpResponse
-
-
-recordFault = (sessionContext, fault, provisions, channels) ->
-  if not provisions
-    provisions = sessionContext.provisions
-    channels = sessionContext.channels
-
-  faults = sessionContext.faults
-
-  for channel of channels
-    provs = sessionContext.faults[channel]?.provisions or []
-    faults[channel] = Object.assign({provisions: provs}, fault)
-
-    if sessionContext.retries[channel]?
-      ++ sessionContext.retries[channel]
-    else
-      sessionContext.retries[channel] = 0
-      if Object.keys(channels).length != 1
-        faults[channel].retryNow = true
-
-    if channels[channel] == 0
-      faults[channel].precondition = true
-
-    sessionContext.faultsTouched ?= {}
-    sessionContext.faultsTouched[channel] = true
-
-    logger.accessError({
-      sessionContext: sessionContext
-      message: 'Channel has faulted'
-      fault: fault
-      channel: channel
-      retries: sessionContext.retries[channel]
-    })
-
-  for provision, i in provisions
-    for channel of channels
-      if (channels[channel] >> i) & 1
-        faults[channel].provisions.push(provision)
-
-  for channel of channels
-    provs = faults[channel].provisions
-    faults[channel].provisions = []
-    appendProvisions(faults[channel].provisions, provs)
-
-  session.clearProvisions(sessionContext)
-  return
-
-
-inform = (sessionContext, rpc) ->
-  session.inform(sessionContext, rpc.cpeRequest, (err, acsResponse) ->
-    return throwError(err, sessionContext.httpResponse) if err
-    res = soap.response({
-      id: rpc.id,
-      acsResponse: acsResponse,
-      cwmpVersion : sessionContext.cwmpVersion
-    })
-
-    if !!cookiesPath = config.get('COOKIES_PATH', sessionContext.deviceId)
-      res.headers['Set-Cookie'] = "session=#{sessionContext.sessionId}; Path=#{cookiesPath}"
-    else
-      res.headers['Set-Cookie'] = "session=#{sessionContext.sessionId}"
-
-    writeResponse(sessionContext, res)
-  )
-
-
-transferComplete = (sessionContext, rpc) ->
-  session.transferComplete(sessionContext, rpc.cpeRequest, (err, acsResponse, operation, fault) ->
-    return throwError(err, sessionContext.httpResponse) if err
-
-    if not operation?
-      logger.accessWarn({
-        sessionContext: sessionContext
-        message: 'Unrecognized command key'
-        rpc: rpc
-      })
-
-    if fault
-      for k, v of operation.retries
-        sessionContext.retries[k] = v
-
-      recordFault(sessionContext, fault)
-
-    res = soap.response({
-      id : rpc.id,
-      acsResponse : acsResponse,
-      cwmpVersion : sessionContext.cwmpVersion
-    })
-    writeResponse(sessionContext, res)
-  )
-
-
-testSchedule = (schedule, timestamp) ->
-  range = schedule.schedule.nextRange(10, new Date(timestamp))
-  prev = schedule.schedule.prevRange(1, new Date(timestamp), new Date(timestamp - schedule.duration))
-
-  if prev
-    first = +(prev[0] ? 0)
-    last = +(prev[1] ? 0) + schedule.duration
-  else
-    first = +(range[0][0] ? 0)
-    last = +(range[0][1] ? 0) + schedule.duration
-
-  for r in range
-    if last < r[0]
-      break
-    last = +(r[1] ? 0) + schedule.duration
-
-  if timestamp >= first
-    return [true, Math.max(0, last - timestamp)]
-  else
-    return [false, first - timestamp]
-
-
-# Append providions and remove duplicates
-appendProvisions = (original, toAppend) ->
-  modified = false
-  stringified = new WeakMap()
-  for p, i in original
-    stringified.set(p, JSON.stringify(p))
-
-  for p, i in toAppend by -1
-    s = JSON.stringify(p)
-    for pp, j in original by -1
-      ss = stringified.get(pp)
-      if s == ss
-        if (not p) or j >= original.length - (toAppend.length - i)
-          p = null
-        else
-          original.splice(j, 1)
-          modified = true
-    if p
-      original.splice(original.length - (toAppend.length - i) + 1, 0, p)
-      stringified.set(p, s)
-      modified = true
-
-  return modified
-
-
-applyPresets = (sessionContext) ->
-  cache.getPresets((err, presetsHash, presets) ->
-    return throwError(err, sessionContext.httpResponse) if err
-
-    # Filter presets based on existing faults
-    blackList = {}
-    whiteList = null
-    whiteListProvisions = null
-
-    RETRY_DELAY = config.get('RETRY_DELAY', sessionContext.deviceId)
-
-    if sessionContext.faults?
-      for channel, fault of sessionContext.faults
-        if fault.retryNow
-          retryTimestamp = 0
-        else
-          retryTimestamp = fault.timestamp + (RETRY_DELAY * Math.pow(2, sessionContext.retries[channel])) * 1000
-
-        if retryTimestamp <= sessionContext.timestamp
-          whiteList = channel
-          whiteListProvisions = fault.provisions
-          break
-
-        blackList[channel] = if fault.precondition then 1 else 2
-
-    deviceEvents = {}
-    for p in sessionContext.deviceData.paths.find(['Events', '*'], false, true)
-      if sessionContext.timestamp <= sessionContext.deviceData.attributes.get(p)?.value?[1][0]
-        deviceEvents[p[1]] = true
-
-    parameters = {}
-    filteredPresets = []
-
-    for preset in presets
-      if whiteList?
-        continue if preset.channel != whiteList
-      else if blackList[preset.channel] == 1
-        continue
-
-      eventsMatch = true
-      for k, v of preset.events
-        if (!v) != (!deviceEvents[k.replace(' ', '_')])
-          eventsMatch = false
-          break
-
-      continue if (not eventsMatch) or (preset.schedule and not
-        (preset.schedule.schedule and testSchedule(preset.schedule, sessionContext.timestamp)[0]))
-
-      filteredPresets.push(preset)
-      for k of preset.precondition
-        sessionContext.channels[preset.channel] = 0
-        p = k.split(/([^a-zA-Z0-9\-\_\.].*)/, 1)[0]
-        parameters[p] = common.parsePath(p)
-
-    declarations = []
-    for k, v of parameters
-      declarations.push([v, 1, {value: 1}])
-
-    session.rpcRequest(sessionContext, declarations, (err, fault, id, acsRequest) ->
-      return throwError(err, sessionContext.httpResponse) if err
-
-      if fault
-        recordFault(sessionContext, fault)
-        session.clearProvisions(sessionContext)
-        return applyPresets(sessionContext)
-
-      if acsRequest
-        return sendAcsRequest(sessionContext, id, acsRequest)
-
-      session.clearProvisions(sessionContext)
-
-      for k, v of parameters
-        unpacked = device.unpack(sessionContext.deviceData, v)
-        if unpacked[0] and (vv = sessionContext.deviceData.attributes.get(unpacked[0]).value?[1])?
-          parameters[k] = vv[0]
-        else
-          delete parameters[k]
-
-      if whiteList?
-        session.addProvisions(sessionContext, whiteList, whiteListProvisions)
-
-      appendProvisionsToFaults = {}
-      for p in filteredPresets
-        if query.testFilter(parameters, p.precondition)
-          if blackList[p.channel] == 2
-            appendProvisionsToFaults[p.channel] =
-              (appendProvisionsToFaults[p.channel] or []).concat(p.provisions)
-          else
-            session.addProvisions(sessionContext, p.channel, p.provisions)
-
-      for channel, provisions of appendProvisionsToFaults
-        if appendProvisions(sessionContext.faults[channel].provisions, provisions)
-          sessionContext.faultsTouched ?= {}
-          sessionContext.faultsTouched[channel] = true
-
-      sessionContext.presetCycles = (sessionContext.presetCycles or 0) + 1
-
-      if sessionContext.presetCycles > MAX_CYCLES
-        fault = {
-          code: 'endless_cycle'
-          message: 'The provision seems to be repeating indefinitely'
-          timestamp: sessionContext.timestamp
-        }
-        recordFault(sessionContext, fault)
-        session.clearProvisions(sessionContext)
-        return sendAcsRequest(sessionContext)
-
-      session.rpcRequest(sessionContext, null, (err, fault, id, acsRequest) ->
-        return throwError(err, sessionContext.httpResponse) if err
-
-        if fault
-          recordFault(sessionContext, fault)
-          session.clearProvisions(sessionContext)
-          return applyPresets(sessionContext)
-
-        if not acsRequest
-          for channel, flags of sessionContext.channels
-            if channel of sessionContext.faults
-              delete sessionContext.faults[channel]
-              sessionContext.faultsTouched ?= {}
-              sessionContext.faultsTouched[channel] = true
-
-          if whiteList?
-            return applyPresets(sessionContext)
-
-        sendAcsRequest(sessionContext, id, acsRequest)
-      )
-    )
-  )
-
-
-nextRpc = (sessionContext) ->
-  session.rpcRequest(sessionContext, null, (err, fault, id, acsRequest) ->
-    return throwError(err, sessionContext.httpResponse) if err
-
-    if fault
-      recordFault(sessionContext, fault)
-      session.clearProvisions(sessionContext)
-      return nextRpc(sessionContext)
-
-    if acsRequest
-      return sendAcsRequest(sessionContext, id, acsRequest)
-
-    for channel, flags of sessionContext.channels
-      if flags and channel of sessionContext.faults
-        delete sessionContext.faults[channel]
-        sessionContext.faultsTouched ?= {}
-        sessionContext.faultsTouched[channel] = true
-
-      if channel.startsWith('task_')
-        taskId = channel.slice(5)
-        sessionContext.doneTasks ?= []
-        sessionContext.doneTasks.push(taskId)
-        for t, j in sessionContext.tasks
-          if t._id == taskId
-            sessionContext.tasks.splice(j, 1)
-            break
-
-    session.clearProvisions(sessionContext)
-
-    for task in sessionContext.tasks
-      channel = "task_#{task._id}"
-
-      # Delete if expired
-      if task.expiry <= sessionContext.timestamp
-        logger.accessInfo({
-          sessionContext: sessionContext
-          message: 'Task expired'
-          task: task
-        })
-        sessionContext.doneTasks ?= []
-        sessionContext.doneTasks.push(String(task._id))
-        if channel of sessionContext.faults
-          delete sessionContext.faults[channel]
-          sessionContext.faultsTouched ?= {}
-          sessionContext.faultsTouched[channel] = true
-        continue
-
-      if sessionContext.faults[channel]
-        continue
-
-      switch task.name
-        when 'getParameterValues'
-          # Set channel in case params array is empty
-          sessionContext.channels["task_#{task._id}"] = 0
-          for p in task.parameterNames
-            session.addProvisions(sessionContext, "task_#{task._id}",
-              [['refresh', p]])
-        when 'setParameterValues'
-          # Set channel in case params array is empty
-          sessionContext.channels["task_#{task._id}"] = 0
-          for p in task.parameterValues
-            session.addProvisions(sessionContext, "task_#{task._id}",
-              [['value', p[0], p[1]]])
-        when 'refreshObject'
-          session.addProvisions(sessionContext, "task_#{task._id}",
-            [['refresh', task.objectName]])
-        when 'reboot'
-          session.addProvisions(sessionContext, "task_#{task._id}",
-            [['reboot']])
-        when 'factoryReset'
-          session.addProvisions(sessionContext, "task_#{task._id}",
-            [['reset']])
-        when 'download'
-          session.addProvisions(sessionContext, "task_#{task._id}",
-            [['download', task.fileType, task.fileName, task.targetFileName]])
-        when 'addObject'
-          alias = ("#{p[0]}:#{JSON.stringify(p[1])}" for p in task.parameterValues or []).join(',')
-          session.addProvisions(sessionContext, "task_#{task._id}",
-            [['instances', "#{task.objectName}.[#{alias}]", "+1"]]);
-        when 'deleteObject'
-          session.addProvisions(sessionContext, "task_#{task._id}",
-            [['instances', task.objectName, 0]]);
-        else
-          return throwError(new Error('Task name not recognized'), sessionContext.httpResponse) if err
-
-      return nextRpc(sessionContext)
-
-    return applyPresets(sessionContext)
-  )
-
-
-sendAcsRequest = (sessionContext, id, acsRequest) ->
-  if not acsRequest?
-    session.end(sessionContext, (err, isNew) ->
-      return throwError(err, sessionContext.httpResponse) if err
-
-      delete currentSessions.get(sessionContext.httpRequest.connection)[sessionContext.sessionId]
-      if isNew
-        logger.accessInfo({
-          sessionContext: sessionContext
-          message: 'New device registered'
-        })
-
-      db.clearTasks(sessionContext.deviceId, sessionContext.doneTasks, (err) ->
-        return throwError(err, sessionContext.httpResponse) if err
-
-        counter = 3
-
-        for k of sessionContext.faultsTouched
-          counter += 2
-          if sessionContext.faults[k]
-            sessionContext.faults[k].retries = sessionContext.retries[k]
-            db.saveFault(sessionContext.deviceId, k, sessionContext.faults[k], (err) ->
-              if err
-                throwError(err, sessionContext.httpResponse) if counter & 1
-                return counter = 0
-
-              if (counter -= 2) == 1
-                return writeResponse(sessionContext, soap.response(null))
-            )
-          else
-            db.deleteFault(sessionContext.deviceId, k, (err) ->
-              if err
-                throwError(err, sessionContext.httpResponse) if counter & 1
-                return counter = 0
-
-              if (counter -= 2) == 1
-                return writeResponse(sessionContext, soap.response(null))
-            )
-
-        if (counter -= 2) == 1
-          return writeResponse(sessionContext, soap.response(null))
-      )
-    )
-    return
-
-  if acsRequest.name is 'Download'
-    if not acsRequest.url?
-      FS_PORT = config.get('FS_PORT')
-      FS_HOSTNAME = config.get('FS_HOSTNAME')
-      FS_SSL = config.get('FS_SSL')
-      acsRequest.url = if FS_SSL then 'https://' else 'http://'
-      acsRequest.url += FS_HOSTNAME
-      acsRequest.url += ":#{FS_PORT}" if FS_PORT != 80
-      acsRequest.url += "/#{encodeURIComponent(acsRequest.fileName)}"
-
-    if not acsRequest.fileSize?
-      return cache.getFiles((err, hash, files) ->
-        return throwError(err, sessionContext.httpResponse) if err
-
-        if acsRequest.fileName of files
-          acsRequest.fileSize = files[acsRequest.fileName].length
-        else
-          acsRequest.fileSize = 0
-
-        return sendAcsRequest(sessionContext, id, acsRequest)
-      )
-
-  rpc = {
-    id : id,
-    acsRequest : acsRequest,
-    cwmpVersion : sessionContext.cwmpVersion
-  }
-
-  logger.accessInfo({
-    sessionContext: sessionContext
-    message: 'ACS request'
-    rpc: rpc
-  })
-
-  res = soap.response(rpc)
-
-  writeResponse(sessionContext, res)
-
-
-getSession = (httpRequest, callback) ->
-  # Separation by comma is important as some devices don't comform to standard
-  COOKIE_REGEX = /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g
-  while match = COOKIE_REGEX.exec(httpRequest.headers.cookie)
-    sessionId = match[2] if match[1] == 'session'
-
-  return callback() if not sessionId?
-
-  sessionContext = currentSessions.get(httpRequest.connection)?[sessionId]
-
-  if sessionContext?
-    return callback(null, sessionContext)
-
-  setTimeout(() ->
-    db.redisClient.eval('local v=redis.call("get",KEYS[1]);redis.call("del",KEYS[1]);return v;', 1, "session_#{sessionId}", (err, sessionContextString) ->
-      return callback(err) if err or not sessionContextString
-      session.deserialize(sessionContextString, (err, sessionContext) ->
-        return callback(err) if err
-        currentSessions.get(httpRequest.connection)[sessionId] = sessionContext
-        callback(null, sessionContext)
-      )
-    )
-  , 100
-  )
-
-
-currentSessions = new WeakMap()
-
-# When socket closes, store active sessions in redis
-onConnection = (socket) ->
-  currentSessions.set(socket, {})
-  socket.on('close', () ->
-    sessions = currentSessions.get(socket)
-    for sessionId, sessionContext of sessions
-      delete sessionContext.httpRequest
-      delete sessionContext.httpResponse
-      session.serialize(sessionContext, (err, sessionContextString) ->
-        return throwError(err) if err
-        # TODO don't set if process is shutting down
-        db.redisClient.setex("session_#{sessionId}", sessionContext.timeout, sessionContextString, (err) ->
-          return throwError(err) if err
-        )
-      )
-  )
-
-
-listener = (httpRequest, httpResponse) ->
-  if httpRequest.method != 'POST'
-    httpResponse.writeHead 405, {'Allow': 'POST'}
-    httpResponse.end('405 Method Not Allowed')
-    return
-
-  if httpRequest.headers['content-encoding']?
-    switch httpRequest.headers['content-encoding']
-      when 'gzip'
-        stream = httpRequest.pipe(zlib.createGunzip())
-      when 'deflate'
-        stream = httpRequest.pipe(zlib.createInflate())
-      else
-        httpResponse.writeHead(415)
-        httpResponse.end('415 Unsupported Media Type')
-        return
-  else
-    stream = httpRequest
-
-  chunks = []
-  bytes = 0
-
-  stream.on('data', (chunk) ->
-    chunks.push(chunk)
-    bytes += chunk.length
-  )
-
-  httpRequest.getBody = () ->
-    # Write all chunks into a Buffer
-    body = new Buffer(bytes)
-    offset = 0
-    chunks.forEach((chunk) ->
-      chunk.copy(body, offset, 0, chunk.length)
-      offset += chunk.length
-    )
-    return body
-
-  stream.on('end', () ->
-    rpc = null
-    parseWarnings = []
-    getSession(httpRequest, f = (err, sessionContext) ->
-      return throwError(err, httpResponse) if err
-      rpc ?= soap.request(httpRequest, sessionContext?.cwmpVersion, parseWarnings)
-      if not sessionContext?
-        if rpc.cpeRequest?.name isnt 'Inform'
-          httpResponse.writeHead(400)
-          httpResponse.end('Session is expired')
-          return
-
-        deviceId = common.generateDeviceId(rpc.cpeRequest.deviceId)
-        return session.init(deviceId, rpc.cwmpVersion, rpc.sessionTimeout ? config.get('SESSION_TIMEOUT', deviceId), (err, sessionContext) ->
-          return throwError(err, httpResponse) if err
-
-          sessionContext.sessionId = crypto.randomBytes(8).toString('hex')
-
-          currentSessions.get(httpRequest.connection)[sessionContext.sessionId] = sessionContext
-
-          httpRequest.connection.setTimeout(sessionContext.timeout * 1000)
-
-          db.getDueTasksAndFaultsAndOperations(deviceId, sessionContext.timestamp, (err, dueTasks, faults, operations) ->
-            return throwError(err, httpResponse) if err
-            sessionContext.tasks = dueTasks
-            sessionContext.faults = faults
-            sessionContext.retries = {}
-            for k, v of faults
-              sessionContext.retries[k] = v.retries
-            sessionContext.operations = operations
-
-            # Delete expired faults
-            for k, v of sessionContext.faults
-              if v.expiry >= sessionContext.timestamp
-                delete sessionContext.faults[k]
-                sessionContext.faultsTouched ?= {}
-                sessionContext.faultsTouched[k] = true
-
-            f(null, sessionContext)
-          )
-        )
-
-      sessionContext.httpRequest = httpRequest
-      sessionContext.httpResponse = httpResponse
-
-      for w in parseWarnings
-        w.sessionContext = sessionContext
-        w.rpc = rpc;
-        logger.accessWarn(w)
-
-      if config.get('DEBUG', sessionContext.deviceId)
-        dump = "# REQUEST #{new Date(Date.now())}\n" + JSON.stringify(httpRequest.headers) + "\n#{httpRequest.getBody()}\n\n"
-        require('fs').appendFile("./debug/#{sessionContext.deviceId}.dump", dump, (err) ->
-          return throwError(err) if err
-        )
-
-      if rpc.cpeRequest?
-        if rpc.cpeRequest.name is 'Inform'
-          logger.accessInfo({
-            sessionContext: sessionContext
-            message: 'Inform'
-            rpc: rpc
-          })
-          inform(sessionContext, rpc)
-        else if rpc.cpeRequest.name is 'TransferComplete'
-          logger.accessInfo({
-            sessionContext: sessionContext
-            message: 'CPE request'
-            rpc: rpc
-          })
-          transferComplete(sessionContext, rpc)
-        else if rpc.cpeRequest.name is 'GetRPCMethods'
-          logger.accessInfo({
-            sessionContext: sessionContext
-            message: 'CPE request'
-            rpc: rpc
-          })
-          res = soap.response({
-            id : rpc.id
-            acsResponse : {name: 'GetRPCMethodsResponse', methodList: ['Inform', 'GetRPCMethods', 'TransferComplete']}
-            cwmpVersion : sessionContext.cwmpVersion
-          })
-          writeResponse(sessionContext, res)
-        else
-          return throwError(new Error('ACS method not supported'), sessionContext.httpResponse)
-      else if rpc.cpeResponse
-        session.rpcResponse(sessionContext, rpc.id, rpc.cpeResponse, (err) ->
-          return throwError(err, sessionContext.httpResponse) if err
-          nextRpc(sessionContext)
-        )
-      else if rpc.cpeFault
-        logger.accessWarn({
-          sessionContext: sessionContext
-          message: 'CPE fault'
-          rpc:rpc
-        })
-        session.rpcFault(sessionContext, rpc.id, rpc.cpeFault, (err, fault) ->
-          return throwError(err, sessionContext.httpResponse) if err
-
-          if fault
-            recordFault(sessionContext, fault)
-            session.clearProvisions(sessionContext)
-          nextRpc(sessionContext)
-        )
-      else # CPE sent empty response
-        session.timeoutOperations(sessionContext, (err, faults, operations) ->
-          return throwError(err, sessionContext.httpResponse) if err
-
-          for fault, i in faults
-            for k, v of operations[i].retries
-              sessionContext.retries[k] = v
-
-            recordFault(sessionContext, fault, operations[i].provisions, operations[i].channels)
-
-          nextRpc(sessionContext)
-        )
-    )
-  )
-
-
-exports.listener = listener
-exports.onConnection = onConnection

lib/db.coffee

@@ -1,663 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-config = require './config'
-mongodb = require 'mongodb'
-redis = require('redis')
-common = require './common'
-
-redisClient = null
-tasksCollection = null
-devicesCollection = null
-presetsCollection = null
-objectsCollection = null
-provisionsCollection = null
-virtualParametersCollection = null
-faultsCollection = null
-filesCollection = null
-operationsCollection = null
-
-
-connect = (callback) ->
-  callbackCounter = 10
-  mongodb.MongoClient.connect(config.get('MONGODB_CONNECTION_URL'), (err, db) ->
-    return callback(err) if err
-    exports.mongoDb = db
-    db.collection('tasks', (err, collection) ->
-      exports.tasksCollection = tasksCollection = collection
-      collection?.ensureIndex({device: 1, timestamp: 1}, (err) ->
-      )
-
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('devices', (err, collection) ->
-      exports.devicesCollection  = devicesCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('presets', (err, collection) ->
-      exports.presetsCollection = presetsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('objects', (err, collection) ->
-      exports.objectsCollection = objectsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('fs.files', (err, collection) ->
-      exports.filesCollection = filesCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('provisions', (err, collection) ->
-      exports.provisionsCollection = provisionsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('virtualParameters', (err, collection) ->
-      exports.virtualParametersCollection = virtualParametersCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('faults', (err, collection) ->
-      exports.faultsCollection = faultsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    db.collection('operations', (err, collection) ->
-      exports.operationsCollection = operationsCollection = collection
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-
-    exports.redisClient = redisClient = redis.createClient(config.get('REDIS_PORT'), config.get('REDIS_HOST'))
-    redisClient.select(config.get('REDIS_DB'), (err) ->
-      if --callbackCounter == 0 or err
-        callbackCounter = 0
-        return callback(err)
-    )
-  )
-
-
-disconnect = () ->
-  exports.mongoDb?.close()
-  redisClient?.quit()
-
-
-# Optimize projection by removing overlaps
-# This modifies given object and returns it
-optimizeProjection = (obj) ->
-  keys = Object.keys(obj).sort()
-  return if keys.length <= 1
-  i = 1
-  while i < keys.length
-    a = keys[i-1]
-    b = keys[i]
-    if b.startsWith(a)
-      if b.charAt(a.length) == '.' or b.charAt(a.length - 1) == '.'
-        delete obj[b]
-        keys.splice(i--, 1)
-    ++ i
-  return obj
-
-
-fetchDevice = (id, timestamp, patterns, callback) ->
-  MAX_DEPTH = config.get('MAX_DEPTH', id)
-
-  projection = {_id: 1}
-  projectionTree = {}
-  func = (path, pats, projTree) ->
-    children = {}
-    for pat in pats
-      fragment = pat[0][path.length] or '*'
-      load = 0
-      if fragment == '*'
-        projection[path.join('.')] = 1
-        return
-
-      projTree[fragment] ?= {}
-
-      if pat[1] & (1 << path.length)
-        projection[path.concat('_timestamp').join('.')] = 1
-        projection[path.concat([fragment, '_timestamp']).join('.')] = 1
-        projection[path.concat([fragment, '_value']).join('.')] = 1
-        projection[path.concat([fragment, '_type']).join('.')] = 1
-        projection[path.concat([fragment, '_object']).join('.')] = 1
-        projection[path.concat([fragment, '_instance']).join('.')] = 1
-        projection[path.concat([fragment, '_writable']).join('.')] = 1
-        projection[path.concat([fragment, '_orig']).join('.')] = 1
-
-      if pat[1] >> (path.length + 1)
-        children[fragment] ?= []
-        children[fragment].push(pat)
-
-    for k, v of children
-      func(path.concat(k), v, projTree[k])
-
-  func([], patterns, projectionTree)
-
-  delete projectionTree['DeviceID']
-  delete projectionTree['Events']
-  delete projectionTree['Tags']
-
-  if projection['']?
-    proj = {}
-  else
-    proj = JSON.parse(JSON.stringify(projection))
-
-  for k, v of proj
-    if k == '' or k == 'Events' or k.startsWith('Events.')
-      if k.length < 7 or k.startsWith('Events.Inform.') or k == 'Events.Inform'
-        proj['_lastInform'] = 1
-      if k.length < 7 or k.startsWith('Events.1_BOOT.') or k == 'Events.1_BOOT'
-        proj['_lastBoot'] = 1
-      if k.length < 7 or k.startsWith('Events.0_BOOTSTRAP.') or k == 'Events.0_BOOTSTRAP'
-        proj['_lastBootstrap'] = 1
-      if k.length < 7 or k.startsWith('Events.Registered.') or k == 'Events.Registered'
-        proj['_registered'] = 1
-      delete proj[k] if k != ''
-
-    if k == '' or k == 'DeviceID' or k.startsWith('DeviceID.')
-      if k.length < 9 or k.startsWith('DeviceID.ID.') or k == 'DeviceID.ID'
-        proj['_id'] = 1
-      if k.length < 9 or k.startsWith('DeviceID.Manufacturer.') or k == 'DeviceID.DeManufacturer'
-        proj['_deviceId._Manufacturer'] = 1
-      if k.length < 9 or k.startsWith('DeviceID.ProductClass.') or k == 'DeviceID.ProductClass'
-        proj['_deviceId._ProductClass'] = 1
-      if k.length < 9 or k.startsWith('DeviceID.OUI.') or k == 'DeviceID.OUI'
-        proj['_deviceId._OUI'] = 1
-      if k.length < 9 or k.startsWith('DeviceID.SerialNumber.') or k == 'DeviceID.SerialNumber'
-        proj['_deviceId._SerialNumber'] = 1
-      if k.length < 9
-        proj['_deviceId'] = 1
-      delete proj[k] if k != ''
-
-    if k == '' or k == 'Tags' or k.startsWith('Tags.')
-      proj['_tags'] = 1
-      delete proj[k] if k != ''
-
-  optimizeProjection(proj)
-
-  res = []
-  loaded = []
-
-  devicesCollection.findOne({'_id' : id}, proj, (err, device) ->
-    return callback(err) if err or not device?
-
-    storeParams = (obj, path, timestamp, descendantsFetched, projTree) ->
-      if descendantsFetched
-        thisFetched = true
-      else
-        if projection[path.join('.')]
-          descendantsFetched = true
-          loaded.push([path, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
-        if projection[path.concat('_writable').join('.')]
-          loaded.push([path, 1 << (path.length - 1)])
-          thisFetched = true
-
-      if obj['_timestamp']?
-        obj['_timestamp'] = +obj['_timestamp']
-
-      if thisFetched
-        attrs = {}
-
-        t = if obj['_timestamp'] > timestamp then obj['_timestamp'] else timestamp
-
-        if obj['_value']?
-          attrs.value = [obj['_timestamp'] or 1, [obj['_value'], obj['_type']]]
-          attrs.value[1][0] = +attrs.value[1][0] if obj['_type'] == 'xsd:dateTime'
-          obj['_object'] = false
-
-        if obj['_writable']?
-          attrs.writable = [timestamp, if obj['_writable'] then 1 else 0]
-
-        if obj['_object']? or obj['_instance']?
-          attrs.object = [t, if (obj['_object'] or obj['_instance']) then 1 else 0]
-
-        res.push([path, t, attrs])
-
-      for k, v of obj
-        if k[0] != '_'
-          kk = k
-          obj['_object'] = true
-          storeParams(v, path.concat(k), obj['_timestamp'] or 1, descendantsFetched, projTree?[k])
-          delete projTree[kk] if projTree
-
-      if obj['_object'] or obj['_instance']
-        if descendantsFetched
-          res.push([path.concat('*'), obj['_timestamp']]) if obj['_timestamp']
-        else
-          for k, v of projTree
-            p = path.concat(k)
-            res.push([p, obj['_timestamp']]) if obj['_timestamp']
-            loaded.push([p, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
-
-    for k, v of device
-      switch k
-        when '_lastInform'
-          res.push([['Events'], timestamp,
-            {object: [timestamp, 1], writable: [timestamp, 0]}])
-          res.push([['Events', 'Inform'], +v,
-            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_lastBoot'
-          res.push([['Events'], timestamp,
-            {object: [timestamp, 1], writable: [timestamp, 0]}])
-          res.push([['Events', '1_BOOT'], +v,
-            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_lastBootstrap'
-          res.push([['Events'], timestamp,
-            {object: [timestamp, 1], writable: [timestamp, 0]}])
-          res.push([['Events', '0_BOOTSTRAP'], +v,
-            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_registered'
-          res.push([['Events'], timestamp,
-            {object: [timestamp, 1], writable: [timestamp, 0]}])
-          # Use current timestamp for registered event attribute timestamps
-          res.push([['Events', 'Registered'], timestamp,
-            {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [+v, 'xsd:dateTime']]}])
-          delete device[k]
-        when '_id'
-          res.push([['DeviceID'], timestamp,
-            {object: [timestamp, 1], writable: [timestamp, 0]}])
-          res.push([['DeviceID', 'ID'], timestamp,
-            {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v, 'xsd:string']]}])
-          delete device[k]
-        when '_tags'
-          res.push([['Tags'], timestamp,
-            {object: [timestamp, 1], writable: [timestamp, 0]}])
-          for t in v
-            t = t.replace(/[^a-zA-Z0-9\-]+/g, '_')
-            res.push([['Tags', t], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 1], value: [timestamp, [true, 'xsd:boolean']]}])
-          delete device[k]
-        when '_deviceId'
-          res.push([['DeviceID'], timestamp,
-            {object: [timestamp, 1], writable: [timestamp, 0]}])
-          if v['_Manufacturer']?
-            res.push([['DeviceID', 'Manufacturer'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_Manufacturer'], 'xsd:string']]}])
-          if v['_OUI']?
-            res.push([['DeviceID', 'OUI'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_OUI'], 'xsd:string']]}])
-          if v['_ProductClass']?
-            res.push([['DeviceID', 'ProductClass'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_ProductClass'], 'xsd:string']]}])
-          if v['_SerialNumber']?
-            res.push([['DeviceID', 'SerialNumber'], timestamp,
-              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_SerialNumber'], 'xsd:string']]}])
-          delete device[k]
-
-    storeParams(device, [], 0, false, projectionTree)
-
-    return callback(null, res, loaded)
-  )
-
-
-saveDevice = (deviceId, deviceData, isNew, callback) ->
-  update = {'$set' : {}, '$unset' : {}, '$addToSet' : {}, '$pull' : {}}
-
-  iter = deviceData.timestamps.diff()
-  while diff = iter.next().value
-    continue if diff[0].wildcard != (1 << (diff[0].length - 1))
-    continue if diff[0][0] in ['Events', 'DeviceID', 'Tags']
-
-    if not diff[2]? and diff[1]?
-      update['$unset'][diff[0].slice(0, -1).concat('_timestamp').join('.')] = 1
-    else if diff[2] != diff[1]
-      parent = deviceData.paths.get(diff[0].slice(0, -1))
-      if parent and (parent.length == 0 or deviceData.attributes.has(parent))
-        update['$set'][diff[0].slice(0, -1).concat('_timestamp').join('.')] = new Date(diff[2])
-
-  iter = deviceData.attributes.diff()
-  while diff = iter.next().value
-    continue if diff[1] == diff[2]
-    path = diff[0]
-    switch path[0]
-      when 'Events'
-        if diff[0].length == 2 and diff[2]?.value?[1][0] != diff[1]?.value?[1][0]
-          if not diff[2]
-            switch path[1]
-              when 'Inform'
-                update['$unset']['_lastInform'] = 1
-              when '1_BOOT'
-                update['$unset']['_lastBoot'] = 1
-              when '0_BOOTSTRAP'
-                update['$unset']['_lastBootstrap'] = 1
-              when 'Registered'
-                update['$unset']['_registered'] = 1
-          else
-            t = new Date(diff[2].value[1][0])
-            switch path[1]
-              when 'Inform'
-                update['$set']['_lastInform'] = t
-              when '1_BOOT'
-                update['$set']['_lastBoot'] = t
-              when '0_BOOTSTRAP'
-                update['$set']['_lastBootstrap'] = t
-              when 'Registered'
-                update['$set']['_registered'] = t
-      when 'DeviceID'
-        if diff[2].value?[1]?[0] != diff[1]?.value?[1]?[0]
-          v = diff[2].value[1][0]
-          switch path[1]
-            when 'ID'
-              update['$set']['_id'] = v
-            when 'Manufacturer'
-              update['$set']['_deviceId._Manufacturer'] = v
-            when 'OUI'
-              update['$set']['_deviceId._OUI'] = v
-            when 'ProductClass'
-              update['$set']['_deviceId._ProductClass'] = v
-            when 'SerialNumber'
-              update['$set']['_deviceId._SerialNumber'] = v
-      when 'Tags'
-        if diff[2]?.value?[1]?[0] != diff[1]?.value?[1]?[0]
-          v = diff[2]?.value?[1][0]
-          if v?
-            update['$addToSet']['_tags'] ?= {'$each' : []}
-            update['$addToSet']['_tags']['$each'].push(diff[0][1])
-          else
-            update['$pull']['_tags'] ?= {'$in' : []}
-            update['$pull']['_tags']['$in'].push(diff[0][1])
-      else
-        if not diff[2]
-          update['$unset'][diff[0].join('.')] = 1
-          continue
-
-        for attrName of diff[2]
-          if diff[2][attrName][1]? or diff[1]?[attrName]?[1]?
-            switch attrName
-              when 'value'
-                if not diff[1] or diff[2].value[1][0] != diff[1].value?[1][0]
-                  if diff[2].value[1][1] == 'xsd:dateTime'
-                    update['$set'][path.concat('_value').join('.')] = new Date(diff[2].value[1][0])
-                  else
-                    update['$set'][path.concat('_value').join('.')] = diff[2].value[1][0]
-                if not diff[1] or diff[2].value[1][1] != diff[1].value?[1][1]
-                  update['$set'][path.concat('_type').join('.')] = diff[2].value[1][1]
-                if not diff[1] or diff[2].value[0] != diff[1].value?[0]
-                  update['$set'][path.concat('_timestamp').join('.')] = new Date(diff[2].value[0])
-              when 'object'
-                if not diff[1]?.object or diff[2].object[1] != diff[1].object?[1]
-                  update['$set'][path.concat('_object').join('.')] = !!diff[2].object[1]
-              when 'writable'
-                if not diff[1]?.writable or diff[2].writable[1] != diff[1].writable?[1]
-                  update['$set'][path.concat('_writable').join('.')] = !!diff[2].writable[1]
-
-        if diff[1]
-          for attrName of diff[1]
-            if attrName not of diff[2]
-              update['$unset'][path.concat("_#{attrName}").join('.')] = 1
-              if attrName is 'value'
-                update['$unset'][path.concat('_type').join('.')] = 1
-                update['$unset'][path.concat('_timestamp').join('.')] = 1
-
-  # Remove empty keys
-  for k of update
-    if k == '$addToSet'
-      for kk of update[k]
-        delete update[k][kk] if update[k][kk]['$each'].length == 0
-    else if k == '$pull'
-      for kk of update[k]
-        delete update[k][kk] if update[k][kk]['$in'].length == 0
-    delete update[k] if Object.keys(update[k]).length == 0
-
-  return callback() if Object.keys(update).length == 0
-
-  if (update['$unset']?)
-    optimizeProjection(update['$unset'])
-
-  devicesCollection.update({'_id' : deviceId}, update, {upsert: isNew}, (err, result) ->
-    if not err and result.result.n != 1
-      return callback(new Error("Device #{deviceId} not found in database"))
-
-    return callback(err)
-  )
-
-
-getDueTasksAndFaultsAndOperations = (deviceId, timestamp, callback) ->
-  redisClient.mget("#{deviceId}_tasks", "#{deviceId}_faults", "#{deviceId}_operations", (err, res) ->
-    return callback(err) if err
-
-    if res[0]
-      tasks = JSON.parse(res[0])
-
-    if res[1]?
-      faults = JSON.parse(res[1])
-
-    if res[2]?
-      operations = JSON.parse(res[2])
-
-    if tasks? and faults? and operations?
-      return callback(null, tasks, faults, operations)
-
-    MAX_CACHE_TTL = config.get('MAX_CACHE_TTL', deviceId)
-
-    if not faults?
-      getFaults(deviceId, (err, flts) ->
-        if err
-          callback?(err)
-          return callback = null
-
-        faults = flts
-        if tasks? and operations?
-          return callback(null, tasks, faults, operations)
-      )
-
-    if not tasks?
-      getDueTasks(deviceId, timestamp, (err, dueTasks, nextTimestamp) ->
-        if err
-          callback?(err)
-          return callback = null
-
-        if nextTimestamp?
-          exp = Math.min(0, Math.trunc((nextTimestamp - Date.now()) / 1000))
-        else
-          exp = MAX_CACHE_TTL
-
-        redisClient.setex("#{deviceId}_tasks", exp, JSON.stringify(dueTasks), (err) ->
-          if err
-            callback?(err)
-            return callback = null
-
-          tasks = dueTasks
-          if faults? and operations?
-            return callback(null, tasks, faults, operations)
-        )
-      )
-
-    if not operations?
-      getOperations(deviceId, (err, _operations) ->
-        if err
-          callback?(err)
-          return callback = null
-
-        operations = _operations
-        if tasks? and faults?
-          return callback(null, tasks, faults, operations)
-      )
-
-  )
-
-
-getFaults = (deviceId, callback) ->
-  faultsCollection.find({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}).toArray((err, res) ->
-    return callback(err) if err
-
-    faults = {}
-    for r in res
-      channel = r._id.slice(deviceId.length + 1)
-      delete r._id
-      delete r.channel
-      delete r.device
-      r.timestamp = +r.timestamp
-      r.provisions = JSON.parse(r.provisions)
-      faults[channel] = r
-
-    MAX_CACHE_TTL = config.get('MAX_CACHE_TTL', deviceId)
-
-    redisClient.setex("#{deviceId}_faults", MAX_CACHE_TTL, JSON.stringify(faults), (err) ->
-      callback(err, faults)
-    )
-  )
-
-
-saveFault = (deviceId, channel, fault, callback) ->
-  fault._id = "#{deviceId}:#{channel}"
-  fault.device = deviceId
-  fault.channel = channel
-  fault.timestamp = new Date(fault.timestamp)
-  fault.provisions = JSON.stringify(fault.provisions)
-  faultsCollection.save(fault, (err) ->
-    return callback(err) if err
-    redisClient.del("#{deviceId}_faults", callback)
-  )
-
-
-deleteFault = (deviceId, channel, callback) ->
-  faultsCollection.remove({_id: "#{deviceId}:#{channel}"}, (err) ->
-    return callback(err) if err
-    redisClient.del("#{deviceId}_faults", callback)
-  )
-
-
-
-getDueTasks = (deviceId, timestamp, callback) ->
-  cur = tasksCollection.find({'device' : deviceId}).sort(['timestamp'])
-
-  tasks = []
-
-  cur.nextObject(f = (err, task) ->
-    return callback(err) if err
-
-    if not task?
-      return callback(null, tasks, null)
-
-    task.timestamp = +task.timestamp if task.timestamp?
-    task.expiry = +task.expiry if task.expiry?
-
-    if task.timestamp >= timestamp
-      return callback(null, tasks, +task.timestamp)
-
-    task._id = String(task._id)
-    tasks.push(task)
-
-    # For API compatibility
-    if task.name is 'download' and task.file?
-      if mongodb.ObjectID.isValid(task.file)
-        q = {_id: {'$in' : [task.file, new mongodb.ObjectID(task.file)]}}
-      else
-        q = {_id: task.file}
-
-      filesCollection.find(q).toArray((err, res) ->
-        return callback(err) if err
-        if res[0]?
-          task.fileType ?= res[0].metadata.fileType
-          task.fileName ?= res[0].filename or res[0]._id.toString()
-        cur.nextObject(f)
-      )
-    else
-      cur.nextObject(f)
-  )
-
-
-clearTasks = (deviceId, taskIds, callback) ->
-  if not taskIds?.length
-    return callback()
-
-  tasksCollection.remove({'_id' : {'$in' : (new mongodb.ObjectID(id) for id in taskIds)}}, (err) ->
-    return callback(err) if err
-    redisClient.del("#{deviceId}_tasks", callback)
-  )
-
-
-getOperations = (deviceId, callback) ->
-  operationsCollection.find({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}).toArray((err, res) ->
-    return callback(err) if err
-
-    operations = {}
-    for r in res
-      commandKey = r._id.slice(deviceId.length + 1)
-      delete r._id
-      r.timestamp = +r.timestamp
-      r.args = JSON.parse(r.args) if r.args
-      r.provisions = JSON.parse(r.provisions)
-      r.retries = JSON.parse(r.retries)
-      operations[commandKey] = r
-
-    MAX_CACHE_TTL = config.get('MAX_CACHE_TTL', deviceId)
-
-    redisClient.setex("#{deviceId}_operations", MAX_CACHE_TTL, JSON.stringify(operations), (err) ->
-      callback(err, operations)
-    )
-  )
-
-
-saveOperation = (deviceId, commandKey, operation, callback) ->
-  operation._id = "#{deviceId}:#{commandKey}"
-  operation.timestamp = new Date(operation.timestamp)
-  operation.provisions = JSON.stringify(operation.provisions)
-  operation.retries = JSON.stringify(operation.retries)
-  operation.args = JSON.stringify(operation.args)
-  operationsCollection.save(operation, (err) ->
-    return callback(err) if err
-    redisClient.del("#{deviceId}_operations", callback)
-  )
-
-
-deleteOperation = (deviceId, commandKey, callback) ->
-  operationsCollection.remove({_id: "#{deviceId}:#{commandKey}"}, (err) ->
-    return callback(err) if err
-    redisClient.del("#{deviceId}_operations", callback)
-  )
-
-
-exports.connect = connect
-exports.disconnect = disconnect
-exports.fetchDevice = fetchDevice
-exports.saveDevice = saveDevice
-exports.getDueTasksAndFaultsAndOperations = getDueTasksAndFaultsAndOperations
-exports.getFaults = getFaults
-exports.saveFault = saveFault
-exports.deleteFault = deleteFault
-exports.clearTasks = clearTasks
-exports.saveOperation = saveOperation
-exports.deleteOperation = deleteOperation

lib/db.ts

@@ -0,0 +1,822 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { MongoClient, ObjectID, Collection, Db } from "mongodb";
+import { get } from "./config";
+import { decodeTag, encodeTag, escapeRegExp } from "./common";
+import { parse } from "./common/expression-parser";
+import {
+  DeviceData,
+  Attributes,
+  SessionFault,
+  Task,
+  Operation,
+  Expression,
+} from "./types";
+import Path from "./common/path";
+
+export let tasksCollection: Collection,
+  devicesCollection: Collection,
+  presetsCollection: Collection,
+  objectsCollection: Collection,
+  provisionsCollection: Collection,
+  virtualParametersCollection: Collection,
+  faultsCollection: Collection,
+  filesCollection: Collection,
+  operationsCollection: Collection,
+  permissionsCollection: Collection,
+  usersCollection: Collection,
+  configCollection: Collection;
+
+let clientPromise: Promise<MongoClient>;
+
+const INVALID_PATH_SUFFIX = "__invalid";
+
+function compareAccessLists(list1: string[], list2: string[]): boolean {
+  if (list1.length !== list2.length) return false;
+  for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
+  return true;
+}
+
+const onConnectCallbacks: ((db: Db) => Promise<void>)[] = [];
+
+export function onConnect(callback: (db: Db) => Promise<void>): void {
+  onConnectCallbacks.push(callback);
+}
+
+export async function connect(): Promise<void> {
+  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
+    useNewUrlParser: true,
+    useUnifiedTopology: true,
+  });
+
+  const client = await clientPromise;
+  const db = client.db();
+  await Promise.all(onConnectCallbacks.map((c) => c(db)));
+}
+
+onConnect(async (db) => {
+  tasksCollection = db.collection("tasks");
+  await tasksCollection.createIndex({ device: 1, timestamp: 1 });
+
+  devicesCollection = db.collection("devices");
+  presetsCollection = db.collection("presets");
+  objectsCollection = db.collection("objects");
+  filesCollection = db.collection("fs.files");
+  provisionsCollection = db.collection("provisions");
+  virtualParametersCollection = db.collection("virtualParameters");
+  faultsCollection = db.collection("faults");
+  operationsCollection = db.collection("operations");
+  permissionsCollection = db.collection("permissions");
+  usersCollection = db.collection("users");
+  configCollection = db.collection("config");
+});
+
+export async function disconnect(): Promise<void> {
+  if (clientPromise) await (await clientPromise).close();
+}
+
+// Optimize projection by removing overlaps
+// This can modify the object
+function optimizeProjection(obj: { [path: string]: 1 }): { [path: string]: 1 } {
+  if (obj[""]) return { "": obj[""] };
+
+  const keys = Object.keys(obj).sort();
+  if (keys.length <= 1) return obj;
+
+  for (let i = 1; i < keys.length; ++i) {
+    const a = keys[i - 1];
+    const b = keys[i];
+    if (b.startsWith(a)) {
+      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
+        delete obj[b];
+        keys.splice(i--, 1);
+      }
+    }
+  }
+  return obj;
+}
+
+export async function fetchDevice(
+  id: string,
+  timestamp: number
+): Promise<[Path, number, Attributes?][]> {
+  const res: [Path, number, Attributes?][] = [
+    [
+      Path.parse("Events"),
+      timestamp,
+      { object: [timestamp, 1], writable: [timestamp, 0] },
+    ],
+    [
+      Path.parse("DeviceID"),
+      timestamp,
+      { object: [timestamp, 1], writable: [timestamp, 0] },
+    ],
+  ];
+
+  const device = await devicesCollection.findOne({ _id: id });
+  if (!device) return null;
+
+  function storeParams(
+    obj,
+    path: string,
+    pathLength: number,
+    ts: number
+  ): void {
+    if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
+    if (obj["_attributesTimestamp"])
+      obj["_attributesTimestamp"] = +obj["_attributesTimestamp"];
+
+    const attrs: Attributes = {};
+    let t = obj["_timestamp"] || 1;
+    if (ts > t) t = ts;
+
+    if (obj["_value"] != null) {
+      attrs.value = [obj["_timestamp"] || 1, [obj["_value"], obj["_type"]]];
+      if (obj["_type"] === "xsd:dateTime")
+        attrs.value[1][0] = +attrs.value[1][0];
+
+      obj["_object"] = false;
+    }
+    if (obj["_writable"] != null)
+      attrs.writable = [ts || 1, obj["_writable"] ? 1 : 0];
+
+    if (obj["_object"] != null) attrs.object = [t, obj["_object"] ? 1 : 0];
+
+    if (obj["_notification"] != null) {
+      attrs.notification = [
+        obj["_attributesTimestamp"] || 1,
+        obj["_notification"],
+      ];
+    }
+
+    if (obj["_accessList"] != null)
+      attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
+
+    try {
+      res.push([Path.parse(path), t, attrs]);
+    } catch (err) {
+      // The path parser is now more strict so we might be in a situation where
+      // the database contains invalid paths from before this change So here we
+      // encode the invalid characters.
+      const splits = path.split(".");
+      splits[splits.length - 1] =
+        encodeTag(splits[splits.length - 1]) + INVALID_PATH_SUFFIX;
+      path = splits.join(".");
+      res.push([Path.parse(path), t, attrs]);
+      return;
+    }
+
+    for (const [k, v] of Object.entries(obj)) {
+      if (!k.startsWith("_")) {
+        obj["_object"] = true;
+        storeParams(v, `${path}.${k}`, pathLength + 1, obj["_timestamp"]);
+      }
+    }
+
+    if (obj["_object"] && obj["_timestamp"])
+      res.push([Path.parse(path + ".*"), obj["_timestamp"]]);
+  }
+
+  const ts: number = device["_timestamp"] || 0;
+  if (ts) res.push([Path.parse("*"), ts]);
+
+  for (const [k, v] of Object.entries(device)) {
+    switch (k) {
+      case "_lastInform":
+        res.push([
+          Path.parse("Events.Inform"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
+        ]);
+        break;
+      case "_lastBoot":
+        res.push([
+          Path.parse("Events.1_BOOT"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
+        ]);
+        break;
+      case "_lastBootstrap":
+        res.push([
+          Path.parse("Events.0_BOOTSTRAP"),
+          +v,
+          {
+            object: [+v, 0],
+            writable: [+v, 0],
+            value: [+v, [+v, "xsd:dateTime"]],
+          },
+        ]);
+        break;
+      case "_registered":
+        // Use current timestamp for registered event attribute timestamps
+        res.push([
+          Path.parse("Events.Registered"),
+          timestamp,
+          {
+            object: [timestamp, 0],
+            writable: [timestamp, 0],
+            value: [timestamp, [+v, "xsd:dateTime"]],
+          },
+        ]);
+        break;
+      case "_id":
+        res.push([
+          Path.parse("DeviceID.ID"),
+          timestamp,
+          {
+            object: [timestamp, 0],
+            writable: [timestamp, 0],
+            value: [timestamp, [v as string, "xsd:string"]],
+          },
+        ]);
+        break;
+      case "_tags":
+        if ((v as string[]).length) {
+          res.push([
+            Path.parse("Tags"),
+            timestamp,
+            { object: [timestamp, 1], writable: [timestamp, 0] },
+          ]);
+        }
+
+        for (const t of v as string[]) {
+          res.push([
+            Path.parse("Tags." + encodeTag(t)),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 1],
+              value: [timestamp, [true, "xsd:boolean"]],
+            },
+          ]);
+        }
+        break;
+      case "_deviceId":
+        if (v["_Manufacturer"] != null) {
+          res.push([
+            Path.parse("DeviceID.Manufacturer"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_Manufacturer"], "xsd:string"]],
+            },
+          ]);
+        }
+
+        if (v["_OUI"] != null) {
+          res.push([
+            Path.parse("DeviceID.OUI"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_OUI"], "xsd:string"]],
+            },
+          ]);
+        }
+
+        if (v["_ProductClass"] != null) {
+          res.push([
+            Path.parse("DeviceID.ProductClass"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_ProductClass"], "xsd:string"]],
+            },
+          ]);
+        }
+
+        if (v["_SerialNumber"] != null) {
+          res.push([
+            Path.parse("DeviceID.SerialNumber"),
+            timestamp,
+            {
+              object: [timestamp, 0],
+              writable: [timestamp, 0],
+              value: [timestamp, [v["_SerialNumber"], "xsd:string"]],
+            },
+          ]);
+        }
+        break;
+      default:
+        if (!k.startsWith("_")) storeParams(v, k, 1, ts);
+    }
+  }
+  return res;
+}
+
+export async function saveDevice(
+  deviceId: string,
+  deviceData: DeviceData,
+  isNew: boolean,
+  sessionTimestamp: number
+): Promise<void> {
+  const update = { $set: {}, $unset: {}, $addToSet: {}, $pull: {} };
+
+  for (const diff of deviceData.timestamps.diff()) {
+    if (diff[0].wildcard !== 1 << (diff[0].length - 1)) continue;
+
+    if (
+      diff[0].segments[0] === "Events" ||
+      diff[0].segments[0] === "DeviceID" ||
+      diff[0].segments[0] === "Tags"
+    )
+      continue;
+
+    const parent = deviceData.paths.get(diff[0].slice(0, -1));
+
+    // Param timestamps may be greater than session timestamp to track revisions
+    if (diff[2] > sessionTimestamp) diff[2] = sessionTimestamp;
+
+    if (diff[2] == null && diff[1] != null) {
+      update["$unset"][
+        parent.length ? parent.toString() + "._timestamp" : "_timestamp"
+      ] = 1;
+    } else {
+      if (parent && (!parent.length || deviceData.attributes.has(parent))) {
+        update["$set"][
+          parent.length ? parent.toString() + "._timestamp" : "_timestamp"
+        ] = new Date(diff[2]);
+      }
+    }
+  }
+
+  for (const diff of deviceData.attributes.diff()) {
+    const path = diff[0];
+    const value1 = (((diff[1] || {}).value || [])[1] || [])[0];
+    const value2 = (((diff[2] || {}).value || [])[1] || [])[0];
+    const valueType1 = (((diff[1] || {}).value || [])[1] || [])[1];
+    const valueType2 = (((diff[2] || {}).value || [])[1] || [])[1];
+    const valueTimestamp1 = ((diff[1] || {}).value || [])[0];
+    const valueTimestamp2 = ((diff[2] || {}).value || [])[0];
+    const object1 = ((diff[1] || {}).object || [])[1];
+    const object2 = ((diff[2] || {}).object || [])[1];
+    const writable2 = ((diff[2] || {}).writable || [])[1];
+    const writable1 = ((diff[1] || {}).writable || [])[1];
+    const attributesTimestamp1 = ((diff[1] || {}).notification || [])[0];
+    const attributesTimestamp2 = ((diff[2] || {}).notification || [])[0];
+    const notification1 = ((diff[1] || {}).notification || [])[1];
+    const notification2 = ((diff[2] || {}).notification || [])[1];
+    const accessList1 = ((diff[1] || {}).accessList || [])[1];
+    const accessList2 = ((diff[2] || {}).accessList || [])[1];
+
+    switch (path.segments[0]) {
+      case "Events":
+        if (path.length === 2 && value2 !== value1) {
+          if (!diff[2]) {
+            switch (path.segments[1]) {
+              case "Inform":
+                update["$unset"]["_lastInform"] = 1;
+                break;
+              case "1_BOOT":
+                update["$unset"]["_lastBoot"] = 1;
+                break;
+              case "0_BOOTSTRAP":
+                update["$unset"]["_lastBootstrap"] = 1;
+                break;
+              case "Registered":
+                update["$unset"]["_registered"] = 1;
+            }
+          } else {
+            const t = new Date(diff[2].value[1][0] as number);
+            switch (path.segments[1]) {
+              case "Inform":
+                update["$set"]["_lastInform"] = t;
+                break;
+              case "1_BOOT":
+                update["$set"]["_lastBoot"] = t;
+                break;
+              case "0_BOOTSTRAP":
+                update["$set"]["_lastBootstrap"] = t;
+                break;
+              case "Registered":
+                update["$set"]["_registered"] = t;
+            }
+          }
+        }
+
+        break;
+      case "DeviceID":
+        if (value2 !== value1) {
+          const v = diff[2].value[1][0];
+          switch (path.segments[1]) {
+            case "ID":
+              update["$set"]["_id"] = v;
+              break;
+            case "Manufacturer":
+              update["$set"]["_deviceId._Manufacturer"] = v;
+              break;
+            case "OUI":
+              update["$set"]["_deviceId._OUI"] = v;
+              break;
+            case "ProductClass":
+              update["$set"]["_deviceId._ProductClass"] = v;
+              break;
+            case "SerialNumber":
+              update["$set"]["_deviceId._SerialNumber"] = v;
+          }
+        }
+        break;
+      case "Tags":
+        if (value2 !== value1) {
+          if (value2 != null) {
+            if (!update["$addToSet"]["_tags"])
+              update["$addToSet"]["_tags"] = { $each: [] };
+            update["$addToSet"]["_tags"]["$each"].push(path.segments[1]);
+          } else {
+            if (!update["$pull"]["_tags"]) {
+              update["$pull"]["_tags"] = {
+                $in: [],
+              };
+            }
+            update["$pull"]["_tags"]["$in"].push(path.segments[1]);
+          }
+        }
+
+        break;
+      default:
+        if (!diff[2]) {
+          let pathStr = path.toString();
+          // Paths with that suffix are encoded and need to be decoded
+          if (pathStr.endsWith(INVALID_PATH_SUFFIX)) {
+            const splits = pathStr.split(".");
+            splits[splits.length - 1] = decodeTag(
+              splits[splits.length - 1].slice(0, 0 - INVALID_PATH_SUFFIX.length)
+            );
+            pathStr = splits.join(".");
+          }
+          update["$unset"][pathStr] = 1;
+          continue;
+        }
+
+        for (const attrName of Object.keys(diff[2])) {
+          // Param timestamps may be greater than session timestamp to track revisions
+          if (diff[2][attrName][0] > sessionTimestamp)
+            diff[2][attrName][0] = sessionTimestamp;
+
+          if (diff[2][attrName][1] != null) {
+            switch (attrName) {
+              case "value":
+                if (value2 !== value1) {
+                  if (
+                    valueType2 === "xsd:dateTime" &&
+                    Number.isInteger(value2 as number)
+                  ) {
+                    update["$set"][path.toString() + "._value"] = new Date(
+                      value2 as number
+                    );
+                  } else {
+                    update["$set"][path.toString() + "._value"] = value2;
+                  }
+                }
+
+                if (valueType2 !== valueType1)
+                  update["$set"][path.toString() + "._type"] = valueType2;
+
+                if (valueTimestamp2 !== valueTimestamp1) {
+                  update["$set"][path.toString() + "._timestamp"] = new Date(
+                    valueTimestamp2
+                  );
+                }
+
+                break;
+              case "object":
+                if (!diff[1] || !diff[1].object || object2 !== object1) {
+                  update["$set"][
+                    path.length ? path.toString() + "._object" : "_object"
+                  ] = !!object2;
+                }
+
+                break;
+              case "writable":
+                if (!diff[1] || !diff[1].writable || writable2 !== writable1) {
+                  update["$set"][
+                    path.length ? path.toString() + "._writable" : "_writable"
+                  ] = !!writable2;
+                }
+
+                break;
+              case "notification":
+                if (
+                  !diff[1] ||
+                  !diff[1].notification ||
+                  notification2 !== notification1
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._notification"
+                      : "_notification"
+                  ] = notification2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][
+                    path.toString() + "._attributesTimestamp"
+                  ] = new Date(attributesTimestamp2);
+                }
+
+                break;
+              case "accessList":
+                if (
+                  !diff[1] ||
+                  !diff[1].accessList ||
+                  !compareAccessLists(accessList2, accessList1)
+                ) {
+                  update["$set"][
+                    path.length
+                      ? path.toString() + "._accessList"
+                      : "_accessList"
+                  ] = accessList2;
+                }
+
+                if (attributesTimestamp2 !== attributesTimestamp1) {
+                  update["$set"][
+                    path.toString() + "._attributesTimestamp"
+                  ] = new Date(attributesTimestamp2);
+                }
+            }
+          }
+        }
+
+        if (diff[1]) {
+          for (const attrName of Object.keys(diff[1])) {
+            if (
+              diff[1][attrName][1] != null &&
+              (!diff[2] || !diff[2][attrName] || diff[2][attrName][1] == null)
+            ) {
+              const p = path.length ? path.toString() + "." : "";
+              update["$unset"][`${p}_${attrName}`] = 1;
+              if (attrName === "value") {
+                update["$unset"][p + "_type"] = 1;
+                update["$unset"][p + "_timestamp"] = 1;
+              } else if (attrName === "notification") {
+                if (accessList2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
+              } else if (attrName === "accessList") {
+                if (notification2 == null)
+                  update["$unset"][`${p}_attributesTimestamp`] = 1;
+              }
+            }
+          }
+        }
+    }
+  }
+
+  update["$unset"] = optimizeProjection(update["$unset"]);
+
+  // Remove overlap possibly caused by parameters changing from objects
+  // to regular parameters or vice versa. Reason being that _timestamp
+  // represents two different things depending on whether the parameter
+  // is an object or not.
+  for (const k of Object.keys(update["$unset"]))
+    if (update["$set"][k] != null) delete update["$unset"][k];
+
+  // Remove empty keys
+  for (const [k, v] of Object.entries(update)) {
+    if (k === "$addToSet") {
+      for (const [kk, vv] of Object.entries(v))
+        if (!vv["$each"].length) delete v[kk];
+    } else if (k === "$pull") {
+      for (const [kk, vv] of Object.entries(v))
+        if (!vv["$in"].length) delete v[kk];
+    }
+    if (!Object.keys(v).length) delete update[k];
+  }
+
+  if (!Object.keys(update).length) return;
+
+  // Mongo doesn't allow $addToSet and $pull at the same time
+  let update2;
+  if (update["$addToSet"] && update["$pull"]) {
+    update2 = { $pull: update["$pull"] };
+    delete update["$pull"];
+  }
+
+  const result = await devicesCollection.updateOne({ _id: deviceId }, update, {
+    upsert: isNew,
+  });
+
+  if (result.result.n !== 1)
+    throw new Error(`Device ${deviceId} not found in database`);
+
+  if (update2) {
+    await devicesCollection.updateOne({ _id: deviceId }, update2);
+    return;
+  }
+}
+
+export async function getFaults(
+  deviceId: string
+): Promise<{ [channel: string]: SessionFault }> {
+  const res = await faultsCollection
+    .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
+    .toArray();
+
+  const faults: { [channel: string]: SessionFault } = {};
+  for (const r of res) {
+    const channel = r._id.slice(deviceId.length + 1);
+    delete r._id;
+    delete r.channel;
+    delete r.device;
+    r.timestamp = +r.timestamp;
+    r.provisions = JSON.parse(r.provisions);
+    faults[channel] = r;
+  }
+
+  return faults;
+}
+
+export async function saveFault(
+  deviceId: string,
+  channel: string,
+  fault: SessionFault
+): Promise<void> {
+  const id = `${deviceId}:${channel}`;
+  const f = Object.assign({}, fault) as Record<string, any>;
+  f["_id"] = id;
+  f["device"] = deviceId;
+  f["channel"] = channel;
+  f["timestamp"] = new Date(fault.timestamp);
+  f["provisions"] = JSON.stringify(fault.provisions);
+  await faultsCollection.replaceOne({ _id: id }, f, { upsert: true });
+}
+
+export async function deleteFault(
+  deviceId: string,
+  channel: string
+): Promise<void> {
+  await faultsCollection.deleteOne({ _id: `${deviceId}:${channel}` });
+}
+
+export async function getDueTasks(
+  deviceId: string,
+  timestamp: number
+): Promise<[Task[], number]> {
+  const cur = tasksCollection.find({ device: deviceId }).sort({ timestamp: 1 });
+  const tasks = [] as Task[];
+
+  for await (const task of cur) {
+    if (task.timestamp) task.timestamp = +task.timestamp;
+    if (task.expiry) task.expiry = +task.expiry;
+    if (task.timestamp >= timestamp) return [tasks, +task.timestamp];
+    task._id = String(task._id);
+
+    tasks.push(task);
+
+    // For API compatibility
+    if (task.name === "download" && task.file) {
+      let q;
+      if (ObjectID.isValid(task.file))
+        q = { _id: { $in: [task.file, new ObjectID(task.file)] } };
+      else q = { _id: task.file };
+
+      const res = await filesCollection.find(q).toArray();
+
+      if (res[0]) {
+        if (!task.fileType) task.fileType = res[0].metadata.fileType;
+
+        if (!task.fileName)
+          task.fileName = res[0].filename || res[0]._id.toString();
+      }
+    }
+  }
+  return [tasks, null];
+}
+
+export async function clearTasks(
+  deviceId: string,
+  taskIds: string[]
+): Promise<void> {
+  await tasksCollection.deleteMany({
+    _id: { $in: taskIds.map((id) => new ObjectID(id)) },
+  });
+}
+
+export async function getOperations(
+  deviceId: string
+): Promise<{ [commandKey: string]: Operation }> {
+  const res = await operationsCollection
+    .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
+    .toArray();
+
+  const operations: { [commandKey: string]: Operation } = {};
+  for (const r of res) {
+    const commandKey = r._id.slice(deviceId.length + 1);
+    delete r._id;
+    // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
+    if (typeof r.provisions !== "string") {
+      operations[commandKey] = r;
+      continue;
+    }
+    r.timestamp = +r.timestamp;
+    if (r.args) r.args = JSON.parse(r.args);
+    r.provisions = JSON.parse(r.provisions);
+    r.retries = JSON.parse(r.retries);
+    operations[commandKey] = r;
+  }
+  return operations;
+}
+
+export async function saveOperation(
+  deviceId: string,
+  commandKey: string,
+  operation: Operation
+): Promise<void> {
+  const id = `${deviceId}:${commandKey}`;
+  const o = Object.assign({}, operation) as Record<string, any>;
+  o["_id"] = id;
+  o["timestamp"] = new Date(operation.timestamp);
+  o["provisions"] = JSON.stringify(operation.provisions);
+  o["retries"] = JSON.stringify(operation.retries);
+  o["args"] = JSON.stringify(operation.args);
+  await operationsCollection.replaceOne({ _id: id }, o, {
+    upsert: true,
+  });
+}
+
+export async function deleteOperation(
+  deviceId: string,
+  commandKey: string
+): Promise<void> {
+  await operationsCollection.deleteOne({ _id: `${deviceId}:${commandKey}` });
+}
+
+export async function getPresets(): Promise<Record<string, any>[]> {
+  return presetsCollection.find().toArray();
+}
+
+export async function getObjects(): Promise<Record<string, any>[]> {
+  return objectsCollection.find().toArray();
+}
+
+export async function getProvisions(): Promise<Record<string, any>[]> {
+  return provisionsCollection.find().toArray();
+}
+
+export async function getVirtualParameters(): Promise<Record<string, any>[]> {
+  return virtualParametersCollection.find().toArray();
+}
+
+export function getFiles(): Promise<Record<string, any>[]> {
+  return filesCollection.find().toArray();
+}
+
+export async function getConfig(): Promise<
+  { id: string; value: Expression }[]
+> {
+  const res = await configCollection.find().toArray();
+  return res.map((c) => ({
+    id: c["_id"],
+    value: parse(c["value"]),
+  }));
+}
+
+interface Permission {
+  role: string;
+  resource: string;
+  access: number;
+  filter: string;
+  validate: string;
+}
+
+export async function getPermissions(): Promise<Permission[]> {
+  return permissionsCollection.find().toArray();
+}
+
+interface User {
+  _id: string;
+  password: string;
+  salt: string;
+  roles: string;
+}
+
+export async function getUsers(): Promise<User[]> {
+  return usersCollection.find().toArray();
+}

lib/debug.ts

@@ -0,0 +1,204 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import {
+  IncomingMessage,
+  ServerResponse,
+  ClientRequest,
+  RequestOptions,
+} from "http";
+import { Socket } from "net";
+import { appendFileSync } from "fs";
+import { stringify } from "./common/yaml";
+import * as config from "./config";
+
+const DEBUG_FILE = "" + config.get("DEBUG_FILE");
+const DEBUG_FORMAT = "" + config.get("DEBUG_FORMAT");
+
+const connectionTimestamps = new WeakMap<Socket, Date>();
+
+function getConnectionTimestamp(connection: Socket): Date {
+  let t = connectionTimestamps.get(connection);
+  if (!t) {
+    t = new Date();
+    connectionTimestamps.set(connection, t);
+  }
+  return t;
+}
+
+export function incomingHttpRequest(
+  httpRequest: IncomingMessage,
+  deviceId: string,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpRequest.socket;
+  const msg = {
+    event: "incoming HTTP request",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    localPort: con.localPort,
+    method: httpRequest.method,
+    url: httpRequest.url,
+    headers: httpRequest.headers,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpResponse(
+  httpResponse: ServerResponse,
+  deviceId: string,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpResponse.socket;
+  const msg = {
+    event: "outgoing HTTP response",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    statusCode: httpResponse.statusCode,
+    headers: httpResponse.getHeaders(),
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpRequest(
+  httpRequest: ClientRequest,
+  deviceId: string,
+  options: RequestOptions,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpRequest.socket;
+  const msg = {
+    event: "outgoing HTTP request",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(con),
+    remotePort: options.port,
+    method: options.method || "GET",
+    url: options.path,
+    headers: httpRequest.getHeaders(),
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingHttpRequestError(
+  httpRequest: ClientRequest,
+  deviceId: string,
+  options: RequestOptions,
+  err: Error
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing HTTP request",
+    timestamp: now,
+    remoteAddress: options.hostname,
+    deviceId: deviceId,
+    connection: null,
+    remotePort: options.port,
+    method: options.method,
+    url: options.path,
+    headers: httpRequest.getHeaders(),
+    error: err.message,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function incomingHttpResponse(
+  httpResponse: IncomingMessage,
+  deviceId: string,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const con = httpResponse.socket;
+  const msg = {
+    event: "incoming HTTP response",
+    timestamp: now,
+    remoteAddress: con.remoteAddress,
+    deviceId: deviceId,
+    connection: getConnectionTimestamp(httpResponse.socket),
+    statusCode: httpResponse.statusCode,
+    headers: httpResponse.headers,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}
+
+export function outgoingUdpMessage(
+  remoteAddress: string,
+  deviceId: string,
+  remotePort: number,
+  body: string
+): void {
+  if (!DEBUG_FILE) return;
+  const now = new Date();
+  const msg = {
+    event: "outgoing UDP message",
+    timestamp: now,
+    remoteAddress: remoteAddress,
+    deviceId: deviceId,
+    remotePort: remotePort,
+    body: body,
+  };
+
+  if (DEBUG_FORMAT === "yaml")
+    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
+  else if (DEBUG_FORMAT === "json")
+    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
+  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
+}

lib/default-provisions.js

@@ -1,116 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const config = require("./config");
-const common = require("./common");
-const device = require("./device");
-
-const MAX_DEPTH = config.get("MAX_DEPTH");
-
-
-function refresh(sessionContext, provision, declarations, startRevision, endRevision) {
-  let path = common.parsePath(provision[1]).slice();
-  let l = path.length;
-  path.length = MAX_DEPTH;
-  path.fill("*", l);
-  let every = 1000 * (provision[2] || 1);
-  let offset = Math.abs(common.djb2(sessionContext.deviceId)) % 86400000;
-  let t = Math.trunc((sessionContext.timestamp + offset) / every) * every - offset;
-
-  for (let i = l; i < path.length; ++ i)
-    declarations.push([path.slice(0, i), t, {object: 1, writable: 1, value: t}]);
-
-  return true;
-}
-
-
-function value(sessionContext, provision, declarations, startRevision, endRevision) {
-  let path = common.parsePath(provision[1]);
-  declarations.push([path, 1, {value: 1}, null, {value: [provision[2]]}]);
-
-  return true;
-}
-
-
-function tag(sessionContext, provision, declarations, startRevision, endRevision) {
-  let path = ["Tags", provision[1]];
-  declarations.push([path, 1, {value: 1}, null, {value: [provision[2]]}]);
-
-  return true;
-}
-
-
-function reboot(sessionContext, provision, declarations, startRevision, endRevision) {
-  declarations.push([["Reboot"], 1, {value: 1}, null, {value: [sessionContext.timestamp]}]);
-
-  return true;
-}
-
-
-function reset(sessionContext, provision, declarations, startRevision, endRevision) {
-  declarations.push([["FactoryReset"], 1, {value: 1}, null, {value: [sessionContext.timestamp]}]);
-
-  return true;
-}
-
-
-function download(sessionContext, provision, declarations, startRevision, endRevision) {
-  let alias = ["FileType", provision[1] || "", "FileName", provision[2] || "",
-    "TargetFileName", provision[3] || ""];
-
-  declarations.push([["Downloads", alias], 1, {}, 1]);
-  declarations.push([["Downloads", alias, "Download"], 1, {value: 1}, null,
-    {value: [sessionContext.timestamp]}]);
-
-  return true;
-}
-
-
-function instances(sessionContext, provision, declarations, startRevision, endRevision) {
-  let count = Number(provision[2]);
-
-  if (Number.isNaN(count))
-    return true;
-
-  let path = common.parsePath(provision[1]);
-
-  if (provision[2][0] === "+" || provision[2][0] === "-") {
-    declarations.push([path, 1]);
-
-    if (endRevision === startRevision)
-      return false;
-
-    let unpacked = device.unpack(sessionContext.deviceData, path, startRevision + 1);
-    count = Math.max(0, unpacked.length + count);
-  }
-
-  declarations.push([path, 1, null, count]);
-
-  return true;
-}
-
-
-exports.refresh = refresh;
-exports.value = value;
-exports.tag = tag;
-exports.reboot = reboot;
-exports.reset = reset;
-exports.download = download;
-exports.instances = instances;

lib/default-provisions.ts

@@ -0,0 +1,278 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./common/path";
+import * as config from "./config";
+import * as device from "./device";
+import * as scheduling from "./scheduling";
+import { SessionContext, Declaration } from "./types";
+
+const MAX_DEPTH = +config.get("MAX_DEPTH");
+
+export function refresh(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (
+    (provision.length !== 2 || typeof provision[1] !== "string") &&
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number") &&
+    (provision.length < 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "number" ||
+      typeof provision[3] !== "boolean")
+  )
+    throw new Error("Invalid arguments");
+
+  const every = 1000 * ((provision[2] as number) || 1);
+  const offset = scheduling.variance(sessionContext.deviceId, every);
+  const t = scheduling.interval(sessionContext.timestamp, every, offset);
+
+  let attrGet;
+  let refreshChildren;
+  if (provision[3] == null) {
+    refreshChildren = true;
+    attrGet = { object: 1, writable: 1, value: t };
+  } else {
+    attrGet = {};
+    refreshChildren = !!provision[3];
+    for (const a of provision.slice(4)) attrGet[a as string] = t;
+  }
+
+  let path = Path.parse(provision[1]);
+  let l = path.length;
+  if (refreshChildren) {
+    const segments = path.segments.slice();
+    l = segments.length;
+    segments.length = MAX_DEPTH;
+    segments.fill("*", l);
+    path = Path.parse(segments.join("."));
+  }
+
+  for (let i = l; i <= path.length; ++i) {
+    declarations.push({
+      path: path.slice(0, i),
+      pathGet: t,
+      pathSet: null,
+      attrGet: attrGet,
+      attrSet: null,
+      defer: true,
+    });
+  }
+
+  return true;
+}
+
+export function value(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (
+    provision.length < 3 ||
+    provision.length > 4 ||
+    typeof provision[1] !== "string"
+  )
+    throw new Error("Invalid arguments");
+
+  let attr: string, val: any;
+
+  if (provision.length === 3) {
+    attr = "value";
+    val = provision[2];
+  } else {
+    attr = (provision[2] as string) || "";
+    val = provision[3];
+  }
+
+  if (attr === "accessList") {
+    val = (val || "")
+      .split(",")
+      .map((s) => s.trim())
+      .filter((s) => !!s);
+  } else if (attr === "value") {
+    val = [val];
+  }
+
+  declarations.push({
+    path: Path.parse(provision[1]),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { [attr]: 1 },
+    attrSet: { [attr]: val },
+    defer: true,
+  });
+
+  return true;
+}
+
+export function tag(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (
+    provision.length !== 3 ||
+    typeof provision[1] !== "string" ||
+    typeof provision[2] !== "boolean"
+  )
+    throw new Error("Invalid arguments");
+
+  declarations.push({
+    path: Path.parse(`Tags.${provision[1]}`),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [provision[2]] },
+    defer: true,
+  });
+
+  return true;
+}
+
+export function reboot(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
+  declarations.push({
+    path: Path.parse("Reboot"),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true,
+  });
+
+  return true;
+}
+
+export function reset(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (provision.length !== 1) throw new Error("Invalid arguments");
+
+  declarations.push({
+    path: Path.parse("FactoryReset"),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true,
+  });
+
+  return true;
+}
+
+export function download(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[]
+): boolean {
+  if (
+    (provision.length !== 3 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string") &&
+    (provision.length !== 4 ||
+      typeof provision[1] !== "string" ||
+      typeof provision[2] !== "string" ||
+      typeof provision[3] !== "string")
+  )
+    throw new Error("Invalid arguments");
+
+  const alias = [
+    `FileType:${JSON.stringify(provision[1] || "")}`,
+    `FileName:${JSON.stringify(provision[2] || "")}`,
+    `TargetFileName:${JSON.stringify(provision[3] || "")}`,
+  ].join(",");
+
+  declarations.push({
+    path: Path.parse(`Downloads.[${alias}]`),
+    pathGet: 1,
+    pathSet: 1,
+    attrGet: null,
+    attrSet: null,
+    defer: true,
+  });
+
+  declarations.push({
+    path: Path.parse(`Downloads.[${alias}].Download`),
+    pathGet: 1,
+    pathSet: null,
+    attrGet: { value: 1 },
+    attrSet: { value: [sessionContext.timestamp] },
+    defer: true,
+  });
+
+  return true;
+}
+
+export function instances(
+  sessionContext: SessionContext,
+  provision: (string | number | boolean)[],
+  declarations: Declaration[],
+  startRevision: number,
+  endRevision: number
+): boolean {
+  if (provision.length !== 3 || typeof provision[1] !== "string")
+    throw new Error("Invalid arguments");
+
+  let count = Number(provision[2]);
+
+  if (Number.isNaN(count)) throw new Error("Invalid arguments");
+
+  const path = Path.parse(provision[1]);
+
+  if (provision[2][0] === "+" || provision[2][0] === "-") {
+    declarations.push({
+      path: path,
+      pathGet: 1,
+      pathSet: null,
+      attrGet: null,
+      attrSet: null,
+      defer: true,
+    });
+
+    if (endRevision === startRevision) return false;
+
+    const unpacked = device.unpack(
+      sessionContext.deviceData,
+      path,
+      startRevision + 1
+    );
+    count = Math.max(0, unpacked.length + count);
+  }
+
+  declarations.push({
+    path: path,
+    pathGet: 1,
+    pathSet: count,
+    attrGet: null,
+    attrSet: null,
+    defer: true,
+  });
+
+  return true;
+}

lib/device.js

@@ -1,380 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const common = require('./common');
-
-
-const CHANGE_FLAGS = {
-  object: 4,
-  writable: 16,
-  value: 64
-};
-
-
-function sanitizeParameterValue(parameterValue) {
-  if (parameterValue[0] != null) {
-    switch (parameterValue[1]) {
-      case 'xsd:boolean':
-        if (typeof parameterValue[0] !== 'boolean') {
-          parameterValue[0] = !!JSON.parse(parameterValue[0]);
-        }
-        break;
-      case 'xsd:string':
-        if (typeof parameterValue[0] !== 'string') {
-          parameterValue[0] = '' + parameterValue[0];
-        }
-        break;
-      case 'xsd:int':
-      case 'xsd:unsignedInt':
-        if (typeof parameterValue[0] !== 'number') {
-          parameterValue[0] = +parameterValue[0];
-        }
-        break;
-      case 'xsd:dateTime':
-        if (typeof parameterValue[0] !== 'number') {
-          if (parameterValue[0].getTime() != null)
-            parameterValue[0] = parameterValue[0].getTime();
-          else if (isNaN(parameterValue[0]))
-            parameterValue[0] = Date.parse(parameterValue[0]);
-          else
-            parameterValue[0] = +parameterValue[0];
-        }
-        break;
-      default:
-        if (parameterValue[1] != null)
-          throw new Error(`Parameter value type "${parameterValue[1]}" not recognized.`);
-
-        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
-    }
-  }
-  return parameterValue;
-}
-
-
-function getAliasDeclarations(path, timestamp) {
-  var decs = [];
-
-  function recursive(pattern, prefix) {
-    var pattern = prefix.concat(pattern);
-
-    for (let i = prefix.length; i < pattern.length; ++ i) {
-      if (Array.isArray(pattern[i])) {
-        let pat = pattern[i];
-        pattern[i] = '*';
-        for (let j = 0; j < pat.length; j += 2)
-          recursive(pat[j], pattern.slice(0, i + 1));
-      }
-    }
-
-    if (prefix.length)
-      decs.push([pattern, timestamp, {value: timestamp}]);
-    else
-      decs.push([pattern, timestamp]);
-  }
-
-  recursive(path, []);
-
-  return decs;
-}
-
-
-function unpack(deviceData, path, revision) {
-  let allMatches = [];
-  if (!path.alias) {
-    for (let p of deviceData.paths.find(path, false, true))
-      if (deviceData.attributes.has(p, revision))
-        allMatches.push(p);
-  }
-  else {
-    let wildcardPath = path.slice();
-    wildcardPath.wildcard = path.wildcard;
-    for (let i = 0; i < wildcardPath.length; ++ i)
-      if (Array.isArray(wildcardPath[i])) {
-        wildcardPath[i] = '*';
-        wildcardPath.wildcard |= 1 << i;
-      }
-
-    for (let p of deviceData.paths.find(wildcardPath, false, true))
-      if (deviceData.attributes.has(p, revision))
-        allMatches.push(p);
-
-    for (let i = path.length - 1; i >= 0; -- i) {
-      if (Array.isArray(path[i])) {
-        for (let j = 0; j < path[i].length; j += 2) {
-          let p = wildcardPath.slice(0, i + 1).concat(path[i][j]);
-          p.alias = (wildcardPath.alias & ((1 << (i + 1)) - 1)) + path[i][j].alias << (i + 1);
-          p.wildcard = (wildcardPath.wildcard & ((1 << (i + 1)) - 1)) + path[i][j].wildcard << (i + 1);
-          let unpacked = unpack(deviceData, p, revision);
-          let filtered = [];
-          for (let up of unpacked) {
-            let attributes = deviceData.attributes.get(up, revision);
-            if (attributes && attributes.value && attributes.value[1] &&
-                sanitizeParameterValue([path[i][j + 1], attributes.value[1][1]])[0] === attributes.value[1][0]) {
-              for (let m = 0; m < allMatches.length; ++ m) {
-                let k;
-                for (k = i; k >= 0; -- k)
-                  if (allMatches[m][k] != up[k])
-                    break
-
-                if (k < 0) {
-                  filtered.push(allMatches[m]);
-                  allMatches[m] = [];
-                }
-              }
-            }
-          }
-          allMatches = filtered;
-        }
-      }
-    }
-  }
-
-  allMatches.sort(function(a, b) {
-    for (let i = 0; i < a.length; ++ i) {
-      if (a === b)
-        continue;
-
-      // Use numeric sorting for numbers
-      let ia = parseInt(a), ib = parseInt(b);
-      if (ia.toString() === a && ib.toString() === b) {
-        a = ia;
-        b = ib;
-      }
-
-      if (a < b)
-        return -1;
-      else
-        return 1;
-    }
-    return 0;
-  });
-
-  return allMatches;
-}
-
-
-function clear(deviceData, path, timestamp, attributes) {
-  for (let p of deviceData.paths.find(path, true, true, timestamp ? 99 : path.length)) {
-    for (let k in deviceData.trackers.get(p))
-      deviceData.changes.add(k);
-
-    let t = deviceData.timestamps.get(p);
-    if (!t)
-      continue;
-
-    if (!(timestamp <= t)) {
-      deviceData.timestamps.delete(p);
-      deviceData.attributes.delete(p);
-    } else if (t && attributes) {
-      let currentAttributes = deviceData.attributes.get(p);
-      if (currentAttributes) {
-        let newAttrs;
-        for (let attrName in attributes) {
-          if (attrName in currentAttributes &&
-              attributes[attrName][0] > currentAttributes[attrName][0]) {
-            if (!newAttrs)
-              newAttrs = JSON.parse(JSON.stringify(currentAttributes));
-            delete newAttrs[attrName];
-          }
-        }
-
-        if (newAttrs)
-          deviceData.attributes.set(p, newAttrs);
-      }
-    }
-  }
-}
-
-
-function compareEquality(a, b) {
-  var t = typeof a;
-  if (a === null ||
-      a === undefined ||
-      t === 'number' ||
-      t === 'boolean' ||
-      t === 'string' ||
-      t === 'symbol')
-    return a === b;
-
-  return JSON.stringify(a) === JSON.stringify(b);
-}
-
-
-function set(deviceData, path, timestamp, attributes, toClear) {
-  path = deviceData.paths.add(path);
-
-  let currentTimestamp = deviceData.timestamps.get(path);
-
-  let currentAttributes;
-
-  if (path.wildcard)
-    attributes = undefined;
-  else
-    currentAttributes = deviceData.attributes.get(path);
-
-  if (attributes) {
-    if (attributes.value && attributes.value[1]
-        && attributes.value[0] >= (attributes.object ? attributes.object[0] : 0))
-      attributes.object = [attributes.value[0], 0];
-
-    if (attributes.object && attributes.object[1]
-      && attributes.object[0] >= (attributes.value ? attributes.value[0] : 0))
-      attributes.value = [attributes.object[0]];
-
-    if (currentAttributes) {
-      let changeFlags = 0;
-
-      for (let attrName in attributes) {
-        timestamp = Math.max(timestamp, attributes[attrName][0]);
-        if (attrName in currentAttributes) {
-          if (attributes[attrName][0] > currentAttributes[attrName][0]) {
-            changeFlags |= CHANGE_FLAGS[attrName];
-            if (compareEquality(attributes[attrName][1], currentAttributes[attrName][1]))
-              changeFlags |= CHANGE_FLAGS[attrName] << 1;
-          }
-          else
-            delete attributes[attrName];
-        }
-        else {
-          changeFlags |= CHANGE_FLAGS[attrName];
-          changeFlags |= CHANGE_FLAGS[attrName] << 1;
-        }
-      }
-
-      if (Object.keys(attributes).length) {
-        if (attributes.object && currentAttributes.object &&
-            attributes.object[1] !== currentAttributes.object[1]) {
-          toClear = toClear || [];
-          toClear.push([path.concat('*'), attributes.object[0]]);
-        }
-        for (let attrName in currentAttributes)
-          if (!(attrName in attributes))
-            attributes[attrName] = currentAttributes[attrName];
-
-        deviceData.attributes.set(path, attributes);
-      }
-
-      if (timestamp > currentTimestamp) {
-        changeFlags |= 1;
-        deviceData.timestamps.set(path, timestamp);
-
-        if (path.length > 1)
-          toClear = set(deviceData, path.slice(0, path.length - 1), timestamp,
-            {object: [timestamp, 1]}, toClear);
-      }
-
-      if (changeFlags)
-        for (let sup of deviceData.paths.find(path, true, false)) {
-          let c = deviceData.trackers.get(sup);
-          for (let k in c)
-            if (c[k] & changeFlags) {
-              deviceData.changes.add(k);
-            }
-        }
-    }
-    else if (!(timestamp <= currentTimestamp)) {
-      deviceData.timestamps.set(path, timestamp);
-      deviceData.attributes.set(path, attributes);
-
-      if (path.length > 1)
-        toClear = set(deviceData, path.slice(0, path.length - 1), timestamp,
-          {object: [timestamp, 1]}, toClear);
-
-      toClear = toClear || [];
-      toClear.push([path, timestamp]);
-    }
-  }
-  else if (!(timestamp <= currentTimestamp)) {
-    deviceData.timestamps.set(path, timestamp);
-
-    if (currentAttributes) {
-      deviceData.attributes.delete(path);
-      toClear = toClear || [];
-      toClear.push([path, timestamp]);
-    }
-
-    if (path.wildcard)
-      for (let sub of deviceData.paths.find(path, false, true))
-        if (timestamp > deviceData.timestamps.get(sub)) {
-          deviceData.timestamps.delete(sub);
-          if (deviceData.attributes.delete(sub)) {
-            toClear = toClear || [];
-            toClear.push([sub, timestamp]);
-          }
-        }
-  }
-
-  return toClear;
-}
-
-
-function track(deviceData, path, marker, timestamp, attributeTimestamps, exist, attributeValues) {
-  path = deviceData.paths.add(path);
-  let f = 0;
-
-  if (timestamp)
-    f |= 1;
-
-  if (exist)
-    f |= 2;
-
-  if (attributeTimestamps)
-    for (let attrName of attributeTimestamps)
-      f |= CHANGE_FLAGS[attrName];
-
-  if (attributeValues)
-    for (let attrName of attributeValues)
-      f |= CHANGE_FLAGS[attrName] << 1;
-
-  let cur = deviceData.trackers.get(path);
-  if (!cur) {
-    cur = {};
-    deviceData.trackers.set(path, cur);
-  }
-
-  cur[marker] |= f;
-}
-
-
-function clearTrackers(deviceData, tracker) {
-  if (Array.isArray(tracker)) {
-    deviceData.trackers.forEach(function(v, k) {
-      for (let t of tracker)
-        delete v[t];
-    });
-    for (let t of tracker)
-      deviceData.changes.delete(t);
-  }
-  else {
-    deviceData.trackers.forEach(function(v, k) {
-      delete v[tracker];
-    });
-    deviceData.changes.delete(tracker);
-  }
-}
-
-
-exports.sanitizeParameterValue = sanitizeParameterValue;
-exports.getAliasDeclarations = getAliasDeclarations;
-exports.unpack = unpack;
-exports.set = set;
-exports.clear = clear;
-exports.track = track;
-exports.clearTrackers = clearTrackers;

lib/device.ts

@@ -0,0 +1,416 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./common/path";
+import {
+  DeviceData,
+  Declaration,
+  Clear,
+  AttributeTimestamps,
+  Attributes,
+} from "./types";
+
+const CHANGE_FLAGS = {
+  object: 2,
+  writable: 4,
+  value: 8,
+  notification: 16,
+  accessList: 32,
+};
+
+function parseBool(v): boolean {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else return null;
+}
+
+export function sanitizeParameterValue(
+  parameterValue: [string | number | boolean, string]
+): [string | number | boolean, string] {
+  if (parameterValue[0] != null) {
+    switch (parameterValue[1]) {
+      case "xsd:boolean":
+        if (typeof parameterValue[0] !== "boolean") {
+          const b = parseBool(parameterValue[0]);
+          if (b == null) parameterValue[0] = "" + parameterValue[0];
+          else parameterValue[0] = b;
+        }
+        break;
+      case "xsd:string":
+      case "xsd:base64":
+      case "xsd:hexBinary":
+        if (typeof parameterValue[0] !== "string")
+          parameterValue[0] = "" + parameterValue[0];
+
+        break;
+      case "xsd:int":
+      case "xsd:unsignedInt":
+        if (typeof parameterValue[0] !== "number") {
+          const i = parseInt(parameterValue[0] as string);
+          if (isNaN(i)) parameterValue[0] = "" + parameterValue[0];
+          else parameterValue[0] = i;
+        }
+        break;
+      case "xsd:dateTime":
+        if (typeof parameterValue[0] !== "number") {
+          // Don't use parseInt because it reads date string as a number
+          let i = +parameterValue[0];
+          if (isNaN(i)) {
+            i = Date.parse(parameterValue[0] as string);
+            if (isNaN(i)) parameterValue[0] = "" + parameterValue[0];
+            else parameterValue[0] = i;
+          } else {
+            parameterValue[0] = i;
+          }
+        }
+        break;
+      default:
+        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
+        break;
+    }
+  }
+
+  return parameterValue;
+}
+
+export function getAliasDeclarations(
+  path: Path,
+  timestamp: number,
+  attrGet = null
+): Declaration[] {
+  const stripped = path.stripAlias();
+  let decs: Declaration[] = [
+    {
+      path: stripped,
+      pathGet: timestamp,
+      pathSet: null,
+      attrGet: attrGet,
+      attrSet: null,
+      defer: true,
+    },
+  ];
+
+  if (path.alias) {
+    for (const [i, alias] of path.segments.entries()) {
+      if (Array.isArray(alias)) {
+        const parent = stripped.slice(0, i + 1);
+        for (const [p] of alias as [Path, string][]) {
+          decs = decs.concat(
+            getAliasDeclarations(parent.concat(p), timestamp, {
+              value: timestamp,
+            })
+          );
+        }
+      }
+    }
+  }
+
+  return decs;
+}
+
+export function unpack(
+  deviceData: DeviceData,
+  path: Path,
+  revision?: number
+): Path[] {
+  let allMatches = [] as Path[];
+  if (!path.alias) {
+    for (const p of deviceData.paths.find(path, false, true))
+      if (deviceData.attributes.has(p, revision)) allMatches.push(p);
+  } else {
+    const wildcardPath = path.stripAlias();
+
+    for (const p of deviceData.paths.find(wildcardPath, false, true))
+      if (deviceData.attributes.has(p, revision)) allMatches.push(p);
+
+    for (let i = path.length - 1; i >= 0; --i) {
+      if (path.alias & (1 << i)) {
+        for (const [param, val] of path.segments[i] as [Path, string][]) {
+          const p = wildcardPath.slice(0, i + 1).concat(param);
+          const unpacked = unpack(deviceData, p, revision);
+          const filtered: Path[] = [];
+          for (const up of unpacked) {
+            const attributes = deviceData.attributes.get(up, revision);
+            if (
+              attributes &&
+              attributes.value &&
+              attributes.value[1] &&
+              sanitizeParameterValue([val, attributes.value[1][1]])[0] ===
+                attributes.value[1][0]
+            ) {
+              for (let m = 0; m < allMatches.length; ++m) {
+                let k;
+                const match = allMatches[m];
+                if (!match) continue;
+                for (k = i; k >= 0; --k)
+                  if (match.segments[k] !== up.segments[k]) break;
+
+                if (k < 0) {
+                  filtered.push(match);
+                  allMatches[m] = null;
+                }
+              }
+            }
+          }
+          allMatches = filtered;
+        }
+      }
+    }
+  }
+
+  allMatches.sort((p1, p2) => {
+    for (let i = 0; i < p1.length; ++i) {
+      const a = p1.segments[i] as string;
+      const b = p2.segments[i] as string;
+      if (a !== b) {
+        // Use numeric sorting for numbers
+        const ia = parseInt(a);
+        const ib = parseInt(b);
+
+        if (ia === +a && ib === +b) return ia - ib;
+        else if (a < b) return -1;
+        else return 1;
+      }
+    }
+    return 0;
+  });
+
+  return allMatches;
+}
+
+export function clear(
+  deviceData: DeviceData,
+  path: Path,
+  timestamp: number,
+  attributes: AttributeTimestamps,
+  changeFlags = 0
+): void {
+  const changeTrackers = {};
+
+  timestamp = timestamp || 0;
+
+  let descendantsTimestamp = timestamp;
+  if (attributes && attributes.object) {
+    if (attributes.object > descendantsTimestamp)
+      descendantsTimestamp = attributes.object;
+    if (!(attributes.object <= attributes.value))
+      attributes.value = attributes.object;
+  }
+
+  for (const p of deviceData.paths.find(
+    path,
+    true,
+    true,
+    descendantsTimestamp ? 99 : path.length
+  )) {
+    const tracker = deviceData.trackers.get(p);
+    for (const k in tracker) changeTrackers[k] |= tracker[k];
+
+    const currentTimestamp = deviceData.timestamps.get(p);
+    if (currentTimestamp === undefined) continue;
+
+    if (
+      timestamp > currentTimestamp ||
+      (descendantsTimestamp > currentTimestamp && p.length > path.length)
+    ) {
+      deviceData.timestamps.delete(p);
+      deviceData.attributes.delete(p);
+      changeFlags |= 1;
+    } else if (attributes && p.length === path.length) {
+      const currentAttributes = deviceData.attributes.get(p);
+      if (currentAttributes) {
+        let newAttrs;
+        for (const attrName in attributes) {
+          if (
+            attrName in currentAttributes &&
+            attributes[attrName] > currentAttributes[attrName][0]
+          ) {
+            changeFlags |= CHANGE_FLAGS[attrName];
+            if (!newAttrs) {
+              newAttrs = Object.assign({}, currentAttributes);
+              deviceData.attributes.set(p, newAttrs);
+            }
+            delete newAttrs[attrName];
+          }
+        }
+      }
+    }
+  }
+
+  // Note: For performance, we're merging all changes together rather than
+  // mark changes based the exact parameters affected.
+  for (const k in changeTrackers)
+    if (changeTrackers[k] & changeFlags) deviceData.changes.add(k);
+}
+
+function compareEquality(a, b): boolean {
+  const t = typeof a;
+  if (
+    a === null ||
+    a === undefined ||
+    t === "number" ||
+    t === "boolean" ||
+    t === "string" ||
+    t === "symbol"
+  )
+    return a === b;
+
+  return JSON.stringify(a) === JSON.stringify(b);
+}
+
+export function set(
+  deviceData: DeviceData,
+  path: Path,
+  timestamp: number,
+  attributes: Attributes,
+  toClear?: Clear[]
+): Clear[] {
+  path = deviceData.paths.add(path);
+
+  const currentTimestamp = deviceData.timestamps.get(path);
+
+  let currentAttributes;
+
+  if (path.wildcard) attributes = undefined;
+  else if (currentTimestamp)
+    currentAttributes = deviceData.attributes.get(path);
+
+  let changeFlags = 0;
+
+  if (attributes) {
+    if (
+      attributes.value &&
+      attributes.value[1] &&
+      attributes.value[0] >= (attributes.object ? attributes.object[0] : 0)
+    )
+      attributes.object = [attributes.value[0], 0];
+
+    if (
+      attributes.object &&
+      attributes.object[1] &&
+      attributes.object[0] >= (attributes.value ? attributes.value[0] : 0)
+    )
+      attributes.value = [attributes.object[0], null];
+
+    const newAttributes = Object.assign({}, currentAttributes, attributes);
+
+    if (currentAttributes) {
+      for (const attrName in attributes) {
+        timestamp = Math.max(timestamp, attributes[attrName][0]);
+        if (!(attrName in currentAttributes))
+          changeFlags |= CHANGE_FLAGS[attrName];
+        else if (attributes[attrName][0] <= currentAttributes[attrName][0])
+          newAttributes[attrName] = currentAttributes[attrName];
+        else if (
+          !compareEquality(
+            attributes[attrName][1],
+            currentAttributes[attrName][1]
+          )
+        )
+          changeFlags |= CHANGE_FLAGS[attrName];
+      }
+    } else {
+      changeFlags |= 1;
+    }
+
+    deviceData.attributes.set(path, newAttributes);
+
+    if (!(timestamp <= currentTimestamp)) {
+      deviceData.timestamps.set(path, timestamp);
+      if (path.length > 1) {
+        toClear = set(
+          deviceData,
+          path.slice(0, path.length - 1),
+          timestamp,
+          { object: [timestamp, 1] },
+          toClear
+        );
+      }
+    }
+  } else if (!(timestamp <= currentTimestamp)) {
+    deviceData.timestamps.set(path, timestamp);
+
+    if (currentAttributes) {
+      deviceData.attributes.delete(path);
+      changeFlags |= 1;
+    } else if (path.wildcard) {
+      for (const p of deviceData.paths.find(path, false, true, path.length)) {
+        if (timestamp > deviceData.timestamps.get(p)) {
+          toClear = toClear || [];
+          toClear.push([p, timestamp]);
+        }
+      }
+    }
+  }
+
+  if (changeFlags) {
+    if (changeFlags & 1) {
+      toClear = toClear || [];
+      toClear.push([path, timestamp, null, changeFlags]);
+    } else if (changeFlags & CHANGE_FLAGS.object) {
+      toClear = toClear || [];
+      toClear.push([path, 0, { object: attributes.object[0] }, changeFlags]);
+    } else {
+      for (const p of deviceData.paths.find(path, true, false, path.length)) {
+        const tracker = deviceData.trackers.get(p);
+        for (const k in tracker)
+          if (tracker[k] & changeFlags) deviceData.changes.add(k);
+      }
+    }
+  }
+
+  return toClear;
+}
+
+export function track(
+  deviceData: DeviceData,
+  path: Path,
+  marker: string,
+  attributes?: string[]
+): void {
+  path = deviceData.paths.add(path);
+  let f = 1;
+
+  if (attributes)
+    for (const attrName of attributes) f |= CHANGE_FLAGS[attrName];
+
+  let cur = deviceData.trackers.get(path);
+  if (!cur) {
+    cur = {};
+    deviceData.trackers.set(path, cur);
+  }
+
+  cur[marker] |= f;
+}
+
+export function clearTrackers(
+  deviceData: DeviceData,
+  tracker: string | string[]
+): void {
+  if (Array.isArray(tracker)) {
+    for (const v of deviceData.trackers.values())
+      for (const t of tracker) delete v[t];
+    for (const t of tracker) deviceData.changes.delete(t);
+  } else {
+    for (const v of deviceData.trackers.values()) delete v[tracker];
+    deviceData.changes.delete(tracker);
+  }
+}

lib/extension-wrapper.js

@@ -1,64 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const jobs = new Set();
-const fileName = process.argv[2];
-let script;
-
-function errorToFault(err) {
-  if (!err)
-    return null;
-
-  return {
-    code: `ext.${err.name}`,
-    message: err.message,
-    detail: {
-      name: err.name,
-      message: err.message,
-      stack: err.stack
-    }
-  }
-}
-
-process.on("uncaughtException", function(err) {
-  let fault = errorToFault(err);
-  jobs.forEach(function(jobId) {
-    process.send([jobId, fault]);
-  });
-  jobs.clear();
-  process.disconnect();
-});
-
-process.on("message", function(message) {
-  jobs.add(message[0]);
-
-  if (!script) {
-    let cwd = process.env["GENIEACS_CONFIG_DIR"] + "/ext";
-    process.chdir(cwd);
-    script = require(`${cwd}/${fileName}`);
-  }
-
-  script[message[1][0]](message[1].slice(1), function(err, res) {
-    if (!jobs.delete(message[0]))
-      return;
-
-    process.send([message[0], errorToFault(err), res]);
-  });
-});

lib/extensions.js

@@ -1,100 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const childProcess = require("child_process");
-const crypto = require ("crypto");
-
-const config = require("./config");
-
-const TIMEOUT = config.get('EXT_TIMEOUT');
-
-const processes = {};
-const jobs = {};
-
-function messageHandler(message) {
-  let func = jobs[message[0]];
-  if (func) {
-    delete jobs[message[0]];
-    func(null, message[1], message[2]);
-  }
-}
-
-
-function run(args, callback) {
-  let scriptName = args[0];
-
-  if (!processes[scriptName]) {
-    let p = childProcess.fork('lib/extension-wrapper', [scriptName]);
-
-    p.on('error', function() {
-      kill(processes[scriptName]);
-      delete processes[scriptName];
-    });
-
-    p.on('disconnect', function() {
-      kill(processes[scriptName]);
-      delete processes[scriptName];
-    });
-
-    p.on('message', messageHandler);
-    processes[scriptName] = p;
-  }
-
-  let id = crypto.randomBytes(8).toString('hex');
-  jobs[id] = callback;
-  setTimeout(function() {
-    if (id in jobs) {
-      delete jobs[id];
-      return callback(null, {code: "timeout", message: "Extension timed out"});
-    }
-  }, TIMEOUT);
-
-  return processes[scriptName].send([id, args.slice(1)]);
-};
-
-
-function kill(process) {
-  if (process.signalCode !== null || process.exitCode !== null)
-    return;
-
-  let timeToKill = Date.now() + 5000;
-
-  process.kill();
-
-  let t = setInterval(function() {
-    if (process.signalCode !== null || process.exitCode !== null) {
-      clearInterval(t);
-    }
-    else if (Date.now() > timeToKill) {
-      process.kill('SIGKILL');
-      clearInterval(t);
-    }
-  }, 100);
-}
-
-
-function killAll() {
-  for (let n in processes)
-    kill(processes[n]);
-}
-
-
-exports.run = run;
-exports.killAll = killAll;

lib/extensions.ts

@@ -0,0 +1,128 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { spawn, ChildProcess } from "child_process";
+import * as crypto from "crypto";
+import * as config from "./config";
+import { Fault } from "./types";
+import { ROOT_DIR } from "./config";
+import * as logger from "./logger";
+import readline from "readline";
+
+const TIMEOUT = +config.get("EXT_TIMEOUT");
+
+const processes: { [script: string]: ChildProcess } = {};
+const jobs = new Map();
+
+export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
+  return new Promise((resolve) => {
+    const scriptName = args[0];
+
+    const id = crypto.randomBytes(8).toString("hex");
+    jobs.set(id, resolve);
+
+    if (!processes[scriptName]) {
+      const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
+        stdio: ["ignore", "pipe", "pipe", "ipc"],
+      });
+      processes[scriptName] = p;
+
+      p.on("error", (err) => {
+        if (processes[scriptName] === p) {
+          if (jobs.delete(id)) {
+            resolve({
+              fault: { code: err.name, message: err.message },
+              value: null,
+            });
+          }
+
+          // eslint-disable-next-line @typescript-eslint/no-floating-promises
+          kill(processes[scriptName]);
+          delete processes[scriptName];
+        }
+      });
+
+      p.on("disconnect", () => {
+        if (processes[scriptName] === p) delete processes[scriptName];
+      });
+
+      p.on("message", (message) => {
+        const func = jobs.get(message[0]);
+        if (func) {
+          jobs.delete(message[0]);
+          // Wait for any disconnect even to fire
+          setTimeout(() => {
+            func({ fault: message[1], value: message[2] });
+          });
+        }
+      });
+
+      const rlstdout = readline.createInterface(p.stdout);
+      rlstdout.on("line", (line) => {
+        logger.info({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
+      });
+
+      const rlstderr = readline.createInterface(p.stderr);
+      rlstderr.on("line", (line) => {
+        logger.warn({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
+      });
+    }
+
+    setTimeout(() => {
+      if (jobs.delete(id)) {
+        resolve({
+          fault: { code: "timeout", message: "Extension timed out" },
+          value: null,
+        });
+      }
+    }, TIMEOUT);
+
+    if (!processes[scriptName].connected) return false;
+
+    return processes[scriptName].send([id, args.slice(1)]);
+  });
+}
+
+function kill(process: ChildProcess): Promise<void> {
+  return new Promise((resolve) => {
+    const timeToKill = Date.now() + 5000;
+
+    process.kill();
+
+    const t = setInterval(() => {
+      if (!process.connected) {
+        clearInterval(t);
+        resolve();
+      } else if (Date.now() > timeToKill) {
+        process.kill("SIGKILL");
+        clearInterval(t);
+        resolve();
+      }
+    }, 100);
+  });
+}
+
+export async function killAll(): Promise<void> {
+  await Promise.all(
+    Object.entries(processes).map(([k, p]) => {
+      delete processes[k];
+      return kill(p);
+    })
+  );
+}

lib/forwarded.ts

@@ -0,0 +1,175 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { IncomingMessage } from "http";
+import { TLSSocket } from "tls";
+import { parseCIDR, parse, IPv6, IPv4 } from "ipaddr.js";
+import * as config from "./config";
+
+interface RequestOrigin {
+  localAddress: string;
+  localPort: number;
+  remoteAddress: string;
+  remotePort: number;
+  host: string;
+  encrypted: boolean;
+}
+
+const FORWARDED_HEADER = "" + config.get("FORWARDED_HEADER");
+const cache = new WeakMap<IncomingMessage, RequestOrigin>();
+const cidrs: [IPv4 | IPv6, number][] = [];
+
+for (const str of FORWARDED_HEADER.split(",").map((s) => s.trim())) {
+  try {
+    cidrs.push(parseCIDR(str));
+  } catch (err) {
+    // Not a valid CIDR format, try parsing as IP
+    try {
+      const ip = parse(str);
+      cidrs.push([ip, ip.toByteArray().length * 8]);
+    } catch (err) {
+      // Not a valid IP either, ignore
+    }
+  }
+}
+
+function parseForwardedHeader(str: string): { [name: string]: string } {
+  str = str.toLowerCase();
+  const res: { [name: string]: string } = {};
+  let keyIdx = 0;
+  let valueIdx = -1;
+  let key: string;
+  for (let i = 0; i < str.length; ++i) {
+    const char = str.charCodeAt(i);
+    if (char === 61 /* = */) {
+      if (keyIdx >= 0) {
+        key = str.slice(keyIdx, i).trim();
+        keyIdx = -1;
+        valueIdx = i + 1;
+      }
+    } else if (char === 59 /* ; */) {
+      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
+      valueIdx = -1;
+      keyIdx = i + 1;
+    } else if (char === 44 /* , */) {
+      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
+      return res;
+    } else if (char === 34 /* " */) {
+      if (valueIdx >= 0) {
+        const quoteIdx = i;
+        if (!str.slice(valueIdx, quoteIdx).trim()) {
+          for (i = i + 1; i < str.length; ++i) {
+            const c = str.charCodeAt(i);
+            if (c === 92 /* \ */) ++i;
+            if (c === 34 /* " */) {
+              res[key] = JSON.parse(str.slice(quoteIdx, i + 1).trim());
+              valueIdx = -1;
+              keyIdx = i + 1;
+              break;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  if (valueIdx >= 0) res[key] = str.slice(valueIdx).trim();
+
+  return res;
+}
+
+export function getRequestOrigin(request: IncomingMessage): RequestOrigin {
+  let origin = cache.get(request);
+  if (!origin) {
+    const soc = request.socket;
+    origin = {
+      localAddress: soc.localAddress,
+      localPort: soc.localPort,
+      remoteAddress: soc.remoteAddress,
+      remotePort: soc.remotePort,
+      host: request.headers["host"],
+      encrypted: !!(request.socket as TLSSocket).encrypted,
+    };
+
+    const header = request.headers["forwarded"];
+    if (header) {
+      const ip = parse(soc.remoteAddress) as IPv4;
+      if (cidrs.some((cidr) => ip.match(cidr as [IPv4, number]))) {
+        const parsed = parseForwardedHeader(header);
+
+        if (parsed["proto"] === "https") {
+          origin.encrypted = true;
+          origin.localPort = 443;
+        } else if (parsed["proto"] === "http") {
+          origin.encrypted = false;
+          origin.localPort = 80;
+        }
+
+        if (parsed["host"]) {
+          origin.host = parsed["host"];
+          const [, port] = parsed["host"].split(":", 2);
+          origin.localPort = +port || origin.localPort;
+        }
+
+        if (parsed["for"]) {
+          if (parsed["for"].startsWith("[")) {
+            const i = parsed["for"].lastIndexOf("]");
+            if (i >= 0) {
+              origin.remoteAddress = parsed["for"].slice(1, i);
+              origin.remotePort =
+                parseInt(parsed["for"].slice(i + 2)) || origin.remotePort;
+            }
+          } else {
+            const i = parsed["for"].lastIndexOf(":");
+            if (i >= 0) {
+              origin.remoteAddress = parsed["for"].slice(0, i);
+              origin.remotePort =
+                parseInt(parsed["for"].slice(i + 1)) || origin.remotePort;
+            } else {
+              origin.remoteAddress = parsed["for"];
+            }
+          }
+        }
+
+        if (parsed["by"]) {
+          if (parsed["by"].startsWith("[")) {
+            const i = parsed["by"].lastIndexOf("]");
+            if (i >= 0) {
+              origin.localAddress = parsed["by"].slice(1, i);
+              origin.localPort =
+                parseInt(parsed["by"].slice(i + 2)) || origin.localPort;
+            }
+          } else {
+            const i = parsed["by"].lastIndexOf(":");
+            if (i >= 0) {
+              origin.localAddress = parsed["by"].slice(0, i);
+              origin.localPort =
+                parseInt(parsed["by"].slice(i + 1)) || origin.localPort;
+            } else {
+              origin.localAddress = parsed["by"];
+            }
+          }
+        }
+      }
+    }
+
+    cache.set(request, origin);
+  }
+  return origin;
+}

lib/fs.coffee

@@ -1,49 +0,0 @@
-###
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-config = require './config'
-url = require 'url'
-mongodb = require 'mongodb'
-querystring = require 'querystring'
-db = require './db'
-
-
-listener = (request, response) ->
-  urlParts = url.parse(request.url, true)
-  if request.method == 'GET'
-    filename = querystring.unescape(urlParts.pathname.substring(1))
-    gs = new mongodb.GridStore(db.mongoDb, filename, 'r', {})
-    gs.open((err, gs) ->
-      if err
-        response.writeHead(404)
-        response.end()
-        return
-      stream = gs.stream(true)
-      response.writeHead(200, {'Content-Type' : 'application/octet-stream', 'Content-Length' : gs.length})
-      stream.pipe(response)
-    )
-  else
-    response.writeHead(405, {'Allow': 'GET'})
-    response.end('405 Method Not Allowed')
-
-
-exports.listener = listener

lib/fs.ts

@@ -0,0 +1,94 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as url from "url";
+import * as querystring from "querystring";
+import { IncomingMessage, ServerResponse } from "http";
+import { Collection, GridFSBucket } from "mongodb";
+import { onConnect } from "./db";
+import * as logger from "./logger";
+import { getRequestOrigin } from "./forwarded";
+import memoize from "./common/memoize";
+
+let filesCollection: Collection;
+let filesBucket: GridFSBucket;
+
+onConnect(async (db) => {
+  filesCollection = db.collection("fs.files");
+  filesBucket = new GridFSBucket(db);
+});
+
+const getFile = memoize(
+  (md5: string, size: number, filename: string): Promise<Buffer> => {
+    return new Promise((resolve, reject) => {
+      const buffer = Buffer.allocUnsafe(size);
+      let i = 0;
+      const downloadStream = filesBucket.openDownloadStreamByName(filename);
+      downloadStream.on("error", reject);
+      downloadStream.on("data", (data: Buffer) => {
+        data.copy(buffer, i);
+        i += data.length;
+      });
+      downloadStream.on("end", () => {
+        if (i !== size) reject(new Error("File size mismatch"));
+        else resolve(buffer);
+      });
+    });
+  }
+);
+
+export async function listener(
+  request: IncomingMessage,
+  response: ServerResponse
+): Promise<void> {
+  if (request.method !== "GET") {
+    response.writeHead(405, { Allow: "GET" });
+    response.end("405 Method Not Allowed");
+    return;
+  }
+
+  const urlParts = url.parse(request.url, true);
+  const filename = querystring.unescape(urlParts.pathname.substring(1));
+
+  const log = {
+    message: "Fetch file",
+    filename: filename,
+    remoteAddress: getRequestOrigin(request).remoteAddress,
+  };
+
+  const file = await filesCollection.findOne({ _id: filename });
+
+  if (!file) {
+    response.writeHead(404);
+    response.end();
+    log.message += " not found";
+    logger.accessError(log);
+    return;
+  }
+
+  const buffer = await getFile(file.md5, file.length, filename);
+
+  response.writeHead(200, {
+    "Content-Type": file.contentType || "application/octet-stream",
+    "Content-Length": file.length,
+  });
+
+  response.end(buffer);
+  logger.accessInfo(log);
+}

lib/gpn-heuristic.ts

@@ -0,0 +1,83 @@
+import Path from "./common/path";
+
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const WILDCARD_MULTIPLIER = 2;
+const UNDISCOVERED_DEPTH = 7;
+
+// Simple heuristic to estimate GPN count given a set of patterns to be
+// discovered. Used to decide whether to use nextLevel = false in GPN.
+// gpnPatterns is [pattern, flags]
+// pattern is a path (array)
+// flags is an int where its bits mark the segments in the pattern that
+// need refreshing. Leading 0s indicate that the pattern up to that
+// point has been discovered.
+export function estimateGpnCount(
+  gpnPatterns: [Path, number][],
+  depth = 0
+): number {
+  const children: { [segment: string]: [Path, number][] } = {};
+  const wildcardChildren: [Path, number][] = [];
+  let wildcardDiscovered = false;
+  let gpnCount = 0;
+
+  for (const pattern of gpnPatterns) {
+    const path = pattern[0];
+    const flags = pattern[1] >> depth;
+
+    const k = path.segments[depth] as string;
+
+    if (!k) {
+      if (flags & 1) gpnCount = 1;
+      continue;
+    }
+
+    if (flags & 1) {
+      gpnCount = 1;
+
+      if (depth > UNDISCOVERED_DEPTH) continue;
+    } else if (k === "*") {
+      wildcardDiscovered = true;
+    }
+
+    if (k === "*") {
+      wildcardChildren.push(pattern);
+    } else {
+      children[k] = children[k] || [];
+      children[k].push(pattern);
+    }
+  }
+
+  let wildcardGpnCount = 0;
+  if (!wildcardDiscovered && wildcardChildren.length) {
+    wildcardGpnCount +=
+      estimateGpnCount(wildcardChildren, depth + 1) * WILDCARD_MULTIPLIER;
+  }
+
+  for (const k of Object.keys(children)) {
+    const c = estimateGpnCount(children[k].concat(wildcardChildren), depth + 1);
+    wildcardGpnCount -= c;
+    gpnCount += c;
+  }
+
+  gpnCount += Math.max(0, wildcardGpnCount);
+
+  return gpnCount;
+}

lib/init.ts

@@ -0,0 +1,426 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as localCache from "./local-cache";
+import { generateSalt, hashPassword } from "./auth";
+import * as db from "./ui/db";
+import { del } from "./cache";
+
+interface Status {
+  users: boolean;
+  presets: boolean;
+  filters: boolean;
+  device: boolean;
+  index: boolean;
+  overview: boolean;
+}
+
+const BOOTSTRAP_SCRIPT = `
+const now = Date.now();
+
+// Clear cached data model to force a refresh
+clear("Device", now);
+clear("InternetGatewayDevice", now);
+`.trim();
+
+const DEFAULT_SCRIPT = `
+const hourly = Date.now(3600000);
+
+// Refresh basic parameters hourly
+declare("InternetGatewayDevice.DeviceInfo.HardwareVersion", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.DeviceInfo.SoftwareVersion", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID", {path: hourly, value: hourly});
+// Don't refresh password field periodically because CPEs always report blank passowrds for security reasons
+declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase", {path: hourly, value: 1});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress", {path: hourly, value: hourly});
+declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress", {path: hourly, value: hourly});
+`.trim();
+
+const INFORM_SCRIPT = `
+// Device ID as user name
+const username = declare("DeviceID.ID", {value: 1}).value[0]
+
+// Password will be fixed a given device because Math.random() is seeded with devcie ID by default.
+const password = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER).toString(36);
+
+const informInterval = 300;
+
+// Refresh values daily
+const daily = Date.now(86400000);
+
+// Unique inform offset per device for better load distribution
+const informTime = daily % 86400000;
+
+declare("InternetGatewayDevice.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
+declare("InternetGatewayDevice.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
+declare("InternetGatewayDevice.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
+
+declare("Device.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
+declare("Device.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
+declare("Device.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
+declare("Device.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
+declare("Device.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
+`.trim();
+
+export async function getStatus(): Promise<Status> {
+  const configSnapshot = await localCache.getCurrentSnapshot();
+  const users = localCache.getUsers(configSnapshot);
+  const presets = localCache.getPresets(configSnapshot);
+  const ui = localCache.getUiConfig(configSnapshot);
+
+  return {
+    users: !Object.keys(users).length,
+    presets: !presets.length,
+    filters: !Object.keys(ui["filters"]).length,
+    device: !Object.keys(ui["device"]).length,
+    index: !Object.keys(ui["index"]).length,
+    overview: !Object.keys(ui["overview"]).length,
+  };
+}
+
+export async function seed(options: Record<string, boolean>): Promise<void> {
+  const resources = {};
+  const proms = [];
+
+  if (options.users) {
+    resources["permissions"] = [
+      { role: "admin", resource: "devices", access: 3, validate: "true" },
+      { role: "admin", resource: "faults", access: 3, validate: "true" },
+      { role: "admin", resource: "files", access: 3, validate: "true" },
+      { role: "admin", resource: "presets", access: 3, validate: "true" },
+      { role: "admin", resource: "provisions", access: 3, validate: "true" },
+      { role: "admin", resource: "config", access: 3, validate: "true" },
+      { role: "admin", resource: "permissions", access: 3, validate: "true" },
+      { role: "admin", resource: "users", access: 3, validate: "true" },
+      {
+        role: "admin",
+        resource: "virtualParameters",
+        access: 3,
+        validate: "true",
+      },
+    ];
+
+    resources["users"] = [
+      { username: "admin", password: "admin", roles: ["admin"] },
+    ];
+  }
+
+  if (options.filters) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.filters.0.label", value: "'Serial number'" },
+      { _id: "ui.filters.0.parameter", value: "DeviceID.SerialNumber" },
+      { _id: "ui.filters.0.type", value: "'string'" },
+      { _id: "ui.filters.1.label", value: "'Product class'" },
+      { _id: "ui.filters.1.parameter", value: "DeviceID.ProductClass" },
+      { _id: "ui.filters.1.type", value: "'string'" },
+      { _id: "ui.filters.2.label", value: "'Tag'" },
+      { _id: "ui.filters.2.type", value: "'tag'" },
+    ]);
+  }
+
+  if (options.device) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.device.0.type", value: "'tags'" },
+      { _id: "ui.device.1.type", value: "'ping'" },
+      { _id: "ui.device.2.type", value: "'parameter-list'" },
+      { _id: "ui.device.2.parameters.0.type", value: "'container'" },
+      { _id: "ui.device.2.parameters.0.element", value: "'span.inform'" },
+      { _id: "ui.device.2.parameters.0.label", value: "'Last inform'" },
+      {
+        _id: "ui.device.2.parameters.0.components.0.type",
+        value: "'parameter'",
+      },
+      { _id: "ui.device.2.parameters.0.components.1.chart", value: "'online'" },
+      {
+        _id: "ui.device.2.parameters.0.components.1.type",
+        value: "'overview-dot'",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.type",
+        value: "'summon-button'",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.0",
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.1",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.2",
+        value:
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.3",
+        value:
+          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.4",
+        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.5",
+        value:
+          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.6",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.7",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress",
+      },
+      {
+        _id: "ui.device.2.parameters.0.components.2.parameters.8",
+        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress",
+      },
+      {
+        _id: "ui.device.2.parameters.0.parameter",
+        value: "DATE_STRING(Events.Inform)",
+      },
+      { _id: "ui.device.2.parameters.1.label", value: "'Serial number'" },
+      {
+        _id: "ui.device.2.parameters.1.parameter",
+        value: "DeviceID.SerialNumber",
+      },
+      { _id: "ui.device.2.parameters.2.label", value: "'Product class'" },
+      {
+        _id: "ui.device.2.parameters.2.parameter",
+        value: "DeviceID.ProductClass",
+      },
+      { _id: "ui.device.2.parameters.3.label", value: "'OUI'" },
+      {
+        _id: "ui.device.2.parameters.3.parameter",
+        value: "DeviceID.OUI",
+      },
+      { _id: "ui.device.2.parameters.4.label", value: "'Manufacturer'" },
+      {
+        _id: "ui.device.2.parameters.4.parameter",
+        value: "DeviceID.Manufacturer",
+      },
+      { _id: "ui.device.2.parameters.5.label", value: "'Hardware version'" },
+      {
+        _id: "ui.device.2.parameters.5.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
+      },
+      { _id: "ui.device.2.parameters.6.label", value: "'Software version'" },
+      {
+        _id: "ui.device.2.parameters.6.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
+      },
+      { _id: "ui.device.2.parameters.7.label", value: "'MAC'" },
+      {
+        _id: "ui.device.2.parameters.7.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress",
+      },
+      { _id: "ui.device.2.parameters.8.label", value: "'IP'" },
+      {
+        _id: "ui.device.2.parameters.8.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
+      },
+      { _id: "ui.device.2.parameters.9.label", value: "'WLAN SSID'" },
+      {
+        _id: "ui.device.2.parameters.9.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
+      },
+      { _id: "ui.device.2.parameters.10.label", value: "'WLAN passphrase'" },
+      {
+        _id: "ui.device.2.parameters.10.parameter",
+        value:
+          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
+      },
+      { _id: "ui.device.3.type", value: "'parameter-table'" },
+      { _id: "ui.device.3.label", value: "'WLAN hosts'" },
+      {
+        _id: "ui.device.3.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host",
+      },
+      { _id: "ui.device.3.childParameters.0.label", value: "'Host name'" },
+      { _id: "ui.device.3.childParameters.0.parameter", value: "HostName" },
+      { _id: "ui.device.3.childParameters.1.label", value: "'IP address'" },
+      { _id: "ui.device.3.childParameters.1.parameter", value: "IPAddress" },
+      { _id: "ui.device.3.childParameters.2.label", value: "'MAC address'" },
+      { _id: "ui.device.3.childParameters.2.parameter", value: "MACAddress" },
+      { _id: "ui.device.4.type", value: "'container'" },
+      { _id: "ui.device.4.element", value: "'div'" },
+      { _id: "ui.device.4.components.0.components.0", value: "'Faults'" },
+      { _id: "ui.device.4.components.0.element", value: "'h3'" },
+      { _id: "ui.device.4.components.0.type", value: "'container'" },
+      { _id: "ui.device.4.components.1.type", value: "'device-faults'" },
+      { _id: "ui.device.5.type", value: "'container'" },
+      { _id: "ui.device.5.element", value: "'div.container-full-width'" },
+      {
+        _id: "ui.device.5.components.0.components.0",
+        value: "'All parameters'",
+      },
+      { _id: "ui.device.5.components.0.element", value: "'h3'" },
+      { _id: "ui.device.5.components.0.type", value: "'container'" },
+      { _id: "ui.device.5.components.1.type", value: "'all-parameters'" },
+      { _id: "ui.device.6.type", value: "'device-actions'" },
+    ]);
+  }
+
+  if (options.index) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.index.0.type", value: "'device-link'" },
+      { _id: "ui.index.0.label", value: "'Serial number'" },
+      { _id: "ui.index.0.parameter", value: "DeviceID.SerialNumber" },
+      { _id: "ui.index.0.components.0.type", value: "'parameter'" },
+      { _id: "ui.index.1.label", value: "'Product class'" },
+      { _id: "ui.index.1.parameter", value: "DeviceID.ProductClass" },
+      { _id: "ui.index.2.label", value: "'Software version'" },
+      {
+        _id: "ui.index.2.parameter",
+        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
+      },
+      { _id: "ui.index.3.label", value: "'IP'" },
+      {
+        _id: "ui.index.3.parameter",
+        value:
+          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
+      },
+      { _id: "ui.index.4.label", value: "'SSID'" },
+      {
+        _id: "ui.index.4.parameter",
+        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
+      },
+      { _id: "ui.index.5.type", value: "'container'" },
+      { _id: "ui.index.5.label", value: "'Last inform'" },
+      { _id: "ui.index.5.element", value: "'span.inform'" },
+      { _id: "ui.index.5.parameter", value: "DATE_STRING(Events.Inform)" },
+      { _id: "ui.index.5.components.0.type", value: "'parameter'" },
+      { _id: "ui.index.5.components.1.chart", value: "'online'" },
+      { _id: "ui.index.5.components.1.type", value: "'overview-dot'" },
+      { _id: "ui.index.6.type", value: "'tags'" },
+      { _id: "ui.index.6.label", value: "'Tags'" },
+      { _id: "ui.index.6.parameter", value: "Tags" },
+      { _id: "ui.index.6.unsortable", value: "true" },
+      { _id: "ui.index.6.writable", value: "false" },
+    ]);
+  }
+
+  if (options.overview) {
+    resources["config"] = (resources["config"] || []).concat([
+      { _id: "ui.overview.charts.online.label", value: "'Online status'" },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.color",
+        value: "'#31a354'",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.filter",
+        value: "Events.Inform > NOW() - 5 * 60 * 1000",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.1_onlineNow.label",
+        value: "'Online now'",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.color",
+        value: "'#a1d99b'",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.filter",
+        value:
+          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.2_past24.label",
+        value: "'Past 24 hours'",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.color",
+        value: "'#e5f5e0'",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.filter",
+        value:
+          "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)",
+      },
+      {
+        _id: "ui.overview.charts.online.slices.3_others.label",
+        value: "'Others'",
+      },
+      { _id: "ui.overview.groups.online.label", value: "''" },
+      { _id: "ui.overview.groups.online.charts.0", value: "'online'" },
+    ]);
+  }
+
+  if (options.presets) {
+    resources["presets"] = [
+      {
+        _id: "bootstrap",
+        weight: 0,
+        channel: "bootstrap",
+        events: "0 BOOTSTRAP",
+        provision: "bootstrap",
+      },
+      { _id: "default", weight: 0, channel: "default", provision: "default" },
+      { _id: "inform", weight: 0, channel: "inform", provision: "inform" },
+    ];
+
+    resources["provisions"] = [
+      { _id: "bootstrap", script: BOOTSTRAP_SCRIPT },
+      { _id: "default", script: DEFAULT_SCRIPT },
+      { _id: "inform", script: INFORM_SCRIPT },
+    ];
+  }
+
+  if (resources["permissions"]) {
+    for (const p of resources["permissions"]) {
+      p["_id"] = `${p["role"]}:${p["resource"]}:${p["access"]}`;
+      proms.push(db.putPermission(p["_id"], p));
+    }
+  }
+
+  if (resources["users"]) {
+    for (const u of resources["users"]) {
+      u["salt"] = await generateSalt(64);
+      u["password"] = await hashPassword(u["password"], u["salt"]);
+      u["roles"] = (u["roles"] || []).join(",");
+      u["_id"] = u["username"];
+      delete u["username"];
+      proms.push(db.putUser(u["_id"], u));
+    }
+  }
+
+  if (resources["provisions"]) {
+    for (const p of resources["provisions"])
+      proms.push(db.putProvision(p["_id"], p));
+  }
+
+  if (resources["presets"])
+    for (const p of resources["presets"]) proms.push(db.putPreset(p["_id"], p));
+
+  if (resources["config"])
+    for (const c of resources["config"]) proms.push(db.putConfig(c["_id"], c));
+
+  await proms;
+  return del("presets_hash");
+}

lib/instance-set.js

@@ -1,138 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-
-class InstanceSet {
-
-  constructor() {
-    this.set = new Set();
-  }
-
-  add(instance) {
-    this.set.add(instance);
-  }
-
-  delete(instance) {
-    this.set.delete(instance);
-  }
-
-  superset(instance) {
-    let res = [];
-    for (let inst of this.set) {
-      let match = true;
-      for (let k in instance) {
-        if (inst[k] !== instance[k]) {
-          match = false;
-          break;
-        }
-      }
-
-      if (match)
-        res.push(inst);
-    }
-
-    res.sort(function(a, b) {
-      let keysA = Object.keys(a);
-      let keysB = Object.keys(b);
-
-      if (keysA.length != keysB.length)
-        return keysB.length - keysA.length;
-
-      keysA.sort();
-      keysB.sort();
-
-      for (let i = 0; i < keysA.length; ++ i) {
-        if (keysA[i] > keysB[i])
-          return 1;
-        else if (keysA[i] < keysB[i])
-          return -1;
-        else if (a[keysA[i]] > b[keysB[i]])
-          return 1;
-        else if (a[keysA[i]] < b[keysB[i]])
-          return -1;
-      }
-
-      return 0;
-    });
-
-    return res;
-  }
-
-  subset(instance) {
-    let res = [];
-
-    for (let inst of this.set) {
-      let match = true;
-      for (let k in inst) {
-        if (inst[k] !== instance[k]) {
-          match = false;
-          break;
-        }
-      }
-
-      if (match)
-        res.push(inst);
-    }
-
-    res.sort(function(a, b) {
-      let keysA = Object.keys(a);
-      let keysB = Object.keys(b);
-
-      if (keysA.length != keysB.length)
-        return keysA.length - keysB.length;
-
-      keysA.sort();
-      keysB.sort();
-
-      for (let i = 0; i < keysA.length; ++ i) {
-        if (keysA[i] > keysB[i])
-          return 1;
-        else if (keysA[i] < keysB[i])
-          return -1;
-        else if (a[keysA[i]] > b[keysB[i]])
-          return 1;
-        else if (a[keysA[i]] < b[keysB[i]])
-          return -1;
-      }
-
-      return 0;
-    });
-
-    return res;
-  }
-
-  forEach(callback) {
-    return this.set.forEach(callback);
-  }
-
-  values() {
-    return this.set.values();
-  }
-
-  clear() {
-    this.set.clear();
-  }
-
-  get size() {
-    return this.set.size;
-  }
-}
-
-module.exports = InstanceSet;

lib/instance-set.ts

@@ -0,0 +1,131 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+interface Instance {
+  [name: string]: string;
+}
+
+export default class InstanceSet {
+  private set: Set<Instance>;
+
+  public constructor() {
+    this.set = new Set();
+  }
+
+  public add(instance: Instance): void {
+    this.set.add(instance);
+  }
+
+  public delete(instance: Instance): void {
+    this.set.delete(instance);
+  }
+
+  public superset(instance: Instance): Instance[] {
+    const res = [];
+    for (const inst of this.set) {
+      let match = true;
+      for (const k in instance) {
+        if (inst[k] !== instance[k]) {
+          match = false;
+          break;
+        }
+      }
+
+      if (match) res.push(inst);
+    }
+
+    res.sort((a, b) => {
+      const keysA = Object.keys(a);
+      const keysB = Object.keys(b);
+
+      if (keysA.length !== keysB.length) return keysB.length - keysA.length;
+
+      keysA.sort();
+      keysB.sort();
+
+      for (let i = 0; i < keysA.length; ++i) {
+        if (keysA[i] > keysB[i]) return 1;
+        else if (keysA[i] < keysB[i]) return -1;
+        else if (a[keysA[i]] > b[keysB[i]]) return 1;
+        else if (a[keysA[i]] < b[keysB[i]]) return -1;
+      }
+
+      return 0;
+    });
+
+    return res;
+  }
+
+  public subset(instance: Instance): Instance[] {
+    const res = [];
+
+    for (const inst of this.set) {
+      let match = true;
+      for (const k in inst) {
+        if (inst[k] !== instance[k]) {
+          match = false;
+          break;
+        }
+      }
+
+      if (match) res.push(inst);
+    }
+
+    res.sort((a, b) => {
+      const keysA = Object.keys(a);
+      const keysB = Object.keys(b);
+
+      if (keysA.length !== keysB.length) return keysA.length - keysB.length;
+
+      keysA.sort();
+      keysB.sort();
+
+      for (let i = 0; i < keysA.length; ++i) {
+        if (keysA[i] > keysB[i]) return 1;
+        else if (keysA[i] < keysB[i]) return -1;
+        else if (a[keysA[i]] > b[keysB[i]]) return 1;
+        else if (a[keysA[i]] < b[keysB[i]]) return -1;
+      }
+
+      return 0;
+    });
+
+    return res;
+  }
+
+  public [Symbol.iterator](): IterableIterator<Instance> {
+    return this.set.values();
+  }
+
+  public forEach(callback: (instance: Instance) => void): void {
+    this.set.forEach(callback);
+  }
+
+  public values(): IterableIterator<Instance> {
+    return this.set.values();
+  }
+
+  public clear(): void {
+    this.set.clear();
+  }
+
+  public get size(): number {
+    return this.set.size;
+  }
+}

lib/local-cache.ts

@@ -0,0 +1,548 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as vm from "vm";
+import * as crypto from "crypto";
+import * as config from "./config";
+import * as db from "./db";
+import * as cache from "./cache";
+import { convertOldPrecondition } from "./mongodb-functions";
+import * as logger from "./logger";
+import * as scheduling from "./scheduling";
+import * as expression from "./common/expression";
+import { parse } from "./common/expression-parser";
+import {
+  Preset,
+  Expression,
+  Provisions,
+  VirtualParameters,
+  Files,
+  Users,
+  Permissions,
+  Config,
+  UiConfig,
+} from "./types";
+
+interface Snapshot {
+  presets: Preset[];
+  provisions: Provisions;
+  virtualParameters: VirtualParameters;
+  files: Files;
+  permissions: Permissions;
+  users: Users;
+  config: Config;
+  ui: UiConfig;
+}
+
+const REFRESH = 5000;
+const EVICT_TIMEOUT = 120000;
+
+const snapshots = new Map<string, Snapshot>();
+let currentSnapshot: string = null;
+let nextRefresh = 1;
+
+function computeHash(snapshot): string {
+  // MD5 hash for presets, provisions, virtual parameters for detecting changes
+  const h = crypto.createHash("md5");
+  for (const p of snapshot.presets) {
+    h.update(JSON.stringify(p.name));
+    h.update(JSON.stringify(p.channel));
+    h.update(JSON.stringify(p.schedule));
+    h.update(JSON.stringify(p.events));
+    h.update(JSON.stringify(p.precondition));
+    h.update(JSON.stringify(p.provisions));
+  }
+
+  let keys;
+
+  keys = Object.keys(snapshot.provisions).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(snapshot.provisions[k].md5);
+
+  keys = Object.keys(snapshot.virtualParameters).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(snapshot.virtualParameters[k].md5);
+
+  keys = Object.keys(snapshot.config).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(JSON.stringify(snapshot.config[k]));
+
+  keys = Object.keys(snapshot.files).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(JSON.stringify(snapshot.files[k]));
+
+  keys = Object.keys(snapshot.users).sort();
+  h.update(JSON.stringify(keys));
+  for (const k of keys) h.update(JSON.stringify(snapshot.users[k]));
+
+  const roles = Object.keys(snapshot.permissions).sort();
+  h.update(JSON.stringify(roles));
+  for (const r of roles) {
+    const levels = Object.keys(snapshot.permissions[r]).sort();
+    h.update(JSON.stringify(levels));
+    for (const l of levels) {
+      keys = Object.keys(snapshot.permissions[r][l]).sort();
+      h.update(JSON.stringify(keys));
+      for (const k of keys)
+        h.update(JSON.stringify(snapshot.permissions[r][l][k]));
+    }
+  }
+
+  return h.digest("hex");
+}
+
+function flattenObject(
+  src: Record<string, unknown>,
+  prefix = "",
+  dst = {}
+): Record<string, unknown> {
+  for (const k of Object.keys(src)) {
+    const v = src[k];
+    if (typeof v === "object" && !Array.isArray(v))
+      flattenObject(v as Record<string, unknown>, `${prefix}${k}.`, dst);
+    else dst[`${prefix}${k}`] = v;
+  }
+  return dst;
+}
+
+async function fetchPresets(): Promise<Preset[]> {
+  const res = await db.getPresets();
+  let objects = await db.getObjects();
+
+  objects = objects.map((obj) => {
+    // Flatten object
+    obj = flattenObject(obj);
+
+    // If no keys are defined, consider all parameters as keys to keep the
+    // same behavior from v1.0
+    if (!obj["_keys"] || !obj["_keys"].length)
+      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
+
+    return obj;
+  });
+
+  res.sort((a, b) => {
+    if (a["weight"] === b["weight"])
+      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
+    else return a["weight"] - b["weight"];
+  });
+
+  const presets = [] as Preset[];
+  for (const preset of res) {
+    let schedule: { md5: string; duration: number; schedule: any } = null;
+    if (preset["schedule"]) {
+      const parts = preset["schedule"].trim().split(/\s+/);
+      schedule = {
+        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
+        duration: null,
+        schedule: null,
+      };
+
+      try {
+        schedule.duration = +parts.shift() * 1000;
+        schedule.schedule = scheduling.parseCron(parts.join(" "));
+      } catch (err) {
+        logger.warn({
+          message: "Invalid preset schedule",
+          preset: preset["_id"],
+          schedule: preset["schedule"],
+        });
+        schedule.schedule = false;
+      }
+    }
+
+    const events = preset["events"] || {};
+    let precondition = true as Expression;
+    if (preset["precondition"]) {
+      try {
+        precondition = parse(preset["precondition"]);
+      } catch (error) {
+        precondition = convertOldPrecondition(
+          JSON.parse(preset["precondition"])
+        );
+      }
+
+      // Simplify expression
+      precondition = expression.evaluate(precondition);
+    }
+
+    const _provisions = preset["provisions"] || [];
+
+    // Generate provisions from the old configuration format
+    for (const c of preset["configurations"]) {
+      switch (c.type) {
+        case "age":
+          _provisions.push(["refresh", c.name, +c.age]);
+          break;
+
+        case "value":
+          _provisions.push(["value", c.name, c.value]);
+          break;
+
+        case "add_tag":
+          _provisions.push(["tag", c.tag, true]);
+          break;
+
+        case "delete_tag":
+          _provisions.push(["tag", c.tag, false]);
+          break;
+
+        case "provision":
+          _provisions.push([c.name].concat(c.args || []));
+          break;
+
+        case "add_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 1]);
+
+              for (const k in obj) {
+                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
+                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
+              }
+            }
+          }
+
+          break;
+
+        case "delete_object":
+          for (const obj of objects) {
+            if (obj["_id"] === c.object) {
+              const alias = obj["_keys"]
+                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
+                .join(",");
+              const p = `${c.name}.[${alias}]`;
+              _provisions.push(["instances", p, 0]);
+            }
+          }
+
+          break;
+
+        default:
+          throw new Error(`Unknown configuration type ${c.type}`);
+      }
+    }
+
+    presets.push({
+      name: preset["_id"],
+      channel: (preset["channel"] as string) || "default",
+      schedule: schedule,
+      events: events,
+      precondition: precondition,
+      provisions: _provisions,
+    });
+  }
+
+  return presets;
+}
+
+async function fetchProvisions(): Promise<Provisions> {
+  const res = await db.getProvisions();
+
+  const provisions = {};
+  for (const r of res) {
+    provisions[r["_id"]] = {};
+    provisions[r["_id"]].md5 = crypto
+      .createHash("md5")
+      .update(r["script"])
+      .digest("hex");
+    provisions[
+      r["_id"]
+    ].script = new vm.Script(
+      `"use strict";(function(){\n${r["script"]}\n})();`,
+      { filename: r["_id"], lineOffset: -1, timeout: 50 }
+    );
+  }
+
+  return provisions;
+}
+
+async function fetchVirtualParameters(): Promise<VirtualParameters> {
+  const res = await db.getVirtualParameters();
+
+  const virtualParameters = {};
+  for (const r of res) {
+    virtualParameters[r["_id"]] = {};
+    virtualParameters[r["_id"]].md5 = crypto
+      .createHash("md5")
+      .update(r["script"])
+      .digest("hex");
+    virtualParameters[
+      r["_id"]
+    ].script = new vm.Script(
+      `"use strict";(function(){\n${r["script"]}\n})();`,
+      { filename: r["_id"], lineOffset: -1, timeout: 50 }
+    );
+  }
+
+  return virtualParameters;
+}
+
+async function fetchPermissions(): Promise<Permissions> {
+  const perms = await db.getPermissions();
+  const permissions: Permissions = {};
+
+  for (const p of perms) {
+    if (!permissions[p.role]) permissions[p.role] = {};
+    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
+
+    permissions[p.role][p.access][p.resource] = {
+      access: p.access,
+      filter: parse(p.filter || "true"),
+    };
+    if (p.validate)
+      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
+  }
+
+  return permissions;
+}
+
+async function fetchFiles(): Promise<Files> {
+  const res = await db.getFiles();
+  const files = {};
+
+  for (const r of res) {
+    const id = r["filename"] || r["_id"].toString();
+    files[id] = {};
+    files[id].length = r["length"];
+    files[id].md5 = r["md5"];
+    files[id].contentType = r["contentType"];
+  }
+
+  return files;
+}
+
+async function fetchUsers(): Promise<Users> {
+  const _users = await db.getUsers();
+  const users = {};
+
+  for (const user of _users) {
+    users[user._id] = {
+      password: user.password,
+      salt: user.salt,
+      roles: user.roles.split(",").map((s) => s.trim()),
+    };
+  }
+
+  return users;
+}
+
+async function fetchConfig(): Promise<[Config, UiConfig]> {
+  const conf = await db.getConfig();
+
+  conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
+
+  const ui = {
+    filters: {},
+    device: {},
+    index: {},
+    overview: {},
+  };
+  const _config = {};
+
+  for (const c of conf) {
+    // Evaluate expressions to simplify them
+    const val = expression.evaluate(c.value);
+    _config[c.id] = val;
+    if (c.id.startsWith("ui.")) {
+      const keys = c.id.split(".");
+      // remove the first key(ui)
+      keys.shift();
+      let ref = ui;
+      while (keys.length > 1) {
+        const k = keys.shift();
+        if (typeof ref[k] !== "object") ref[k] = {};
+        ref = ref[k];
+      }
+      ref[keys[0]] = val;
+    }
+  }
+
+  return [_config, ui];
+}
+
+async function refresh(): Promise<void> {
+  if (!nextRefresh) {
+    await new Promise((resolve) => setTimeout(resolve, 20));
+    await refresh();
+    return;
+  }
+
+  nextRefresh = 0;
+  const now = Date.now();
+
+  const dbHash = await cache.get("presets_hash");
+
+  if (currentSnapshot && dbHash === currentSnapshot) {
+    nextRefresh = now + (REFRESH - (now % REFRESH));
+    return;
+  }
+
+  const lockToken = await cache.acquireLock("presets_hash_lock", 5000, 5000);
+
+  const res = await Promise.all([
+    fetchPresets(),
+    fetchProvisions(),
+    fetchVirtualParameters(),
+    fetchFiles(),
+    fetchPermissions(),
+    fetchUsers(),
+    fetchConfig(),
+  ]);
+
+  const snapshot = {
+    presets: res[0],
+    provisions: res[1],
+    virtualParameters: res[2],
+    files: res[3],
+    permissions: res[4],
+    users: res[5],
+    config: res[6][0],
+    ui: res[6][1],
+  };
+
+  if (currentSnapshot) {
+    const h = currentSnapshot;
+    const s = snapshots.get(h);
+    setTimeout(() => {
+      if (snapshots.get(h) === s) snapshots.delete(h);
+    }, EVICT_TIMEOUT).unref();
+  }
+
+  currentSnapshot = computeHash(snapshot);
+  snapshots.set(currentSnapshot, snapshot);
+  await cache.set("presets_hash", currentSnapshot, 300);
+  await cache.releaseLock("presets_hash_lock", lockToken);
+
+  nextRefresh = now + (REFRESH - (now % REFRESH));
+}
+
+export async function getCurrentSnapshot(): Promise<string> {
+  if (Date.now() > nextRefresh) await refresh();
+  return currentSnapshot;
+}
+
+export function hasSnapshot(hash: string): boolean {
+  return snapshots.has(hash);
+}
+
+export function getPresets(snapshotKey: string): Preset[] {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.presets;
+}
+
+export function getProvisions(snapshotKey: string): Provisions {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.provisions;
+}
+
+export function getVirtualParameters(snapshotKey: string): VirtualParameters {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.virtualParameters;
+}
+
+export function getFiles(snapshotKey: string): Files {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+  return snapshot.files;
+}
+
+export function getConfig(
+  snapshotKey: string,
+  key: string,
+  context: Record<string, unknown>,
+  now: number,
+  cb?: (e: Expression) => Expression
+): string | number | boolean | null {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  const oldOpts = {
+    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
+    "cwmp.debug": "DEBUG",
+    "cwmp.retryDelay": "RETRY_DELAY",
+    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
+    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
+    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
+    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
+    "cwmp.cookiesPath": "COOKIES_PATH",
+    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
+    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
+    "cwmp.connectionRequestAllowBasicAuth":
+      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
+    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
+    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
+    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
+  };
+
+  if (!(key in snapshot.config)) {
+    if (key in oldOpts) {
+      let id;
+      if (context && context["id"]) {
+        id = context["id"];
+      } else if (cb) {
+        id = cb(["PARAM", "DeviceID.ID"]);
+        if (Array.isArray(id)) id = null;
+      }
+      return config.get(oldOpts[key], id);
+    }
+    return null;
+  }
+
+  const v = expression.evaluate(snapshot.config[key], context, now, cb);
+  return Array.isArray(v) ? null : v;
+}
+
+export function getConfigExpression(
+  snapshotKey: string,
+  key: string
+): Expression {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.config[key];
+}
+
+export function getUsers(snapshotKey: string): Users {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.users;
+}
+
+export function getPermissions(snapshotKey: string): Permissions {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.permissions;
+}
+
+export function getUiConfig(snapshotKey: string): UiConfig {
+  const snapshot = snapshots.get(snapshotKey);
+  if (!snapshot) throw new Error("Cache snapshot does not exist");
+
+  return snapshot.ui;
+}

lib/logger.js

@@ -1,319 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const fs = require("fs");
-const os = require("os");
-
-const config = require("./config");
-
-const REOPEN_EVERY = 60000;
-
-const LOG_FORMAT = config.get("LOG_FORMAT");
-const ACCESS_LOG_FORMAT = config.get("ACCESS_LOG_FORMAT") || LOG_FORMAT;
-
-const defaultMeta = {};
-
-let LOG_SYSTEMD = false;
-let ACCESS_LOG_SYSTEMD = false;
-
-let LOG_FILE, ACCESS_LOG_FILE;
-
-let logStream = fs.createWriteStream(null, {fd: process.stderr.fd});
-let logStat = fs.fstatSync(logStream.fd);
-let accessLogStream = fs.createWriteStream(null, {fd: process.stdout.fd});
-let accessLogStat = fs.fstatSync(accessLogStream.fd);
-
-
-// Reopen if original files have been moved (e.g. logrotate)
-function reopen() {
-  let counter = 1;
-
-  if (LOG_FILE) {
-    ++ counter;
-    fs.stat(LOG_FILE, function(err, stat) {
-      if (err && !err.message.startsWith("ENOENT:"))
-        throw err;
-
-      if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
-        logStream.end();
-        logStream = fs.createWriteStream(null,
-          {fd: fs.openSync(LOG_FILE, "a")});
-        logStat = fs.fstatSync(logStream.fd);
-      }
-
-      if (-- counter === 0)
-        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-    });
-  }
-
-  if (ACCESS_LOG_FILE) {
-    ++ counter;
-    fs.stat(ACCESS_LOG_FILE, function(err, stat) {
-      if (err && !err.message.startsWith("ENOENT:"))
-        throw err;
-
-      if (!(stat && stat.dev === accessLogStat.dev && stat.ino === accessLogStat.ino)) {
-        accessLogStream.end();
-        accessLogStream = fs.createWriteStream(null,
-          {fd: fs.openSync(ACCESS_LOG_FILE, "a")});
-        accessLogStat = fs.fstatSync(accessLogStream.fd);
-      }
-
-      if (-- counter === 0)
-        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-    });
-  }
-
-  if (-- counter === 0)
-    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-}
-
-
-function init(service, version) {
-  defaultMeta.hostname = os.hostname();
-  defaultMeta.pid = process.pid;
-  defaultMeta.name = `genieacs-${service}`;
-  defaultMeta.version = version;
-
-  LOG_FILE = config.get(`${service.toUpperCase()}_LOG_FILE`);
-  ACCESS_LOG_FILE = config.get(`${service.toUpperCase()}_ACCESS_LOG_FILE`);
-
-  if (LOG_FILE) {
-    logStream = fs.createWriteStream(null,
-      {fd: fs.openSync(LOG_FILE, "a")});
-    logStat = fs.fstatSync(logStream.fd);
-  }
-
-  if (ACCESS_LOG_FILE) {
-    accessLogStream = fs.createWriteStream(null,
-      {fd: fs.openSync(ACCESS_LOG_FILE, "a")});
-    accessLogStat = fs.fstatSync(accessLogStream.fd);
-  }
-
-  // Determine if logs are going to journald
-  const JOURNAL_STREAM = process.env["JOURNAL_STREAM"];
-
-  if (JOURNAL_STREAM) {
-    let [dev, inode] = JOURNAL_STREAM.split(":");
-    dev = parseInt(dev);
-    inode = parseInt(inode);
-
-    LOG_SYSTEMD = (logStat.dev === dev && logStat.ino === inode);
-    ACCESS_LOG_SYSTEMD = (accessLogStat.dev === dev && accessLogStat.ino === inode);
-  }
-
-  if (LOG_FILE || ACCESS_LOG_FILE) {
-    // Can't use setInterval as we need all workers to cehck at the same time
-    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-  }
-}
-
-
-function close() {
-  accessLogStream.end();
-  logStream.end();
-}
-
-
-function flatten(details) {
-  if (details.sessionContext) {
-    details.deviceId = details.sessionContext.deviceId;
-    details.remoteAddress = details.sessionContext.httpRequest.connection.remoteAddress;
-    delete details.sessionContext;
-  }
-
-  if (details.exception) {
-    details.exceptionName = details.exception.name;
-    details.exceptionMessage = details.exception.message;
-    details.exceptionStack = details.exception.stack;
-    delete details.exception;
-  }
-
-  if (details.task) {
-    details.taskId = details.task._id;
-    delete details.task;
-  }
-
-  if (details.rpc) {
-    if (details.rpc.acsRequest) {
-      details.acsRequestId = details.rpc.id;
-      details.acsRequestName = details.rpc.acsRequest.name;
-      if (details.rpc.acsRequest.commandKey)
-        details.acsRequestCommandKey = details.rpc.acsRequest.commandKey;
-    }
-    else if (details.rpc.cpeRequest) {
-      details.cpeRequestId = details.rpc.id;
-      if (details.rpc.cpeRequest.name === "Inform") {
-        details.informEvent = details.rpc.cpeRequest.event.join(",");
-        details.informRetryCount = details.rpc.cpeRequest.retryCount;
-      }
-      else {
-        details.cpeRequestName = details.rpc.cpeRequest.type;
-        if (details.rpc.cpeRequest.commandKey)
-          details.cpeRequestCommandKey = details.rpc.cpeRequest.commandKey;
-      }
-    }
-    else if (details.rpc.cpeFault) {
-      details.acsRequestId = details.rpc.id;
-      details.cpeFaultCode = details.rpc.cpeFault.faultCode;
-      details.cpeFaultString = details.rpc.cpeFault.faultString;
-    }
-    delete details.rpc;
-  }
-
-  if (details.fault) {
-    details.faultCode = details.fault.code;
-    details.faultMessage = details.fault.message;
-    delete details.fault;
-  }
-
-  return details;
-}
-
-
-function formatJson(details, systemd) {
-  if (systemd) {
-    let severity = "";
-    if (details.severity === "info")
-      severity = "<6>";
-    else if (details.severity === "warn")
-      severity = "<4>";
-    else if (details.severity === "error")
-      severity = "<3>";
-
-    return `${severity}${JSON.stringify(flatten(details))}${os.EOL}`;
-  }
-
-  return `${JSON.stringify(flatten(details))}${os.EOL}`;
-}
-
-
-function formatSimple(details, systemd) {
-  flatten(details);
-
-  let remote = "";
-  if (details.remoteAddress) {
-    if (details.deviceId)
-      remote = `${details.remoteAddress} ${details.deviceId}: `;
-    else
-      remote = `${details.remoteAddress}: `;
-  }
-
-  const keys = Object.keys(details);
-  const skip = {
-    deviceId: 1,
-    remoteAddress: 1,
-    severity: 1,
-    timestamp: 1,
-    message: 1
-  };
-
-  let meta = "";
-
-  let kv = [];
-  for (let k of keys)
-    if (!skip[k])
-      kv.push(`${k}=${JSON.stringify(details[k])}`);
-
-  if (kv.length)
-    meta = `; ${kv.join(" ")}`;
-
-  if (systemd) {
-    let severity = "";
-    if (details.severity === "info")
-      severity = "<6>";
-    else if (details.severity === "warn")
-      severity = "<4>";
-    else if (details.severity === "error")
-      severity = "<3>";
-
-    return `${severity}${remote}${details.message}${meta}${os.EOL}`;
-  }
-
-  return `${details.timestamp} [${details.severity.toUpperCase()}] ${remote}${details.message}${meta}${os.EOL}`;
-}
-
-
-function log(details) {
-  details.timestamp = new Date().toISOString();
-  if (LOG_FORMAT === "json") {
-    details = Object.assign({}, defaultMeta, details);
-    logStream.write(formatJson(details, LOG_SYSTEMD));
-  } else {
-    logStream.write(formatSimple(details, LOG_SYSTEMD));
-  }
-}
-
-
-function info(details) {
-  details.severity = "info";
-  log(details);
-}
-
-
-function warn(details) {
-  details.severity = "warn";
-  log(details);
-}
-
-
-function error(details) {
-  details.severity = "error";
-  log(details);
-}
-
-
-function accessLog(details) {
-  details.timestamp = new Date().toISOString();
-  if (ACCESS_LOG_FORMAT === "json") {
-    Object.assign(details, defaultMeta);
-    accessLogStream.write(formatJson(details, ACCESS_LOG_SYSTEMD));
-  } else {
-    accessLogStream.write(formatSimple(details, ACCESS_LOG_SYSTEMD));
-  }
-}
-
-
-function accessInfo(details) {
-  details.severity = "info";
-  accessLog(details);
-}
-
-
-function accessWarn(details) {
-  details.severity = "warn";
-  accessLog(details);
-}
-
-
-function accessError(details) {
-  details.severity = "error";
-  accessLog(details);
-}
-
-
-exports.init = init;
-exports.close = close;
-exports.info = info;
-exports.warn = warn;
-exports.error = error;
-exports.accessInfo = accessInfo;
-exports.accessWarn = accessWarn;
-exports.accessError = accessError;

lib/logger.ts

@@ -0,0 +1,350 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as fs from "fs";
+import * as os from "os";
+
+import * as config from "./config";
+import { getRequestOrigin } from "./forwarded";
+import {
+  SessionContext,
+  AcsRequest,
+  CpeRequest,
+  CpeFault,
+  InformRequest,
+  Fault,
+} from "./types";
+
+const REOPEN_EVERY = 60000;
+
+const LOG_FORMAT = config.get("LOG_FORMAT");
+const ACCESS_LOG_FORMAT = config.get("ACCESS_LOG_FORMAT") || LOG_FORMAT;
+
+const defaultMeta: { [name: string]: any } = {};
+
+let LOG_SYSTEMD = false;
+let ACCESS_LOG_SYSTEMD = false;
+
+let LOG_FILE, ACCESS_LOG_FILE;
+
+declare global {
+  /* eslint-disable-next-line @typescript-eslint/no-namespace */
+  namespace NodeJS {
+    export interface WritableStream {
+      fd?: number;
+    }
+  }
+}
+
+declare module "fs" {
+  interface WriteStream {
+    fd?: number;
+  }
+}
+
+let logStream = fs.createWriteStream(null, { fd: process.stderr.fd });
+let logStat = fs.fstatSync(logStream.fd);
+let accessLogStream = fs.createWriteStream(null, { fd: process.stdout.fd });
+let accessLogStat = fs.fstatSync(accessLogStream.fd);
+
+// Reopen if original files have been moved (e.g. logrotate)
+function reopen(): void {
+  let counter = 1;
+
+  if (LOG_FILE) {
+    ++counter;
+    fs.stat(LOG_FILE, (err, stat) => {
+      if (err && !err.message.startsWith("ENOENT:")) throw err;
+
+      if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
+        logStream.end();
+        logStream = fs.createWriteStream(null, {
+          fd: fs.openSync(LOG_FILE, "a"),
+        });
+        logStat = fs.fstatSync(logStream.fd);
+      }
+
+      if (--counter === 0)
+        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+    });
+  }
+
+  if (ACCESS_LOG_FILE) {
+    ++counter;
+    fs.stat(ACCESS_LOG_FILE, (err, stat) => {
+      if (err && !err.message.startsWith("ENOENT:")) throw err;
+
+      if (
+        !(
+          stat &&
+          stat.dev === accessLogStat.dev &&
+          stat.ino === accessLogStat.ino
+        )
+      ) {
+        accessLogStream.end();
+        accessLogStream = fs.createWriteStream(null, {
+          fd: fs.openSync(ACCESS_LOG_FILE, "a"),
+        });
+        accessLogStat = fs.fstatSync(accessLogStream.fd);
+      }
+
+      if (--counter === 0)
+        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+    });
+  }
+
+  if (--counter === 0)
+    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+}
+
+export function init(service: string, version: string): void {
+  defaultMeta.hostname = os.hostname();
+  defaultMeta.pid = process.pid;
+  defaultMeta.name = `genieacs-${service}`;
+  defaultMeta.version = version;
+
+  LOG_FILE = config.get(`${service.toUpperCase()}_LOG_FILE`);
+  ACCESS_LOG_FILE = config.get(`${service.toUpperCase()}_ACCESS_LOG_FILE`);
+
+  if (LOG_FILE) {
+    logStream = fs.createWriteStream(null, { fd: fs.openSync(LOG_FILE, "a") });
+    logStat = fs.fstatSync(logStream.fd);
+  }
+
+  if (ACCESS_LOG_FILE) {
+    accessLogStream = fs.createWriteStream(null, {
+      fd: fs.openSync(ACCESS_LOG_FILE, "a"),
+    });
+    accessLogStat = fs.fstatSync(accessLogStream.fd);
+  }
+
+  // Determine if logs are going to journald
+  const JOURNAL_STREAM = process.env["JOURNAL_STREAM"];
+
+  if (JOURNAL_STREAM) {
+    const [dev, inode] = JOURNAL_STREAM.split(":").map(parseInt);
+
+    LOG_SYSTEMD = logStat.dev === dev && logStat.ino === inode;
+    ACCESS_LOG_SYSTEMD =
+      accessLogStat.dev === dev && accessLogStat.ino === inode;
+  }
+
+  if (LOG_FILE || ACCESS_LOG_FILE)
+    // Can't use setInterval as we need all workers to cehck at the same time
+    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+}
+
+export function close(): void {
+  accessLogStream.end();
+  logStream.end();
+}
+
+export function flatten(
+  details: Record<string, unknown>
+): Record<string, unknown> {
+  if (details.sessionContext) {
+    const sessionContext = details.sessionContext as SessionContext;
+    details.deviceId = sessionContext.deviceId;
+    details.remoteAddress = getRequestOrigin(
+      sessionContext.httpRequest
+    ).remoteAddress;
+    delete details.sessionContext;
+  }
+
+  if (details.exception) {
+    const err = details.exception as Error;
+    details.exceptionName = err.name;
+    details.exceptionMessage = err.message;
+    details.exceptionStack = err.stack;
+    delete details.exception;
+  }
+
+  if (details.task) {
+    details.taskId = details.task["_id"];
+    delete details.task;
+  }
+
+  if (details.rpc) {
+    const rpc = details.rpc as {
+      id: string;
+      acsRequest?: AcsRequest;
+      cpeRequest?: CpeRequest;
+      cpeFault?: CpeFault;
+    };
+    if (rpc.acsRequest) {
+      details.acsRequestId = rpc.id;
+      details.acsRequestName = rpc.acsRequest.name;
+      if (rpc.acsRequest["commandKey"])
+        details.acsRequestCommandKey = rpc.acsRequest["commandKey"];
+    } else if (rpc.cpeRequest) {
+      details.cpeRequestId = rpc.id;
+      if (rpc.cpeRequest.name === "Inform") {
+        details.informEvent = (rpc.cpeRequest as InformRequest).event.join(",");
+        details.informRetryCount = (rpc.cpeRequest as InformRequest).retryCount;
+      } else {
+        details.cpeRequestName = rpc.cpeRequest.name;
+        if (rpc.cpeRequest["commandKey"])
+          details.cpeRequestCommandKey = rpc.cpeRequest["commandKey"];
+      }
+    } else if (rpc.cpeFault) {
+      details.acsRequestId = rpc.id;
+      details.cpeFaultCode = rpc.cpeFault.detail.faultCode;
+      details.cpeFaultString = rpc.cpeFault.detail.faultString;
+    }
+    delete details.rpc;
+  }
+
+  if (details.fault) {
+    const fault = details.fault as Fault;
+    details.faultCode = fault.code;
+    details.faultMessage = fault.message;
+    delete details.fault;
+  }
+
+  // For genieacs-ui
+  if (details.context) {
+    details.remoteAddress = getRequestOrigin(
+      details.context["req"]
+    ).remoteAddress;
+    if (details.context["state"].user)
+      details.user = details.context["state"].user.username;
+    delete details.context;
+  }
+
+  for (const [k, v] of Object.entries(details))
+    if (v == null) delete details[k];
+
+  return details;
+}
+
+function formatJson(
+  details: Record<string, unknown>,
+  systemd: boolean
+): string {
+  if (systemd) {
+    let severity = "";
+    if (details.severity === "info") severity = "<6>";
+    else if (details.severity === "warn") severity = "<4>";
+    else if (details.severity === "error") severity = "<3>";
+
+    return `${severity}${JSON.stringify(flatten(details))}${os.EOL}`;
+  }
+
+  return `${JSON.stringify(flatten(details))}${os.EOL}`;
+}
+
+function formatSimple(
+  details: Record<string, unknown>,
+  systemd: boolean
+): string {
+  const skip = {
+    user: true,
+    remoteAddress: true,
+    severity: true,
+    timestamp: true,
+    message: true,
+    deviceId: !!details.sessionContext,
+  };
+
+  flatten(details);
+
+  let remote = "";
+  if (details.remoteAddress) {
+    if (details.deviceId && skip["deviceId"])
+      remote = `${details.remoteAddress} ${details.deviceId}: `;
+    else if (details.user)
+      remote = `${details.user}@${details.remoteAddress}: `;
+    else remote = `${details.remoteAddress}: `;
+  }
+
+  const keys = Object.keys(details);
+
+  let meta = "";
+
+  const kv = [];
+  for (const k of keys)
+    if (!skip[k]) kv.push(`${k}=${JSON.stringify(details[k])}`);
+
+  if (kv.length) meta = `; ${kv.join(" ")}`;
+
+  if (systemd) {
+    let severity = "";
+    if (details.severity === "info") severity = "<6>";
+    else if (details.severity === "warn") severity = "<4>";
+    else if (details.severity === "error") severity = "<3>";
+
+    return `${severity}${remote}${details.message}${meta}${os.EOL}`;
+  }
+
+  return `${
+    details.timestamp
+  } [${(details.severity as string).toUpperCase()}] ${remote}${
+    details.message
+  }${meta}${os.EOL}`;
+}
+
+function log(details: Record<string, unknown>): void {
+  details.timestamp = new Date().toISOString();
+  if (LOG_FORMAT === "json") {
+    details = Object.assign({}, defaultMeta, details);
+    logStream.write(formatJson(details, LOG_SYSTEMD));
+  } else {
+    logStream.write(formatSimple(details, LOG_SYSTEMD));
+  }
+}
+
+export function info(details: Record<string, unknown>): void {
+  details.severity = "info";
+  log(details);
+}
+
+export function warn(details: Record<string, unknown>): void {
+  details.severity = "warn";
+  log(details);
+}
+
+export function error(details: Record<string, unknown>): void {
+  details.severity = "error";
+  log(details);
+}
+
+export function accessLog(details: Record<string, unknown>): void {
+  details.timestamp = new Date().toISOString();
+  if (ACCESS_LOG_FORMAT === "json") {
+    Object.assign(details, defaultMeta);
+    accessLogStream.write(formatJson(details, ACCESS_LOG_SYSTEMD));
+  } else {
+    accessLogStream.write(formatSimple(details, ACCESS_LOG_SYSTEMD));
+  }
+}
+
+export function accessInfo(details: Record<string, unknown>): void {
+  details.severity = "info";
+  accessLog(details);
+}
+
+export function accessWarn(details: Record<string, unknown>): void {
+  details.severity = "warn";
+  accessLog(details);
+}
+
+export function accessError(details: Record<string, unknown>): void {
+  details.severity = "error";
+  accessLog(details);
+}

lib/mongodb-functions.ts

@@ -0,0 +1,684 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { ObjectID } from "mongodb";
+import { map, parse, stringify, parseList } from "./common/expression-parser";
+import { likePatternToRegExp } from "./common/expression";
+import { Expression, Fault, Task } from "./types";
+import { decodeTag, encodeTag } from "./common";
+
+const isArray = Array.isArray;
+
+export function processDeviceFilter(filter: Expression): Expression {
+  return map(filter, (exp) => {
+    if (!isArray(exp)) return exp;
+
+    if (exp[0] === "PARAM") {
+      const p = exp[1];
+      if (p === "DeviceID.ID") return ["PARAM", "_id"];
+      else if (p === "DeviceID") return ["PARAM", "_deviceId"];
+      else if (p.startsWith("DeviceID."))
+        return ["PARAM", "_deviceId._" + p.slice(9)];
+      else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
+      else if (p === "Events.Registered") return ["PARAM", "_registered"];
+      else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
+      else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"];
+    } else if (
+      isArray(exp[1]) &&
+      exp[1][0] === "PARAM" &&
+      exp[1][1].startsWith("Tags.")
+    ) {
+      const t = decodeTag(exp[1][1].slice(5));
+      if (exp[0] === "IS NULL") return ["_tags", t, false];
+      else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
+      else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
+      else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
+    } else if (
+      [
+        "=",
+        "<>",
+        ">",
+        ">=",
+        "<",
+        "<=",
+        "LIKE",
+        "NOT LIKE",
+        "IS NULL",
+        "IS NOT NULL",
+      ].includes(exp[0])
+    ) {
+      let e = map(exp, (ee) => {
+        if (
+          isArray(ee) &&
+          ee[0] === "PARAM" &&
+          typeof ee[1] === "string" &&
+          !ee[1].startsWith("_")
+        )
+          return ["PARAM", `${ee[1]}._value`];
+        return ee;
+      });
+
+      if (typeof e[2] === "number") {
+        const alt = (e as any[]).slice();
+        alt[2] = new Date(e[2]);
+        e = ["OR", e, alt];
+      }
+      return e;
+    }
+    return exp;
+  });
+}
+
+export function processTasksFilter(filter: Expression): Expression {
+  return map(filter, (exp) => {
+    if (!isArray(exp)) return exp;
+    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
+      const e = exp.slice();
+      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectID(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
+        e[2] = new Date(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
+        e[2] = new Date(e[2]);
+      return e;
+    }
+    return exp;
+  });
+}
+
+export function processFaultsFilter(filter: Expression): Expression {
+  return map(filter, (exp) => {
+    if (!isArray(exp)) return exp;
+    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
+      const e = exp.slice();
+      if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]);
+      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
+        e[2] = new Date(e[2]);
+      return e;
+    }
+    return exp;
+  });
+}
+
+export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
+  function recursive(filter, negate): Record<string, unknown> {
+    const op = filter[0];
+
+    if (op === "AND") {
+      filter = filter.filter((f) => f !== true);
+      if (filter.length === 2) return recursive(filter[1], negate);
+    }
+
+    if (op === "OR") {
+      filter = filter.filter((f) => f !== false);
+      if (filter.length === 2) return recursive(filter[1], negate);
+    }
+
+    if ((!negate && op === "AND") || (negate && op === "OR"))
+      return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if ((!negate && op === "OR") || (negate && op === "AND"))
+      return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
+
+    if (op === "NOT") return recursive(filter[1], !negate);
+
+    if (isArray(filter[2]))
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "LIKE" || op === "NOT LIKE") {
+      if (op === "NOT LIKE") negate = !negate;
+      let param;
+      let flags;
+      if (!Array.isArray(filter[1]))
+        throw new Error(`Invalid LHS operand of ${op} clause`);
+      if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
+        if (filter[2] !== filter[2].toUpperCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
+        param = filter[1][2][1];
+        flags = "i";
+      } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
+        if (filter[2] !== filter[2].toLowerCase())
+          throw new Error(`Invalid RHS operand of ${op} clause`);
+        param = filter[1][2][1];
+        flags = "i";
+      } else if (filter[1][0] === "PARAM") {
+        param = filter[1][1];
+        flags = "";
+      } else {
+        throw new Error(`Invalid LHS operand of ${op} clause`);
+      }
+      const r = likePatternToRegExp(filter[2], filter[3], flags);
+      if (negate) return { [param]: { $nin: [r, null] } };
+      else return { [param]: r };
+    }
+
+    if (op === "_tags") {
+      let t = filter[2];
+      if (negate) t = !t;
+      if (t) return { _tags: filter[1] };
+      else return { _tags: { $ne: filter[1] } };
+    }
+
+    if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
+      throw new Error(`Invalid LHS operand of ${op} clause`);
+
+    if (op === "IS NULL") {
+      const p = filter[1][1];
+      return { [p]: null };
+    }
+
+    if (op === "IS NOT NULL") {
+      const p = filter[1][1];
+      return { [p]: { $ne: null } };
+    }
+
+    if (isArray(filter[2]) || filter[2] == null)
+      throw new Error(`Invalid RHS operand of ${op} clause`);
+
+    if (op === "=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (!negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === "<>") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: v };
+      else return { [p]: { $nin: [v, null] } };
+    }
+
+    if (op === ">") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lte: v } };
+      else return { [p]: { $gt: v } };
+    }
+
+    if (op === ">=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $lt: v } };
+      else return { [p]: { $gte: v } };
+    }
+
+    if (op === "<") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gte: v } };
+      else return { [p]: { $lt: v } };
+    }
+
+    if (op === "<=") {
+      const p = filter[1][1];
+      const v = filter[2];
+      if (negate) return { [p]: { $gt: v } };
+      else return { [p]: { $lte: v } };
+    }
+
+    throw new Error(`Unrecognized operator ${op}`);
+  }
+
+  if (!isArray(exp)) {
+    if (exp === true) return {};
+    throw new Error("Primitives are not valid queries");
+  }
+
+  return recursive(exp, false);
+}
+
+export function processDeviceProjection(
+  projection: Record<string, 1>
+): Record<string, 1> {
+  if (!projection) return projection;
+  const p = {};
+  for (const [k, v] of Object.entries(projection)) {
+    if (k === "DeviceID.ID") {
+      p["_id"] = 1;
+    } else if (k.startsWith("DeviceID")) {
+      p["_deviceId._SerialNumber"] = v;
+      p["_deviceId._OUI"] = v;
+      p["_deviceId._ProductClass"] = v;
+      p["_deviceId._Manufacturer"] = v;
+    } else if (k.startsWith("Tags")) {
+      p["_tags"] = v;
+    } else if (k.startsWith("Events")) {
+      p["_lastInform"] = v;
+      p["_registered"] = v;
+      p["_lastBoot"] = v;
+      p["_lastBootstrap"] = v;
+    } else {
+      p[k] = v;
+    }
+  }
+
+  return p;
+}
+
+export function processDeviceSort(
+  sort: Record<string, number>
+): Record<string, number> {
+  if (!sort) return sort;
+  const s = {};
+  for (const [k, v] of Object.entries(sort)) {
+    if (k === "DeviceID.ID") s["_id"] = v;
+    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
+    else if (k === "Events.Inform") s["_lastInform"] = v;
+    else if (k === "Events.Registered") s["_registered"] = v;
+    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
+    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
+    else s[`${k}._value`] = v;
+  }
+
+  return s;
+}
+
+function parseDate(d: Date): number | string {
+  const n = +d;
+  return isNaN(n) ? "" + d : n;
+}
+
+interface FlatAttributes {
+  object?: boolean;
+  objectTimestamp?: number;
+  writable?: boolean;
+  writableTimestamp?: number;
+  value?: [string | number | boolean, string];
+  valueTimestamp?: number;
+  notification?: number;
+  notificationTimestamp?: number;
+  accessList?: string[];
+  accessListTimestamp?: number;
+}
+
+interface FlatDevice {
+  [param: string]: FlatAttributes;
+}
+
+export function flattenDevice(device: Record<string, unknown>): FlatDevice {
+  function recursive(
+    input,
+    root: string,
+    output: FlatDevice,
+    timestamp: number
+  ): void {
+    for (const [name, tree] of Object.entries(input)) {
+      if (!root) {
+        if (name === "_lastInform") {
+          output["Events.Inform"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_registered") {
+          output["Events.Registered"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_lastBoot") {
+          output["Events.1_BOOT"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_lastBootstrap") {
+          output["Events.0_BOOTSTRAP"] = {
+            value: [parseDate(tree as Date), "xsd:dateTime"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_id") {
+          output["DeviceID.ID"] = {
+            value: [tree as string, "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_deviceId") {
+          output["DeviceID.Manufacturer"] = {
+            value: [tree["_Manufacturer"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.OUI"] = {
+            value: [tree["_OUI"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.ProductClass"] = {
+            value: [tree["_ProductClass"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+          output["DeviceID.SerialNumber"] = {
+            value: [tree["_SerialNumber"], "xsd:string"],
+            valueTimestamp: timestamp,
+            writable: false,
+            writableTimestamp: timestamp,
+            object: false,
+            objectTimestamp: timestamp,
+          };
+        } else if (name === "_tags") {
+          output["Tags"] = {
+            writable: false,
+            writableTimestamp: timestamp,
+            object: true,
+            objectTimestamp: timestamp,
+          };
+
+          for (const t of tree as string[]) {
+            output[`Tags.${encodeTag(t)}`] = {
+              value: [true, "xsd:boolean"],
+              valueTimestamp: timestamp,
+              writable: true,
+              writableTimestamp: timestamp,
+              object: false,
+              objectTimestamp: timestamp,
+            };
+          }
+        }
+      }
+
+      if (name.startsWith("_")) continue;
+
+      let childrenTimestamp = timestamp;
+
+      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
+      else if (+input["_timestamp"] > timestamp)
+        childrenTimestamp = +input["_timestamp"];
+
+      const attrs: FlatAttributes = {};
+      if (tree["_value"] != null) {
+        attrs.value = [
+          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
+          tree["_type"],
+        ];
+        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
+        attrs.object = false;
+        attrs.objectTimestamp = childrenTimestamp;
+      } else if (tree["_object"] != null) {
+        attrs.object = tree["_object"];
+        attrs.objectTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_writable"] != null) {
+        attrs.writable = tree["_writable"];
+        attrs.writableTimestamp = childrenTimestamp;
+      }
+
+      if (tree["_notification"] != null) {
+        attrs.notification = tree["_notification"];
+        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      if (tree["_accessList"] != null) {
+        attrs.accessList = tree["_accessList"];
+        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
+      }
+
+      const r = root ? `${root}.${name}` : name;
+      output[r] = attrs;
+
+      if (attrs.object || tree["object"] == null)
+        recursive(tree, r, output, childrenTimestamp);
+    }
+  }
+
+  const newDevice: FlatDevice = {};
+  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
+  recursive(device, "", newDevice, timestamp);
+  return newDevice;
+}
+
+export function flattenFault(fault: unknown): Fault {
+  const f = Object.assign({}, fault) as Fault;
+  if (f.timestamp) f.timestamp = +f.timestamp;
+  if (f["expiry"]) f["expiry"] = +f["expiry"];
+  return f as Fault;
+}
+
+export function flattenTask(task: unknown): Task {
+  const t = Object.assign({}, task) as Task;
+  t._id = "" + t._id;
+  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
+  if (t.expiry) t.expiry = +t.expiry;
+  return t;
+}
+
+export function convertOldPrecondition(
+  query: Record<string, unknown>
+): Expression {
+  function recursive(_query): Expression {
+    const expressions: Expression[] = [];
+    for (const [k, v] of Object.entries(_query)) {
+      if (k[0] === "$") {
+        if (k === "$and") {
+          const and: Expression = ["AND"];
+          for (const vv of Object.values(v)) and.push(recursive(vv));
+          expressions.push(and);
+        } else if (k === "$or") {
+          const or: Expression = ["OR"];
+          for (const vv of Object.values(v)) or.push(recursive(vv));
+          expressions.push(or);
+        } else {
+          throw new Error(`Operator ${k} not supported`);
+        }
+      } else if (k === "_tags") {
+        if (typeof v === "object") {
+          if (isArray(v)) throw new Error(`Invalid type`);
+          const conjs: Expression[] = [];
+          for (const [op, val] of Object.entries(v)) {
+            if (op === "$ne") {
+              if (typeof v["$ne"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else if (op === "$eq") {
+              if (typeof v["$eq"] !== "string")
+                throw new Error("Only string values are allowed for _tags");
+              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
+            } else {
+              throw new Error(`Invalid tag query`);
+            }
+          }
+          if (conjs.length === 1) expressions.push(conjs[0]);
+          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
+        } else {
+          expressions.push([
+            "IS NOT NULL",
+            ["PARAM", `Tags.${encodeTag(v as string)}`],
+          ]);
+        }
+      } else if (k.startsWith("Tags.")) {
+        let exists: boolean;
+        if (typeof v === "boolean") exists = v;
+        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
+        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
+        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
+        else throw new Error(`Invalid tag query`);
+
+        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
+      } else if (typeof v === "object") {
+        if (isArray(v)) throw new Error(`Invalid type`);
+
+        const exps: Expression[] = [];
+        for (const [kk, vv] of Object.entries(v)) {
+          if (kk === "$eq") {
+            exps.push(["=", ["PARAM", k], vv]);
+          } else if (kk === "$ne") {
+            exps.push([
+              "OR",
+              ["<>", ["PARAM", k], vv],
+              ["IS NULL", ["PARAM", k]],
+            ]);
+          } else if (kk === "$lt") {
+            exps.push(["<", ["PARAM", k], vv]);
+          } else if (kk === "$lte") {
+            exps.push(["<=", ["PARAM", k], vv]);
+          } else if (kk === "$gt") {
+            exps.push([">", ["PARAM", k], vv]);
+          } else if (kk === "$gte") {
+            exps.push([">=", ["PARAM", k], vv]);
+          } else {
+            throw new Error(`Operator ${kk} not supported`);
+          }
+          if (!["string", "number", "boolean"].includes(typeof vv))
+            throw new Error(`Invalid value for ${kk} operator`);
+        }
+        if (exps.length === 1) {
+          expressions.push(exps[0]);
+        } else if (exps.length > 1) {
+          const and: Expression = ["AND"];
+          expressions.push(and.concat(exps));
+        }
+      } else {
+        expressions.push(["=", ["PARAM", k], v]);
+      }
+    }
+    if (expressions.length === 1) return expressions[0];
+    if (expressions.length === 0) return true;
+    return ["AND", ...expressions];
+  }
+
+  // empty filter
+  if (!Object.keys(query).length) return true;
+
+  return recursive(query);
+}
+
+export function flattenPreset(
+  preset: Record<string, unknown>
+): Record<string, unknown> {
+  const p = Object.assign({}, preset);
+  if (p.precondition) {
+    try {
+      // Try parse to check expression validity
+      parse(p.precondition as string);
+    } catch (error) {
+      p.precondition = convertOldPrecondition(
+        JSON.parse(p.precondition as string)
+      );
+      p.precondition = (p.precondition as string).length
+        ? stringify(p.precondition as Expression)
+        : "";
+    }
+  }
+
+  if (p.events) {
+    const e = [];
+    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
+    p.events = e.join(", ");
+  }
+
+  const provision = p.configurations[0];
+  if (
+    (p.configurations as any[]).length === 1 &&
+    provision.type === "provision" &&
+    provision.name &&
+    provision.name.length
+  ) {
+    p.provision = provision.name;
+    p.provisionArgs = provision.args
+      ? provision.args.map((a) => stringify(a)).join(", ")
+      : "";
+  }
+
+  delete p.configurations;
+  return p;
+}
+
+export function flattenFile(
+  file: Record<string, unknown>
+): Record<string, unknown> {
+  const f = {};
+  f["_id"] = file["_id"];
+  if (file.metadata) {
+    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
+    f["metadata.oui"] = file["metadata"]["oui"] || "";
+    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
+    f["metadata.version"] = file["metadata"]["version"] || "";
+  }
+  return f;
+}
+
+export function preProcessPreset(
+  data: Record<string, unknown>
+): Record<string, unknown> {
+  const preset = Object.assign({}, data);
+
+  if (!preset.precondition) preset.precondition = "";
+  // Try parse to check expression validity
+  parse(preset.precondition as string);
+
+  preset.weight = parseInt(preset.weight as string) || 0;
+
+  const events = {};
+  if (preset.events) {
+    for (let e of (preset.events as string).split(",")) {
+      let v = true;
+      e = e.trim();
+      if (e.startsWith("-")) {
+        v = false;
+        e = e.slice(1).trim();
+      }
+      if (e) events[e] = v;
+    }
+  }
+
+  preset.events = events;
+
+  if (!preset.provision) throw new Error("Invalid preset provision");
+
+  const configuration = {
+    type: "provision",
+    name: preset.provision,
+    args: null,
+  };
+
+  if (preset.provisionArgs)
+    configuration.args = parseList(preset.provisionArgs as string);
+
+  delete preset.provision;
+  delete preset.provisionArgs;
+  preset.configurations = [configuration];
+  return preset;
+}

lib/nbi.coffee

@@ -1,516 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-url = require 'url'
-mongodb = require 'mongodb'
-querystring = require 'querystring'
-vm = require 'vm'
-
-config = require './config'
-common = require './common'
-db = require './db'
-query = require './query'
-apiFunctions = require './api-functions'
-
-VERSION = require('../package.json').version
-
-# regular expression objects
-DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/tasks\/?$/
-TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-\_\%]+)(\/[a-zA-Z_]*)?$/
-TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/tags\/([a-zA-Z0-9\-\_\%]+)\/?$/
-PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-\_\%]+)\/?$/
-OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-\_\%]+)\/?$/
-FILES_REGEX = /^\/files\/([a-zA-Z0-9\%\!\*\'\(\)\;\:\@\&\=\+\$\,\?\#\[\]\-\_\.\~]+)\/?$/
-PING_REGEX = /^\/ping\/([a-zA-Z0-9\-\_\.]+)\/?$/
-QUERY_REGEX = /^\/([a-zA-Z0-9_]+s)\/?$/
-DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/?$/
-PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-\_\%]+)\/?$/
-VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-\_\%]+)\/?$/
-FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-\_\%\:]+)\/?$/
-
-
-
-throwError = (err, httpResponse) ->
-  if httpResponse
-    httpResponse.writeHead(500, {'Connection' : 'close'})
-    httpResponse.end("#{err.name}: #{err.message}")
-
-  throw err
-
-
-listener = (request, response) ->
-  chunks = []
-  bytes = 0
-
-  response.setHeader('GenieACS-Version', VERSION)
-
-  request.addListener 'data', (chunk) ->
-    chunks.push(chunk)
-    bytes += chunk.length
-
-  request.getBody = () ->
-    # Write all chunks into a Buffer
-    body = new Buffer(bytes)
-    offset = 0
-    chunks.forEach((chunk) ->
-      chunk.copy(body, offset, 0, chunk.length)
-      offset += chunk.length
-    )
-    return body
-
-  request.addListener 'end', () ->
-    body = request.getBody()
-    urlParts = url.parse(request.url, true)
-    if PRESETS_REGEX.test(urlParts.pathname)
-      presetName = querystring.unescape(PRESETS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        preset = JSON.parse(body)
-        preset._id = presetName
-
-        db.presetsCollection.save(preset, (err) ->
-          return throwError(err, response) if err
-          db.redisClient.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.presetsCollection.remove({'_id' : presetName}, (err) ->
-          return throwError(err, response) if err
-          db.redisClient.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if OBJECTS_REGEX.test(urlParts.pathname)
-      objectName = querystring.unescape(OBJECTS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        object = JSON.parse(body)
-        object._id = objectName
-
-        db.objectsCollection.save(object, (err) ->
-          return throwError(err, response) if err
-          db.redisClient.del('objects', 'presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.objectsCollection.remove({'_id' : objectName}, (err) ->
-          return throwError(err, response) if err
-          db.redisClient.del('objects', 'presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if PROVISIONS_REGEX.test(urlParts.pathname)
-      provisionName = querystring.unescape(PROVISIONS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        object = {
-          _id: provisionName
-          script: body.toString()
-        }
-        try
-          new vm.Script("\"use strict\";(function(){\n#{object.script}\n})();")
-        catch err
-          response.writeHead(400)
-          response.end("#{err.name}: #{err.message}")
-          return
-
-        db.provisionsCollection.save(object, (err) ->
-          return throwError(err, response) if err
-          db.redisClient.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.provisionsCollection.remove({'_id' : provisionName}, (err) ->
-          return throwError(err, response) if err
-          db.redisClient.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if VIRTUAL_PARAMETERS_REGEX.test(urlParts.pathname)
-      virtualParameterName = querystring.unescape(VIRTUAL_PARAMETERS_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        object = {
-          _id: virtualParameterName
-          script: body.toString()
-        }
-        try
-          new vm.Script("\"use strict\";(function(){\n#{object.script}\n})();")
-        catch err
-          response.writeHead(400)
-          response.end("#{err.name}: #{err.message}")
-          return
-
-        db.virtualParametersCollection.save(object, (err) ->
-          return throwError(err, response) if err
-          db.redisClient.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.virtualParametersCollection.remove({'_id' : virtualParameterName}, (err) ->
-          return throwError(err, response) if err
-          db.redisClient.del('presets_hash', (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if TAGS_REGEX.test(urlParts.pathname)
-      r = TAGS_REGEX.exec(urlParts.pathname)
-      deviceId = querystring.unescape(r[1])
-      tag = querystring.unescape(r[2])
-      if request.method == 'POST'
-        db.devicesCollection.update({'_id' : deviceId}, {'$addToSet' : {'_tags' : tag}}, {safe: true}, (err) ->
-          return throwError(err, response) if err
-          db.redisClient.del("#{deviceId}_presets_hash", (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else if request.method == 'DELETE'
-        db.devicesCollection.update({'_id' : deviceId}, {'$pull' : {'_tags' : tag}}, {safe: true}, (err) ->
-          return throwError(err, response) if err
-          db.redisClient.del("#{deviceId}_presets_hash", (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'POST, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if FAULTS_REGEX.test(urlParts.pathname)
-      if request.method == 'DELETE'
-        faultId = querystring.unescape(FAULTS_REGEX.exec(urlParts.pathname)[1])
-        deviceId = faultId.split(':', 1)[0]
-        channel = faultId.slice(deviceId.length + 1)
-        db.faultsCollection.remove({_id : faultId}, (err) ->
-          return throwError(err, response) if err
-          if channel.startsWith('task_')
-            return db.tasksCollection.remove({_id : new mongodb.ObjectID(channel.slice(5))}, (err) ->
-              return throwError(err, response) if err
-              db.redisClient.del("#{deviceId}_tasks", "#{deviceId}_faults", (err) ->
-                return throwError(err, response) if err
-                response.writeHead(200)
-                response.end()
-              )
-            )
-
-          db.redisClient.del("#{deviceId}_faults", (err) ->
-            return throwError(err, response) if err
-            response.writeHead(200)
-            response.end()
-          )
-        )
-      else
-        response.writeHead 405, {'Allow': 'DELETE'}
-        response.end('405 Method Not Allowed')
-    else if DEVICE_TASKS_REGEX.test(urlParts.pathname)
-      if request.method == 'POST'
-        deviceId = querystring.unescape(DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1])
-        if body.length
-          task = JSON.parse(body)
-          task.device = deviceId
-          apiFunctions.insertTasks(task, (err) ->
-            return throwError(err, response) if err
-            db.redisClient.del("#{deviceId}_tasks", (err) ->
-              return throwError(err, response) if err
-
-              if urlParts.query.connection_request?
-                apiFunctions.connectionRequest(deviceId, (err) ->
-                  if err
-                    response.writeHead(202, err.message, {'Content-Type' : 'application/json'})
-                    response.end(JSON.stringify(task))
-                  else
-                    apiFunctions.watchTask(deviceId, task._id, config.get('DEVICE_ONLINE_THRESHOLD', deviceId), (err, status) ->
-                      return throwError(err, response) if err
-
-                      if status is 'timeout'
-                        response.writeHead(202, 'Task queued but not processed', {'Content-Type' : 'application/json'})
-                        response.end(JSON.stringify(task))
-                      else if status is 'fault'
-                        db.tasksCollection.findOne({_id : task._id}, (err, task) ->
-                          return throwError(err, response) if err
-
-                          response.writeHead(202, 'Task faulted', {'Content-Type' : 'application/json'})
-                          response.end(JSON.stringify(task))
-                        )
-                      else
-                        response.writeHead(200, {'Content-Type' : 'application/json'})
-                        response.end(JSON.stringify(task))
-                    )
-                )
-              else
-                response.writeHead(202, {'Content-Type' : 'application/json'})
-                response.end(JSON.stringify(task))
-            )
-          )
-        else if urlParts.query.connection_request?
-          # no task, send connection request only
-          apiFunctions.connectionRequest(deviceId, (err) ->
-            if err
-              response.writeHead 504
-              response.end("#{err.name}: #{err.message}")
-              return
-            response.writeHead 200
-            response.end()
-          )
-        else
-          response.writeHead(400)
-          response.end()
-      else
-        response.writeHead 405, {'Allow': 'POST'}
-        response.end('405 Method Not Allowed')
-    else if TASKS_REGEX.test(urlParts.pathname)
-      r = TASKS_REGEX.exec(urlParts.pathname)
-      taskId = new mongodb.ObjectID(querystring.unescape(r[1]))
-      action = r[2]
-      if not action? or action is '/'
-        if request.method == 'DELETE'
-          db.tasksCollection.findOne({'_id' : taskId}, {'device' : 1}, (err, task) ->
-            return throwError(err, response) if err
-            deviceId = task.device
-            db.tasksCollection.remove({'_id' : taskId}, (err) ->
-              return throwError(err, response) if err
-              db.faultsCollection.remove({_id : "#{deviceId}:task_#{String(taskId)}"}, (err) ->
-                return throwError(err, response) if err
-                db.redisClient.del("#{deviceId}_tasks", "#{deviceId}_faults", (err) ->
-                  return throwError(err, response) if err
-                  response.writeHead(200)
-                  response.end()
-                )
-              )
-            )
-          )
-        else
-          response.writeHead 405, {'Allow': 'PUT DELETE'}
-          response.end('405 Method Not Allowed')
-      else if action is '/retry'
-        if request.method == 'POST'
-          db.tasksCollection.findOne({'_id' : taskId}, {'device' : 1}, (err, task) ->
-            return throwError(err, response) if err
-            deviceId = task.device
-            db.faultsCollection.remove({_id : "#{deviceId}:task_#{String(taskId)}"}, (err) ->
-              return throwError(err, response) if err
-              db.redisClient.del("#{deviceId}_faults", (err) ->
-                return throwError(err, response) if err
-                response.writeHead(200)
-                response.end()
-              )
-            )
-          )
-        else
-          response.writeHead 405, {'Allow': 'POST'}
-          response.end('405 Method Not Allowed')
-      else
-        response.writeHead(404)
-        response.end()
-    else if FILES_REGEX.test(urlParts.pathname)
-      filename = querystring.unescape(FILES_REGEX.exec(urlParts.pathname)[1])
-      if request.method == 'PUT'
-        metadata = {
-          fileType : request.headers.filetype,
-          oui : request.headers.oui,
-          productClass : request.headers.productclass,
-          version : request.headers.version,
-        }
-
-        gs = new mongodb.GridStore(db.mongoDb, filename, filename, 'w', {metadata : metadata})
-        gs.open((err, gs) ->
-          gs.write(body, (err, res) ->
-            return throwError(err, response) if err
-            gs.close((err) ->
-              return throwError(err, response) if err
-              response.writeHead(201)
-              response.end()
-            )
-          )
-        )
-      else if request.method == 'DELETE'
-        mongodb.GridStore.unlink(db.mongoDb, filename, (err) ->
-          response.writeHead(200)
-          response.end()
-        )
-      else
-        response.writeHead 405, {'Allow': 'PUT, DELETE'}
-        response.end('405 Method Not Allowed')
-    else if PING_REGEX.test(urlParts.pathname)
-      host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1])
-      require('child_process').exec("ping -w 1 -i 0.2 -c 3 #{host}", (err, stdout, stderr) ->
-        if err
-          response.writeHead(404, {'Cache-Control' : 'no-cache'})
-          response.end("#{err.name}: #{err.message}")
-          return
-        response.writeHead(200, {'Content-Type' : 'text/plain', 'Cache-Control' : 'no-cache'})
-        response.end(stdout)
-      )
-    else if DELETE_DEVICE_REGEX.test(urlParts.pathname)
-      if request.method isnt 'DELETE'
-        response.writeHead(405, {'Allow' : 'DELETE'})
-        response.end('405 Method Not Allowed')
-        return
-
-      deviceId = querystring.unescape(DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1])
-      apiFunctions.deleteDevice(deviceId, (err) ->
-        return throwError(err, response) if err
-        response.writeHead(200)
-        response.end()
-      )
-    else if QUERY_REGEX.test(urlParts.pathname)
-      collectionName = QUERY_REGEX.exec(urlParts.pathname)[1]
-
-      # Convert to camel case
-      i = collectionName.indexOf('_')
-      while i >= 0
-        ++ i
-        up = if i < collectionName.length then collectionName[i].toUpperCase() else ''
-        collectionName = collectionName.slice(0, i - 1) + up + collectionName.slice(i + 1)
-        i = collectionName.indexOf('_', i)
-
-      if request.method not in ['GET', 'HEAD']
-        response.writeHead 405, {'Allow' : 'GET, HEAD'}
-        response.end('405 Method Not Allowed')
-        return
-      collection = db["#{collectionName}Collection"]
-      if not collection?
-        response.writeHead 404
-        response.end('404 Not Found')
-        return
-
-      if urlParts.query.query?
-        try
-          q = JSON.parse(urlParts.query.query)
-        catch err
-          response.writeHead(400)
-          response.end("#{err.name}: #{err.message}")
-          return
-      else
-        q = {}
-
-      switch collectionName
-        when 'devices'
-          q = query.expand(q)
-        when 'tasks'
-          q = query.sanitizeQueryTypes(q, {
-            _id: ((v) -> return new mongodb.ObjectID(v))
-            timestamp: ((v) -> return new Date(v))
-            retries: Number
-          })
-        when 'faults'
-          q = query.sanitizeQueryTypes(q, {
-            timestamp: ((v) -> return new Date(v))
-            retries: Number
-          })
-
-      if urlParts.query.projection?
-        projection = {}
-        for p in urlParts.query.projection.split(',')
-          p = p.trim()
-          projection[p] = 1
-
-      cur = collection.find(q, projection, {batchSize : 50})
-      if urlParts.query.sort?
-        s = JSON.parse(urlParts.query.sort)
-        sort = {}
-        for k,v of s
-          if k[k.lastIndexOf('.') + 1] != '_' and collectionName is 'devices'
-            sort["#{k}._value"] = v
-          else
-            sort[k] = v
-        cur.sort(sort)
-
-      cur.skip(parseInt(urlParts.query.skip)) if urlParts.query.skip?
-      cur.limit(limit = parseInt(urlParts.query.limit)) if urlParts.query.limit?
-      cur.count(false, (err, total) ->
-        response.writeHead(200, {'Content-Type' : 'application/json', 'total' : total})
-        if request.method is 'HEAD'
-          response.end()
-          return
-        response.write("[\n")
-        i = 0
-        cur.each((err, item) ->
-          if err
-            throwError(err, response)
-            return false
-
-          if item?
-            response.write(",\n") if i++
-            response.write(JSON.stringify(item))
-
-          if not item? or (limit? and i >= limit)
-            response.end("\n]")
-        )
-      )
-
-    else
-      response.writeHead 404
-      response.end('404 Not Found')
-
-
-exports.listener = listener

lib/nbi.ts

@@ -0,0 +1,827 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as url from "url";
+import { Collection, GridFSBucket, ObjectId } from "mongodb";
+import * as querystring from "querystring";
+import * as vm from "vm";
+import * as config from "./config";
+import { onConnect } from "./db";
+import * as query from "./query";
+import * as apiFunctions from "./api-functions";
+import { IncomingMessage, ServerResponse } from "http";
+import * as cache from "./cache";
+import { version as VERSION } from "../package.json";
+import { ping } from "./ping";
+import * as logger from "./logger";
+
+const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
+const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
+const TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tags\/([a-zA-Z0-9\-_%]+)\/?$/;
+const PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-_%]+)\/?$/;
+const OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-_%]+)\/?$/;
+const FILES_REGEX = /^\/files\/([a-zA-Z0-9%!*'();:@&=+$,?#[\]\-_.~]+)\/?$/;
+const PING_REGEX = /^\/ping\/([a-zA-Z0-9\-_.:]+)\/?$/;
+const QUERY_REGEX = /^\/([a-zA-Z0-9_]+)\/?$/;
+const DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/?$/;
+const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
+const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
+const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
+
+const collections = {
+  tasks: null as Collection,
+  devices: null as Collection,
+  presets: null as Collection,
+  objects: null as Collection,
+  files: null as Collection,
+  provisions: null as Collection,
+  virtualParameters: null as Collection,
+  faults: null as Collection,
+};
+let filesBucket: GridFSBucket;
+
+onConnect(async (db) => {
+  for (const k of Object.keys(collections)) {
+    if (k === "files") collections[k] = db.collection("fs.files");
+    else collections[k] = db.collection(k);
+  }
+  filesBucket = new GridFSBucket(db);
+});
+
+function throwError(err: Error, httpResponse?: ServerResponse): never {
+  if (httpResponse) {
+    httpResponse.writeHead(500, { Connection: "close" });
+    httpResponse.end(`${err.name}: ${err.message}`);
+  }
+  throw err;
+}
+
+export function listener(
+  request: IncomingMessage,
+  response: ServerResponse
+): void {
+  const chunks = [];
+  let bytes = 0;
+  response.setHeader("GenieACS-Version", VERSION);
+
+  request.addListener("data", (chunk): void => {
+    chunks.push(chunk);
+    bytes += chunk.length;
+  });
+
+  function getBody(): Buffer {
+    // Write all chunks into a Buffer
+    const body = Buffer.allocUnsafe(bytes);
+    let offset = 0;
+    for (const chunk of chunks) {
+      chunk.copy(body, offset, 0, chunk.length);
+      offset += chunk.length;
+    }
+    return body;
+  }
+
+  request.addListener("end", () => {
+    const body = getBody();
+    const urlParts = url.parse(request.url, true);
+
+    logger.accessInfo(
+      Object.assign({}, urlParts.query, {
+        remoteAddress: request.connection.remoteAddress,
+        message: `${request.method} ${urlParts.pathname}`,
+      })
+    );
+
+    if (PRESETS_REGEX.test(urlParts.pathname)) {
+      const presetName = querystring.unescape(
+        PRESETS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const preset = JSON.parse(body.toString());
+        preset._id = presetName;
+        collections.presets.replaceOne(
+          { _id: presetName },
+          preset,
+          { upsert: true },
+          (err) => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch((err) => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        collections.presets.deleteOne({ _id: presetName }, (err) => {
+          if (err) return void throwError(err, response);
+
+          cache
+            .del("presets_hash")
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch((err) => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (OBJECTS_REGEX.test(urlParts.pathname)) {
+      const objectName = querystring.unescape(
+        OBJECTS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const object = JSON.parse(body.toString());
+        object._id = objectName;
+        collections.objects.replaceOne(
+          { _id: objectName },
+          object,
+          { upsert: true },
+          (err) => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch((err) => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        collections.objects.deleteOne({ _id: objectName }, (err) => {
+          if (err) return void throwError(err, response);
+
+          cache
+            .del("presets_hash")
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch((err) => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (PROVISIONS_REGEX.test(urlParts.pathname)) {
+      const provisionName = querystring.unescape(
+        PROVISIONS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const object = {
+          _id: provisionName,
+          script: body.toString(),
+        };
+
+        try {
+          new vm.Script(`"use strict";(function(){\n${object.script}\n})();`);
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+
+        collections.provisions.replaceOne(
+          { _id: provisionName },
+          object,
+          { upsert: true },
+          (err) => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch((err) => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        collections.provisions.deleteOne({ _id: provisionName }, (err) => {
+          if (err) return void throwError(err, response);
+
+          cache
+            .del("presets_hash")
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch((err) => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (VIRTUAL_PARAMETERS_REGEX.test(urlParts.pathname)) {
+      const virtualParameterName = querystring.unescape(
+        VIRTUAL_PARAMETERS_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const object = {
+          _id: virtualParameterName,
+          script: body.toString(),
+        };
+
+        try {
+          new vm.Script(`"use strict";(function(){\n${object.script}\n})();`);
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+
+        collections.virtualParameters.replaceOne(
+          { _id: virtualParameterName },
+          object,
+          { upsert: true },
+          (err) => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch((err) => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else if (request.method === "DELETE") {
+        collections.virtualParameters.deleteOne(
+          { _id: virtualParameterName },
+          (err) => {
+            if (err) return void throwError(err, response);
+
+            cache
+              .del("presets_hash")
+              .then(() => {
+                response.writeHead(200);
+                response.end();
+              })
+              .catch((err) => {
+                setTimeout(() => {
+                  throwError(err, response);
+                });
+              });
+          }
+        );
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (TAGS_REGEX.test(urlParts.pathname)) {
+      const r = TAGS_REGEX.exec(urlParts.pathname);
+      const deviceId = querystring.unescape(r[1]);
+      const tag = querystring.unescape(r[2]);
+      if (request.method === "POST") {
+        collections.devices.updateOne(
+          { _id: deviceId },
+          { $addToSet: { _tags: tag } },
+          (err) => {
+            if (err) return void throwError(err, response);
+            response.writeHead(200);
+            response.end();
+          }
+        );
+      } else if (request.method === "DELETE") {
+        collections.devices.updateOne(
+          { _id: deviceId },
+          { $pull: { _tags: tag } },
+          (err) => {
+            if (err) return void throwError(err, response);
+
+            response.writeHead(200);
+            response.end();
+          }
+        );
+      } else {
+        response.writeHead(405, { Allow: "POST, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (FAULTS_REGEX.test(urlParts.pathname)) {
+      if (request.method === "DELETE") {
+        const faultId = querystring.unescape(
+          FAULTS_REGEX.exec(urlParts.pathname)[1]
+        );
+        const deviceId = faultId.split(":", 1)[0];
+        const channel = faultId.slice(deviceId.length + 1);
+        collections.faults.deleteOne({ _id: faultId }, (err) => {
+          if (err) return void throwError(err, response);
+
+          if (channel.startsWith("task_")) {
+            const objId = new ObjectId(channel.slice(5));
+            return void collections.tasks.deleteOne({ _id: objId }, (err) => {
+              if (err) return void throwError(err, response);
+
+              cache
+                .del(`${deviceId}_tasks_faults_operations`)
+                .then(() => {
+                  response.writeHead(200);
+                  response.end();
+                })
+                .catch((err) => {
+                  setTimeout(() => {
+                    throwError(err, response);
+                  });
+                });
+            });
+          }
+
+          cache
+            .del(`${deviceId}_tasks_faults_operations`)
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch((err) => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        });
+      } else {
+        response.writeHead(405, { Allow: "DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (DEVICE_TASKS_REGEX.test(urlParts.pathname)) {
+      if (request.method === "POST") {
+        const deviceId = querystring.unescape(
+          DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1]
+        );
+        if (body.length) {
+          const task = JSON.parse(body.toString());
+          task.device = deviceId;
+          apiFunctions
+            .insertTasks(task)
+            .then(async () => {
+              const lastInform = Date.now();
+
+              const socketTimeout: number = request.socket["timeout"];
+
+              // Disable socket timeout while waiting for session
+              if (socketTimeout) request.socket.setTimeout(0);
+
+              const notInSession = await apiFunctions.awaitSessionEnd(
+                deviceId,
+                30000
+              );
+              await cache.del(`${deviceId}_tasks_faults_operations`);
+              if (urlParts.query.connection_request == null) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              if (!notInSession) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const status = await apiFunctions.connectionRequest(deviceId);
+
+              if (status) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, status, {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const onlineThreshold =
+                (urlParts.query.timeout &&
+                  parseInt(urlParts.query.timeout as string)) ||
+                (config.get("DEVICE_ONLINE_THRESHOLD", deviceId) as number);
+
+              const sessionStarted = await apiFunctions.awaitSessionStart(
+                deviceId,
+                lastInform,
+                onlineThreshold
+              );
+              if (!sessionStarted) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const sessionEnded = await apiFunctions.awaitSessionEnd(
+                deviceId,
+                120000
+              );
+              if (!sessionEnded) {
+                if (socketTimeout) request.socket.setTimeout(socketTimeout);
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+                return;
+              }
+
+              const prom1 = collections.tasks.findOne(
+                { _id: task._id },
+                { projection: { _id: 1 } }
+              );
+              const prom2 = collections.faults.findOne(
+                { _id: `${deviceId}:task_${task._id}` },
+                {
+                  projection: { _id: 1 },
+                }
+              );
+
+              const [t, f] = await Promise.all([prom1, prom2]);
+
+              // Restore socket timeout
+              if (socketTimeout) request.socket.setTimeout(socketTimeout);
+
+              if (f) {
+                response.writeHead(202, "Task faulted", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(t || task));
+              } else if (t) {
+                response.writeHead(202, "Task queued but not processed", {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(t));
+              } else {
+                response.writeHead(200, {
+                  "Content-Type": "application/json",
+                });
+                response.end(JSON.stringify(task));
+              }
+            })
+            .catch((err) => {
+              setTimeout(() => {
+                throwError(err, response);
+              });
+            });
+        } else if (urlParts.query.connection_request != null) {
+          // No task, send connection request only
+          apiFunctions
+            .connectionRequest(deviceId)
+            .then(() => {
+              response.writeHead(200);
+              response.end();
+            })
+            .catch((err) => {
+              response.writeHead(504);
+              response.end(`${err.name}: ${err.message}`);
+            });
+        } else {
+          response.writeHead(400);
+          response.end();
+        }
+      } else {
+        response.writeHead(405, { Allow: "POST" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (TASKS_REGEX.test(urlParts.pathname)) {
+      const r = TASKS_REGEX.exec(urlParts.pathname);
+      const taskId = querystring.unescape(r[1]);
+      const action = r[2];
+      if (!action || action === "/") {
+        if (request.method === "DELETE") {
+          collections.tasks.findOne(
+            { _id: new ObjectId(taskId) },
+            { projection: { device: 1 } },
+            (err, task) => {
+              if (err) return void throwError(err, response);
+
+              if (!task) {
+                response.writeHead(404);
+                response.end("Task not found");
+                return;
+              }
+
+              const deviceId = task.device;
+              collections.tasks.deleteOne(
+                { _id: new ObjectId(taskId) },
+                (err) => {
+                  if (err) return void throwError(err, response);
+
+                  collections.faults.deleteOne(
+                    { _id: `${deviceId}:task_${taskId}` },
+                    (err) => {
+                      if (err) return void throwError(err, response);
+
+                      cache
+                        .del(`${deviceId}_tasks_faults_operations`)
+                        .then(() => {
+                          response.writeHead(200);
+                          response.end();
+                        })
+                        .catch((err) => {
+                          setTimeout(() => {
+                            throwError(err, response);
+                          });
+                        });
+                    }
+                  );
+                }
+              );
+            }
+          );
+        } else {
+          response.writeHead(405, { Allow: "PUT DELETE" });
+          response.end("405 Method Not Allowed");
+        }
+      } else if (action === "/retry") {
+        if (request.method === "POST") {
+          collections.tasks.findOne(
+            { _id: new ObjectId(taskId) },
+            { projection: { device: 1 } },
+            (err, task) => {
+              if (err) return void throwError(err, response);
+
+              const deviceId = task.device;
+              collections.faults.deleteOne(
+                { _id: `${deviceId}:task_${taskId}` },
+                (err) => {
+                  if (err) return void throwError(err, response);
+
+                  cache
+                    .del(`${deviceId}_tasks_faults_operations`)
+                    .then(() => {
+                      response.writeHead(200);
+                      response.end();
+                    })
+                    .catch((err) => {
+                      setTimeout(() => {
+                        throwError(err, response);
+                      });
+                    });
+                }
+              );
+            }
+          );
+        } else {
+          response.writeHead(405, { Allow: "POST" });
+          response.end("405 Method Not Allowed");
+        }
+      } else {
+        response.writeHead(404);
+        response.end();
+      }
+    } else if (FILES_REGEX.test(urlParts.pathname)) {
+      const filename = querystring.unescape(
+        FILES_REGEX.exec(urlParts.pathname)[1]
+      );
+      if (request.method === "PUT") {
+        const metadata = {
+          fileType: request.headers.filetype,
+          oui: request.headers.oui,
+          productClass: request.headers.productclass,
+          version: request.headers.version,
+        };
+        filesBucket.delete((filename as unknown) as ObjectId, () => {
+          const uploadStream = filesBucket.openUploadStreamWithId(
+            filename,
+            filename,
+            {
+              metadata: metadata,
+            }
+          );
+
+          uploadStream.on("error", (err) => {
+            throwError(err, response);
+          });
+
+          uploadStream.end(body, () => {
+            response.writeHead(201);
+            response.end();
+          });
+        });
+      } else if (request.method === "DELETE") {
+        filesBucket.delete((filename as unknown) as ObjectId, (err) => {
+          if (err) {
+            if (err.message.startsWith("FileNotFound")) {
+              response.writeHead(404);
+              response.end("404 Not Found");
+              return;
+            }
+            return void throwError(err, response);
+          }
+
+          response.writeHead(200);
+          response.end();
+        });
+      } else {
+        response.writeHead(405, { Allow: "PUT, DELETE" });
+        response.end("405 Method Not Allowed");
+      }
+    } else if (PING_REGEX.test(urlParts.pathname)) {
+      const host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1]);
+      ping(host, (err, res, stdout) => {
+        if (err) {
+          if (!res) {
+            response.writeHead(500, { Connection: "close" });
+            response.end(`${err.name}: ${err.message}`);
+            return;
+          }
+          response.writeHead(404, { "Cache-Control": "no-cache" });
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+
+        response.writeHead(200, {
+          "Content-Type": "text/plain",
+          "Cache-Control": "no-cache",
+        });
+        response.end(stdout);
+      });
+    } else if (DELETE_DEVICE_REGEX.test(urlParts.pathname)) {
+      if (request.method !== "DELETE") {
+        response.writeHead(405, { Allow: "DELETE" });
+        response.end("405 Method Not Allowed");
+        return;
+      }
+
+      const deviceId = querystring.unescape(
+        DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1]
+      );
+      apiFunctions
+        .deleteDevice(deviceId)
+        .then(() => {
+          response.writeHead(200);
+          response.end();
+        })
+        .catch((err) => {
+          setTimeout(() => {
+            throwError(err, response);
+          });
+        });
+    } else if (QUERY_REGEX.test(urlParts.pathname)) {
+      let collectionName = QUERY_REGEX.exec(urlParts.pathname)[1];
+
+      // Convert to camel case
+      let i = collectionName.indexOf("_");
+      while (i++ >= 0) {
+        const up =
+          i < collectionName.length ? collectionName[i].toUpperCase() : "";
+        collectionName =
+          collectionName.slice(0, i - 1) + up + collectionName.slice(i + 1);
+        i = collectionName.indexOf("_", i);
+      }
+
+      if (request.method !== "GET" && request.method !== "HEAD") {
+        response.writeHead(405, { Allow: "GET, HEAD" });
+        response.end("405 Method Not Allowed");
+        return;
+      }
+
+      const collection = collections[collectionName];
+      if (!collection) {
+        response.writeHead(404);
+        response.end("404 Not Found");
+        return;
+      }
+
+      let q = {};
+      if (urlParts.query.query) {
+        try {
+          q = JSON.parse(urlParts.query.query as string);
+        } catch (err) {
+          response.writeHead(400);
+          response.end(`${err.name}: ${err.message}`);
+          return;
+        }
+      }
+
+      switch (collectionName) {
+        case "devices":
+          q = query.expand(q);
+          break;
+        case "tasks":
+          q = query.sanitizeQueryTypes(q, {
+            _id: (v) => new ObjectId(v as string),
+            timestamp: (v) => new Date(v as number),
+            retries: Number,
+          });
+          break;
+        case "faults":
+          q = query.sanitizeQueryTypes(q, {
+            timestamp: (v) => new Date(v as number),
+            retries: Number,
+          });
+      }
+
+      let projection = null;
+      if (urlParts.query.projection) {
+        projection = {};
+        for (const p of (urlParts.query.projection as string).split(","))
+          projection[p.trim()] = 1;
+      }
+
+      const cur = collection.find(q, { projection: projection });
+
+      if (urlParts.query.sort) {
+        const s = JSON.parse(urlParts.query.sort as string);
+        const sort = {};
+        for (const [k, v] of Object.entries(s)) {
+          if (k[k.lastIndexOf(".") + 1] !== "_" && collectionName === "devices")
+            sort[`${k}._value`] = v;
+          else sort[k] = v;
+        }
+
+        cur.sort(sort);
+      }
+
+      if (urlParts.query.skip)
+        cur.skip(parseInt(urlParts.query.skip as string));
+
+      if (urlParts.query.limit)
+        cur.limit(parseInt(urlParts.query.limit as string));
+
+      cur.count(false, (err, total) => {
+        if (err) return void throwError(err);
+
+        response.writeHead(200, {
+          "Content-Type": "application/json",
+          total: total,
+        });
+
+        if (request.method === "HEAD") {
+          response.end();
+          return;
+        }
+
+        response.write("[\n");
+        i = 0;
+        cur.forEach(
+          (item) => {
+            if (i++) response.write(",\n");
+            response.write(JSON.stringify(item));
+          },
+          (err) => {
+            if (err) return void throwError(err);
+            response.end("\n]");
+          }
+        );
+      });
+    } else {
+      response.writeHead(404);
+      response.end("404 Not Found");
+    }
+  });
+}

lib/path-set.js

@@ -1,179 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-
-class PathSet {
-
-  constructor() {
-    this.lengthIndex = [];
-    this.fragmentIndex = [];
-  }
-
-
-  get depth() {
-    return this.lengthIndex.length;
-  }
-
-
-  add(path) {
-    let p = this.get(path);
-
-    if (p)
-      return p;
-
-    if (!this.lengthIndex[path.length])
-      this.lengthIndex[path.length] = new Set();
-
-    this.lengthIndex[path.length].add(path);
-
-    let wildcard = 0;
-    for (let i = 0; i < path.length; ++ i) {
-      let fragment = path[i];
-      if (fragment === "*")
-        wildcard = wildcard | (1 << i);
-
-      if (!this.fragmentIndex[i])
-        this.fragmentIndex[i] = {};
-
-      if (!this.fragmentIndex[i][fragment])
-        this.fragmentIndex[i][fragment] = new Set();
-
-      this.fragmentIndex[i][fragment].add(path);
-    }
-
-    if (path.wildcard == null) {
-      path.alias = 0;
-      path.wildcard = wildcard;
-      Object.freeze(path);
-    }
-    return path;
-  }
-
-
-  get(pattern) {
-    if (!this.lengthIndex[pattern.length])
-      return null;
-
-    if (this.lengthIndex[pattern.length].has(pattern))
-      return pattern;
-
-    let sets = [];
-
-    sets.push(this.lengthIndex[pattern.length]);
-
-    for (let i = 0; i < pattern.length; ++ i) {
-      if (!this.fragmentIndex[i])
-        return null;
-
-      let fragment = pattern[i];
-
-      if (!this.fragmentIndex[i][fragment])
-        return null;
-
-      sets.push(this.fragmentIndex[i][fragment]);
-    }
-
-    sets.sort(function(a, b) {
-      return a.size - b.size;
-    });
-
-    let smallestSet = sets.shift();
-    for (let path of smallestSet) {
-      for (let s of sets) {
-        if (!s.has(path)) {
-          path = null;
-          break
-        }
-      }
-      if (path)
-        return path;
-    }
-
-    return null;
-  }
-
-
-  find(pattern, superset = false, subset = false, depth = pattern.length) {
-    let res = [];
-    let groups = [[]];
-
-    for (let i = pattern.length; i <= depth; ++ i)
-      if (this.lengthIndex[i])
-        groups[0].push(this.lengthIndex[i]);
-
-    if (!groups[0].length)
-      return res;
-
-    for (let i = 0; i < pattern.length; ++ i) {
-      if (!this.fragmentIndex[i])
-        return res;
-
-      let fragment = pattern[i];
-
-      if (fragment === "*" && subset)
-        continue;
-
-      let g = [];
-
-      if (this.fragmentIndex[i][fragment])
-        g.push(this.fragmentIndex[i][fragment]);
-
-      if (fragment !== "*" && superset && this.fragmentIndex[i]["*"])
-        g.push(this.fragmentIndex[i]["*"]);
-
-      if (!g.length)
-        return res;
-
-      groups.push(g);
-    }
-
-    groups.sort(function(a, b) {
-      return a.reduce(function(prev, cur) {return prev + cur.size;}, 0) -
-        b.reduce(function(prev, cur) {return prev + cur.size;}, 0);
-    });
-
-    let smallestGroup = groups.shift();
-    for (let set of smallestGroup) {
-      for (let path of set) {
-        for (let group of groups) {
-          let found = false;
-          for (let s of group) {
-            if (s.has(path)) {
-              found = true;
-              break;
-            }
-          }
-          if (!found) {
-            path = null;
-            break;
-          }
-        }
-        if (path)
-          res.push(path);
-      }
-    }
-
-    return res;
-  }
-
-}
-
-
-module.exports = PathSet;

lib/ping.ts

@@ -0,0 +1,90 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { platform } from "os";
+import { exec } from "child_process";
+
+export interface PingResult {
+  packetsTransmitted: number;
+  packetsReceived: number;
+  packetLoss: number;
+  min: number;
+  avg: number;
+  max: number;
+  mdev: number;
+}
+
+export function ping(
+  host: string,
+  callback: (err: Error, res?: PingResult, stdout?: string) => void
+): void {
+  let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
+  host = host.replace("[", "").replace("]", "");
+  switch (platform()) {
+    case "linux":
+      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
+      parseRegExp1 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
+      parseRegExp2 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
+      break;
+
+    case "freebsd":
+      // Send a single packet because on FreeBSD only superuser can send
+      // packets that are only 200 ms apart.
+      cmd = `ping -t 1 -c 3 ${host}`;
+      parseRegExp1 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
+      parseRegExp2 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
+      break;
+
+    default:
+      return callback(new Error("Platform not supported"));
+  }
+
+  exec(cmd, (err, stdout) => {
+    let parsed: PingResult;
+    if (stdout) {
+      const m1 = stdout.match(parseRegExp1);
+      if (m1) {
+        parsed = {
+          packetsTransmitted: +m1[1],
+          packetsReceived: +m1[2],
+          packetLoss: +m1[3],
+          min: +m1[4],
+          avg: +m1[5],
+          max: +m1[6],
+          mdev: +m1[7],
+        };
+      } else {
+        const m2 = stdout.match(parseRegExp2);
+        if (m2) {
+          parsed = {
+            packetsTransmitted: +m2[1],
+            packetsReceived: +m2[2],
+            packetLoss: +m2[3],
+            min: null,
+            avg: null,
+            max: null,
+            mdev: null,
+          };
+        }
+      }
+    }
+
+    callback(err, parsed, stdout);
+  });
+}

lib/query.coffee

@@ -1,371 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This file incorporates work covered by the following copyright and
-# permission notice:
-#
-# Copyright 2013 Fanoos Telecom
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-###
-
-common = require './common'
-mongodb = require 'mongodb'
-
-
-stringToRegexp = (input, flags) ->
-  if (input.indexOf('*') == -1)
-    return false
-
-  output = input.replace(/[\[\]\\\^\$\.\|\?\+\(\)]/, "\\$&")
-  if output[0] == '*'
-    output = output.replace(/^\*+/g, '')
-  else
-    output = '^' + output
-
-  if output[output.length - 1] == '*'
-    output = output.replace(/\*+$/g, '')
-  else
-    output = output + '$'
-
-  output = output.replace(/[\*]/, '.*')
-
-  return new RegExp(output, flags)
-
-
-normalize = (input) ->
-  if common.typeOf(input) is common.STRING_TYPE
-    vals = [input]
-    if (m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input))
-      vals.push({'$regex' : new RegExp(m[1], m[2])})
-
-    f = parseFloat(input)
-    if not isNaN(f)
-      vals.push(f)
-
-    d = new Date(input)
-    if input.length >= 8 and d.getFullYear() > 1983
-      vals.push(d)
-
-    r = stringToRegexp(input)
-    if r isnt false
-      vals.push({'$regex' : r})
-
-    return vals
-  return input
-
-
-expandValue = (value) ->
-  if common.typeOf(value) is common.ARRAY_TYPE
-    a = []
-    for j in value
-      a = a.concat(expandValue(j))
-    return [a]
-  else if common.typeOf(value) isnt common.OBJECT_TYPE
-    n = normalize(value)
-    if common.typeOf(n) isnt common.ARRAY_TYPE
-      return [n]
-    else
-      return n
-
-  objs = []
-  indices = []
-  keys = []
-  values = []
-  for k,v of value
-    keys.push(k)
-    values.push(expandValue(v))
-    indices.push(0)
-
-  i = 0
-  while i < indices.length
-    obj = {}
-    for i in [0...keys.length]
-      obj[keys[i]] = values[i][indices[i]]
-    objs.push(obj)
-
-    for i in [0...indices.length]
-      indices[i] += 1
-      if indices[i] < values[i].length
-        break
-      indices[i] = 0
-  return objs
-
-
-permute = (param, val) ->
-  conditions = []
-
-  values = expandValue(val)
-  if param[param.lastIndexOf('.') + 1] != '_'
-    param += '._value'
-
-  for v in values
-    obj = {}
-    obj[param] = v
-    conditions.push(obj)
-
-  return conditions
-
-
-expand = (query) ->
-  new_query = {}
-  for k,v of query
-    if k[0] == '$' # operator
-      expressions = []
-      for e in v
-        expressions.push(expand(e))
-      new_query[k] = expressions
-    else
-      conditions = permute(k, v)
-      if conditions.length > 1
-        new_query['$and'] ?= []
-        if v?['$ne']?
-          for c in conditions
-            new_query['$and'].push(c)
-        else
-          new_query['$and'].push({'$or' : conditions})
-      else
-        Object.assign(new_query, conditions[0])
-
-  return new_query
-
-
-# Generate parameter projection from a query
-# If second arg is given, it's edited and returned
-queryProjection = (query, proj) ->
-  proj ?= {}
-  for k,v of query
-    if k.charAt(0) == '$' # this is a logical operator
-      for q in v
-        queryProjection(q, proj)
-    else
-      proj[k] = 1
-  return proj
-
-
-testExpressions = (params, expressions, lop) ->
-  for f in expressions
-    res = test(params, f)
-
-    switch lop
-      when '$and'
-        return false if not res
-      when '$or'
-        return true if res
-      when '$nor'
-        return false if res
-
-  return switch lop
-    when '$and' then true
-    when '$or' then false
-    when '$nor' then true
-    else throw new Error('Unknown logical operator')
-
-
-test = (params, query) ->
-  for k,v of query
-    if k.charAt(0) == '$' # this is a logical operator
-      res = testExpressions(params, v, k)
-    else
-      value = params[k]
-
-      if common.typeOf(v) isnt common.OBJECT_TYPE
-        # TODO comparing array to regex, array to array, and object to object
-        if common.typeOf(value) is common.ARRAY_TYPE
-          res = value.indexOf(v) != -1
-        else
-          if common.typeOf(v) is common.REGEXP_TYPE
-            res = v.test(value)
-          else
-            res = v == value
-      else
-        for k2,v2 of v
-          switch k2
-            when '$ne'
-              if common.typeOf(value) is common.ARRAY_TYPE
-                res = value.indexOf(v2) == -1
-              else
-                res = value != v2
-            when '$lt'
-              res = value < v2
-            when '$lte'
-              res = value <= v2
-            when '$gt'
-              res = value > v2
-            when '$gte'
-              res = value >= v2
-            when '$regex'
-              res = v2.test(value)
-            when '$in'
-              throw new Error('Operator not supported')
-            when '$nin'
-              throw new Error('Operator not supported')
-            when '$all'
-              throw new Error('Operator not supported')
-            when '$exists'
-              throw new Error('Operator not supported')
-            else
-              throw new Error('Operator not supported')
-
-    if not res
-      return false
-
-  return true
-
-
-matchType = (src, dst) ->
-  if typeof src is 'string'
-    return String(dst)
-  else if typeof src is 'number'
-    if +dst == parseFloat(dst)
-      return +dst
-    else if not isNaN(Date.parse(dst))
-      return Date.parse(dst)
-  else if typeof src is 'boolean'
-    v = String(dst).trim()
-    if v is 'true' or v is 'TRUE' or v is 'True' or v is '1'
-      return true
-    else if v is 'false' or v is 'FALSE' or v is 'False' or v is '0'
-      return false
-
-  return dst
-
-
-testFilter = (obj, filter) ->
-  for k, v of filter
-    [param, op] = k.split(/([^a-zA-Z0-9\-\_\.].*)/, 2)
-    val = matchType(obj[param], v)
-    switch op
-      when '=', undefined
-        return false if obj[param] != val
-      when '>'
-        return false if not (obj[param] > val)
-      when '>='
-        return false if not (obj[param] >= val)
-      when '<'
-        return false if not (obj[param] < val)
-      when '<='
-        return false if not (obj[param] <= val)
-      when '!'
-        return false if obj[param] == val
-      else
-        throw new Error("Unrecognized operator #{op}")
-
-  return true
-
-
-convertMongoQueryToFilters = (query, filters) ->
-  filters ?= {}
-  for k, v of query
-    if k[0] == '$'
-      if k == '$and'
-        for vv in v
-          convertMongoQueryToFilters(vv, filters)
-      else
-        throw new Error("Operator #{k} not supported")
-    else if k == '_tags'
-      if common.typeOf(v) is common.OBJECT_TYPE
-        for kk, vv of v
-          vv = vv.replace(/[^a-zA-Z0-9\-]+/g, '_')
-          switch kk
-            when '$ne'
-              filters["Tags.#{vv}!"] = true
-            else
-              throw new Error("Operator #{kk} not supported")
-      else
-        v = v.replace(/[^a-zA-Z0-9\-]+/g, '_')
-        filters["Tags.#{v}"] = true
-    else
-      switch k
-        when '_id' then k = 'DeviceID.ID'
-        when '_deviceId._Manufacturer' then k = 'DeviceID.Manufacturer'
-        when '_deviceId._OUI' then k = 'DeviceID.OUI'
-        when '_deviceId._ProductClass' then k = 'DeviceID.ProductClass'
-        when '_deviceId._SerialNumber' then k = 'DeviceID.SerialNumber'
-        when '_lastInform' then k = 'Events.Inform'
-        when '_lastBootstrap' then k = 'Events.0_BOOTSTRAP'
-        when '_lastBoot' then k = 'Events.1_BOOT'
-        when '_registered' then k = 'Events.Registered'
-
-      if common.typeOf(v) is common.OBJECT_TYPE
-        for kk, vv of v
-          switch kk
-            when '$eq'
-              filters[k] = vv
-            when '$ne'
-              filters["#{k}!"] = vv
-            when '$lt'
-              filters["#{k}<"] = vv
-            when '$lte'
-              filters["#{k}<="] = vv
-            when '$gt'
-              filters["#{k}>"] = vv
-            when '$gte'
-              filters["#{k}>="] = vv
-            else
-              throw new Error("Oprator #{kk} not supported")
-      else
-        filters[k] = v
-
-  return filters
-
-
-sanitizeQueryTypes = (query, types) ->
-  for k, v of query
-    if k[0] == '$' # logical operator
-      sanitizeQueryTypes(vv, types) for vv in v
-    else if k of types
-      if common.typeOf(v) is common.OBJECT_TYPE
-        for kk, vv of v
-          switch kk # operator
-            when '$in', '$nin'
-              vv[i] = types[k](vv[i]) for i in [0...vv.length]
-            when '$eq', '$gt', '$gte', '$lt', '$lte', '$ne'
-              v[kk] = types[k](vv)
-            when '$exists', '$type' then
-              # ignore
-            else
-              throw new Error('Operator not supported')
-      else
-        query[k] = types[k](query[k])
-
-  return query
-
-
-exports.expand = expand
-exports.queryProjection = queryProjection
-exports.test = test
-exports.sanitizeQueryTypes = sanitizeQueryTypes
-exports.convertMongoQueryToFilters = convertMongoQueryToFilters
-exports.testFilter = testFilter

lib/query.ts

@@ -0,0 +1,175 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+function isObject(obj: any): boolean {
+  return Object.prototype.toString.call(obj) === "[object Object]";
+}
+
+function stringToRegexp(input, flags?): RegExp | false {
+  if (input.indexOf("*") === -1) return false;
+
+  let output = input.replace(/[[\]\\^$.|?+()]/, "\\$&");
+  if (output[0] === "*") output = output.replace(/^\*+/g, "");
+  else output = "^" + output;
+
+  if (output[output.length - 1] === "*") output = output.replace(/\*+$/g, "");
+  else output = output + "$";
+
+  output = output.replace(/[*]/, ".*");
+  return new RegExp(output, flags);
+}
+
+function normalize(input): any {
+  if (typeof input === "string") {
+    const vals: any = [input];
+    const m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input);
+    if (m) vals.push({ $regex: new RegExp(m[1], m[2]) });
+
+    if (+input === parseFloat(input)) vals.push(+input);
+
+    const d = new Date(input);
+    if (input.length >= 8 && d.getFullYear() > 1983) vals.push(d);
+
+    const r = stringToRegexp(input);
+    if (r !== false) vals.push({ $regex: r });
+
+    return vals;
+  }
+  return input;
+}
+
+function expandValue(value: unknown): unknown[] {
+  if (Array.isArray(value)) {
+    let a = [];
+    for (const j of value) a = a.concat(expandValue(j));
+    return [a];
+  } else if (!isObject(value)) {
+    const n = normalize(value);
+    if (!Array.isArray(n)) return [n];
+    else return n;
+  }
+
+  const objs = [];
+  const indices = [];
+  const keys = [];
+  const values = [];
+  for (const [k, v] of Object.entries(value)) {
+    keys.push(k);
+    values.push(expandValue(v));
+    indices.push(0);
+  }
+
+  let i = 0;
+  while (i < indices.length) {
+    const obj = {};
+    for (let j = 0; j < keys.length; ++j) obj[keys[j]] = values[j][indices[j]];
+    objs.push(obj);
+
+    for (i = 0; i < indices.length; ++i) {
+      indices[i] += 1;
+      if (indices[i] < values[i].length) break;
+      indices[i] = 0;
+    }
+  }
+  return objs;
+}
+
+function permute(param, val): any[] {
+  const conditions = [];
+  const values = expandValue(val);
+
+  if (param[param.lastIndexOf(".") + 1] !== "_") param += "._value";
+
+  for (const v of values) {
+    const obj = {};
+    obj[param] = v;
+    conditions.push(obj);
+  }
+
+  return conditions;
+}
+
+export function expand(
+  query: Record<string, unknown>
+): Record<string, unknown> {
+  const newQuery = {};
+  for (const [k, v] of Object.entries(query)) {
+    if (k[0] === "$") {
+      // Operator
+      newQuery[k] = (v as any[]).map((e) => expand(e));
+    } else {
+      const conditions = permute(k, v);
+      if (conditions.length > 1) {
+        newQuery["$and"] = newQuery["$and"] || [];
+        if (v && (v["$ne"] != null || v["$not"] != null)) {
+          if (Object.keys(v).length > 1)
+            throw new Error("Cannot mix $ne or $not with other operators");
+          for (const c of conditions) newQuery["$an"].push(c);
+        } else {
+          newQuery["$and"].push({ $or: conditions });
+        }
+      } else {
+        Object.assign(newQuery, conditions[0]);
+      }
+    }
+  }
+
+  return newQuery;
+}
+
+export function sanitizeQueryTypes(
+  query: Record<string, unknown>,
+  types: Record<string, (v: unknown) => unknown>
+): Record<string, unknown> {
+  for (const [k, v] of Object.entries(query)) {
+    if (k[0] === "$") {
+      // Logical operator
+      for (const vv of v as any[]) sanitizeQueryTypes(vv, types);
+    } else if (k in types) {
+      if (isObject(v)) {
+        for (const [kk, vv] of Object.entries(v)) {
+          switch (kk) {
+            case "$in":
+            case "$nin":
+              for (let i = 0; i < vv.length; ++i) vv[i] = types[k](vv[i]);
+              break;
+            case "$eq":
+            case "$gt":
+            case "$gte":
+            case "$lt":
+            case "$lte":
+            case "$ne":
+              v[kk] = types[k](vv);
+              break;
+            case "$exists":
+            case "$type":
+              // Ignore
+              break;
+            default:
+              throw new Error("Operator not supported");
+          }
+        }
+      } else {
+        query[k] = types[k](query[k]);
+      }
+    }
+  }
+
+  return query;
+}

lib/sandbox.js

@@ -1,328 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const vm = require("vm");
-
-const seedrandom = require("seedrandom");
-
-const common = require("./common");
-const device = require("./device");
-const extensions = require("./extensions");
-const logger = require("./logger");
-
-
-// Used for throwing to exit user script and commit
-const COMMIT = Symbol();
-
-// Used to execute extensions and restart
-const EXT = Symbol();
-
-const context = vm.createContext();
-
-let state;
-
-
-class SandboxDate {
-  constructor(arg) {
-    if (arguments.length)
-      return new (Function.prototype.bind.apply(Date, arguments));
-
-    return new Date(state.sessionContext.timestamp);
-  }
-
-  static now() {
-    return state.sessionContext.timestamp;
-  }
-};
-
-function random() {
-  if (!state.rng)
-    state.rng = seedrandom(sessionContext.timestamp);
-
-  return state.rng();
-}
-
-
-class ParameterWrapper {
-  constructor(path, attributes, unpacked, unpackedRevision) {
-    for (let attrName of attributes) {
-      Object.defineProperty(this, attrName, {get: function() {
-        if (state.uncommitted)
-          commit();
-
-        if (state.revision !== unpackedRevision) {
-          unpackedRevision = state.revision;
-          unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision + 1);
-        }
-
-        if (!unpacked.length)
-          return undefined;
-
-        let attr = state.sessionContext.deviceData.attributes.get(unpacked[0], state.revision + 1)[attrName];
-
-        if (!attr)
-          return undefined;
-
-        return attr[1];
-      }});
-    }
-
-    Object.defineProperty(this, "path", {get: function() {
-      if (state.uncommitted)
-        commit();
-
-      if (state.revision !== unpackedRevision) {
-        unpackedRevision = state.revision;
-        unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision + 1);
-      }
-
-      if (!unpacked.length)
-        return undefined;
-
-      return unpacked[0].join(".");
-    }});
-
-    Object.defineProperty(this, "size", {get: function() {
-      if (state.uncommitted)
-        commit();
-
-      if (state.revision !== unpackedRevision) {
-        unpackedRevision = state.revision;
-        unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision + 1);
-      }
-
-      if (!unpacked.length)
-        return undefined;
-
-      return unpacked.length;
-    }});
-
-    this[Symbol.iterator] = function*() {
-      if (state.uncommitted)
-        commit();
-
-      if (state.revision !== unpackedRevision) {
-        unpackedRevision = state.revision;
-        unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision + 1);
-      }
-
-      for (let p of unpacked)
-        yield new ParameterWrapper(p, attributes, [p], state.revision);
-    }
-  }
-}
-
-
-function declare(path, timestamps, values) {
-  state.uncommitted = true;
-  if (!timestamps)
-    timestamps = {};
-
-  if (!values)
-    values = {};
-
-  let parsedPath = common.parsePath(path);
-
-  let declaration = [parsedPath, 1];
-  let attrs = {};
-
-  for (let attrName in values) {
-    if (!timestamps[attrName])
-      timestamps[attrName] = 1;
-
-    if (attrName === "path")
-      declaration[3] = values.path;
-    else {
-      attrs[attrName] = 1;
-      declaration[4] = declaration[4] || {};
-      if (attrName === "value" && !Array.isArray(values.value))
-        declaration[4].value = [values.value];
-      else
-        declaration[4][attrName] = values[attrName];
-    }
-  }
-
-  for (let attrName in timestamps) {
-    if (attrName === "path")
-      declaration[1] = timestamps.path;
-    else {
-      attrs[attrName] = 1;
-      declaration[2] = declaration[2] || {};
-      declaration[2][attrName] = timestamps[attrName];
-    }
-  }
-
-  state.declarations.push(declaration);
-
-  return new ParameterWrapper(parsedPath, Object.keys(attrs));
-}
-
-
-function clear(path, timestamp, attributes) {
-  state.uncommitted = true;
-
-  if (state.revision === state.maxRevision)
-    state.clear.push([common.parsePath(path), timestamp, attributes]);
-}
-
-
-function commit() {
-  ++ state.revision;
-  state.uncommitted = false;
-
-  if (state.revision === state.maxRevision + 1)
-    throw COMMIT;
-  else if (state.revision > state.maxRevision + 1)
-    throw new Error("Declare function should not be called from within a try/catch block");
-}
-
-
-function ext() {
-  ++ state.extCounter;
-  let extCall = Array.from(arguments).map(String);
-  let key = `${state.revision}: ${JSON.stringify(extCall)}`;
-
-  if (key in state.sessionContext.extensionsCache)
-    return state.sessionContext.extensionsCache[key];
-
-  state.extensions[key] = extCall;
-  throw EXT;
-}
-
-
-function log(msg, meta) {
-  if (state.revision === state.maxRevision && state.extCounter >= 0) {
-    const details = Object.assign({}, meta, {
-      sessionContext: state.sessionContext,
-      message: `Script: ${msg}`
-    });
-
-    delete details.hostname;
-    delete details.pid;
-    delete details.name;
-    delete details.version;
-    delete details.deviceId;
-    delete details.remoteAddress;
-
-    logger.accessInfo(details);
-  }
-}
-
-
-Object.defineProperty(context, "Date", {value: SandboxDate});
-Object.defineProperty(context, "declare", {value: declare});
-Object.defineProperty(context, "clear", {value: clear});
-Object.defineProperty(context, "commit", {value: commit});
-Object.defineProperty(context, "ext", {value: ext});
-Object.defineProperty(context, "log", {value: log});
-
-// Monkey-patch Math.random() to make it deterministic
-context.random = random;
-vm.runInContext("Math.random = random;", context);
-delete context.random;
-
-
-function errorToFault(err) {
-  if (!err)
-    return null;
-
-  // Trim the stack trace at the self-executing anonymous wrapper function
-  let stackTrimIndex = err.stack.match(/\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/).index;
-
-  return {
-    code: `script.${err.name}`,
-    message: err.message,
-    detail: {
-      name: err.name,
-      message: err.message,
-      stack: err.stack.slice(0, stackTrimIndex)
-    }
-  }
-}
-
-
-function run(script, globals, sessionContext, startRevision, maxRevision, callback, extCounter = 0) {
-  state = {
-    sessionContext: sessionContext,
-    revision: startRevision,
-    maxRevision: maxRevision,
-    uncommitted: false,
-    declarations: [],
-    extensions: {},
-    clear: [],
-    rng: null,
-    extCounter: extCounter
-  };
-
-  for (let n of Object.keys(context))
-    delete context[n];
-
-  Object.assign(context, globals);
-
-  let ret, status;
-
-  try {
-    ret = script.runInContext(context, {displayErrors: false});
-    status = 0;
-  }
-  catch (err) {
-    if (err === COMMIT)
-      status = 1;
-    else if (err === EXT)
-      status = 2;
-    else
-      return callback(null, errorToFault(err));
-  }
-
-  let _state = state;
-  let args = Array.from(arguments);
-
-  // Need to maintain a counter of ext() calls to avoid calling certain
-  // functions (e.g. log()) multiple times as calling ext() results in
-  // re-execution of script without revision being incremented.
-  args[6] = 0 - (state.extCounter - extCounter);
-
-  let counter = 3;
-  for (let key of Object.keys(_state.extensions)) {
-    counter += 2;
-    extensions.run(_state.extensions[key], function(err, fault, res) {
-      if (err || fault) {
-        if (counter & 1)
-          callback(err, fault);
-        return counter = 0;
-      }
-      _state.sessionContext.extensionsCache[key] = res;
-      if ((counter -= 2) === 1)
-        if (status === 2)
-          return run.apply(null, args);
-        else
-          return callback(null, null, _state.clear, _state.declarations, status === 0, ret);
-    });
-  }
-
-  if ((counter -= 2) === 1)
-    if (status === 2)
-      return run.apply(null, args);
-    else
-      return callback(null, null, _state.clear, _state.declarations, status === 0, ret);
-}
-
-
-exports.run = run;

lib/sandbox.ts

@@ -0,0 +1,448 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as vm from "vm";
+import seedrandom from "seedrandom";
+import * as device from "./device";
+import * as extensions from "./extensions";
+import * as logger from "./logger";
+import * as scheduling from "./scheduling";
+import Path from "./common/path";
+import { Fault, SessionContext, ScriptResult } from "./types";
+
+// Used for throwing to exit user script and commit
+const COMMIT = Symbol();
+
+// Used to execute extensions and restart
+const EXT = Symbol();
+
+const UNDEFINED = undefined;
+
+const context = vm.createContext();
+
+let state;
+
+const runningExtensions = new WeakMap<
+  SessionContext,
+  Map<string, Promise<Fault>>
+>();
+function runExtension(sessionContext, key, extCall): Promise<Fault> {
+  let re = runningExtensions.get(sessionContext);
+  if (!re) {
+    re = new Map<string, Promise<Fault>>();
+    runningExtensions.set(sessionContext, re);
+  }
+
+  let prom = re.get(key);
+  if (!prom) {
+    re.set(
+      key,
+      (prom = new Promise((resolve, reject) => {
+        extensions
+          .run(extCall)
+          .then(({ fault, value }) => {
+            re.delete(key);
+            if (!fault) sessionContext.extensionsCache[key] = value;
+            resolve(fault);
+          })
+          .catch(reject);
+      }))
+    );
+  }
+
+  return prom;
+}
+
+class SandboxDate {
+  public constructor(
+    ...argumentList: [
+      number?,
+      number?,
+      number?,
+      number?,
+      number?,
+      number?,
+      number?
+    ]
+  ) {
+    if (argumentList.length) return new Date(...argumentList);
+
+    return new Date(state.sessionContext.timestamp);
+  }
+
+  public static now(intervalOrCron, variance): number {
+    let t = state.sessionContext.timestamp;
+
+    if (typeof intervalOrCron === "number") {
+      if (variance == null) variance = intervalOrCron;
+
+      let offset = 0;
+      if (variance)
+        offset = scheduling.variance(state.sessionContext.deviceId, variance);
+
+      t = scheduling.interval(t, intervalOrCron, offset);
+    } else if (typeof intervalOrCron === "string") {
+      let offset = 0;
+      if (variance)
+        offset = scheduling.variance(state.sessionContext.deviceId, variance);
+      const cron = scheduling.parseCron(intervalOrCron);
+      t = scheduling.cron(t, cron, offset)[0];
+    } else if (intervalOrCron) {
+      throw new Error("Invalid Date.now() argument");
+    }
+
+    return t;
+  }
+
+  public static parse(dateString: string): number {
+    return Date.parse(dateString);
+  }
+
+  public static UTC(
+    ...args: [number, number?, number?, number?, number?, number?, number?]
+  ): number {
+    return Date.UTC(...args);
+  }
+}
+
+function random(): number {
+  if (!state.rng) state.rng = seedrandom(state.sessionContext.deviceId);
+
+  return state.rng();
+}
+
+random.seed = function (s) {
+  state.rng = seedrandom(s);
+};
+
+class ParameterWrapper {
+  public constructor(path: Path, attributes, unpacked?, unpackedRevision?) {
+    for (const attrName of attributes) {
+      Object.defineProperty(this, attrName, {
+        get: function () {
+          if (state.uncommitted) commit();
+
+          if (state.revision !== unpackedRevision) {
+            unpackedRevision = state.revision;
+            unpacked = device.unpack(
+              state.sessionContext.deviceData,
+              path,
+              state.revision
+            );
+          }
+
+          if (!unpacked.length) return UNDEFINED;
+
+          const attr = state.sessionContext.deviceData.attributes.get(
+            unpacked[0],
+            state.revision
+          )[attrName];
+
+          if (!attr) return UNDEFINED;
+
+          return attr[1];
+        },
+      });
+    }
+
+    Object.defineProperty(this, "path", {
+      get: function () {
+        if (state.uncommitted) commit();
+
+        if (state.revision !== unpackedRevision) {
+          unpackedRevision = state.revision;
+          unpacked = device.unpack(
+            state.sessionContext.deviceData,
+            path,
+            state.revision
+          );
+        }
+
+        if (!unpacked.length) return UNDEFINED;
+
+        return unpacked[0].toString();
+      },
+    });
+
+    Object.defineProperty(this, "size", {
+      get: function () {
+        if (state.uncommitted) commit();
+
+        if (state.revision !== unpackedRevision) {
+          unpackedRevision = state.revision;
+          unpacked = device.unpack(
+            state.sessionContext.deviceData,
+            path,
+            state.revision
+          );
+        }
+
+        if (!unpacked.length) return UNDEFINED;
+
+        return unpacked.length;
+      },
+    });
+
+    this[Symbol.iterator] = function* () {
+      if (state.uncommitted) commit();
+
+      if (state.revision !== unpackedRevision) {
+        unpackedRevision = state.revision;
+        unpacked = device.unpack(
+          state.sessionContext.deviceData,
+          path,
+          state.revision
+        );
+      }
+
+      for (const p of unpacked)
+        yield new ParameterWrapper(p, attributes, [p], state.revision);
+    };
+  }
+}
+
+function declare(
+  path: string,
+  timestamps: { [attr: string]: number },
+  values: { [attr: string]: any }
+): ParameterWrapper {
+  state.uncommitted = true;
+  if (!timestamps) timestamps = {};
+
+  if (!values) values = {};
+
+  const parsedPath = Path.parse(path);
+
+  const declaration = {
+    path: parsedPath,
+    pathGet: 1,
+    pathSet: null,
+    attrGet: null,
+    attrSet: null,
+    defer: true,
+  };
+
+  const attrs = new Set();
+
+  for (const [attrName, attrValue] of Object.entries(values)) {
+    if (attrName === "path") {
+      declaration.pathSet = attrValue;
+    } else {
+      attrs.add(attrName);
+      if (!declaration.attrGet) declaration.attrGet = {};
+      if (!declaration.attrSet) declaration.attrSet = {};
+      declaration.attrGet[attrName] = 1;
+      if (attrName === "value" && !Array.isArray(values.value))
+        declaration.attrSet.value = [values.value];
+      else declaration.attrSet[attrName] = values[attrName];
+    }
+  }
+
+  for (const [attrName, attrTimestamp] of Object.entries(timestamps)) {
+    if (attrName === "path") {
+      declaration.pathGet = attrTimestamp;
+    } else {
+      attrs.add(attrName);
+      if (!declaration.attrGet) declaration.attrGet = {};
+      declaration.attrGet[attrName] = attrTimestamp;
+    }
+  }
+
+  state.declarations.push(declaration);
+
+  return new ParameterWrapper(parsedPath, attrs);
+}
+
+function clear(path: string, timestamp: number, attributes?): void {
+  state.uncommitted = true;
+
+  if (state.revision === state.maxRevision)
+    state.clear.push([Path.parse(path), timestamp, attributes]);
+}
+
+function commit(): void {
+  ++state.revision;
+  state.uncommitted = false;
+
+  if (state.revision === state.maxRevision + 1) {
+    for (const d of state.declarations) d.defer = false;
+    throw COMMIT;
+  } else if (state.revision > state.maxRevision + 1) {
+    throw new Error(
+      "Declare function should not be called from within a try/catch block"
+    );
+  }
+}
+
+function ext(...args: unknown[]): any {
+  ++state.extCounter;
+  const extCall = args.map(String);
+  const key = `${state.revision}: ${JSON.stringify(extCall)}`;
+
+  if (key in state.sessionContext.extensionsCache)
+    return state.sessionContext.extensionsCache[key];
+
+  state.extensions[key] = extCall;
+  throw EXT;
+}
+
+function log(msg: string, meta: Record<string, unknown>): void {
+  if (state.revision === state.maxRevision && state.extCounter >= 0) {
+    const details = Object.assign({}, meta, {
+      sessionContext: state.sessionContext,
+      message: `Script: ${msg}`,
+    });
+
+    delete details["hostname"];
+    delete details["pid"];
+    delete details["name"];
+    delete details["version"];
+    delete details["deviceId"];
+    delete details["remoteAddress"];
+
+    logger.accessInfo(details);
+  }
+}
+
+Object.defineProperty(context, "Date", { value: SandboxDate });
+Object.defineProperty(context, "declare", { value: declare });
+Object.defineProperty(context, "clear", { value: clear });
+Object.defineProperty(context, "commit", { value: commit });
+Object.defineProperty(context, "ext", { value: ext });
+Object.defineProperty(context, "log", { value: log });
+
+// Monkey-patch Math.random() to make it deterministic
+context.random = random;
+vm.runInContext("Math.random = random;", context);
+delete context.random;
+
+function errorToFault(err: Error): Fault {
+  if (!err) return null;
+
+  if (!err.name) return { code: "script", message: `${err}` };
+
+  const fault: Fault = {
+    code: `script.${err.name}`,
+    message: err.message,
+    detail: {
+      name: err.name,
+      message: err.message,
+    },
+  };
+
+  if (err.stack) {
+    fault.detail["stack"] = err.stack;
+    // Trim the stack trace at the self-executing anonymous wrapper function
+    const stackTrimIndex = fault.detail["stack"].match(
+      /\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/
+    );
+    if (stackTrimIndex) {
+      fault.detail["stack"] = fault.detail["stack"].slice(
+        0,
+        stackTrimIndex.index
+      );
+    }
+  }
+
+  return fault;
+}
+
+export async function run(
+  script: vm.Script,
+  globals: Record<string, unknown>,
+  sessionContext: SessionContext,
+  startRevision: number,
+  maxRevision: number,
+  extCounter = 0
+): Promise<ScriptResult> {
+  state = {
+    sessionContext: sessionContext,
+    revision: startRevision,
+    maxRevision: maxRevision,
+    uncommitted: false,
+    declarations: [],
+    extensions: {},
+    clear: [],
+    rng: null,
+    extCounter: extCounter,
+  };
+
+  for (const n of Object.keys(context)) delete context[n];
+
+  Object.assign(context, globals);
+
+  let ret, status;
+
+  try {
+    ret = script.runInContext(context, { displayErrors: false });
+    status = 0;
+  } catch (err) {
+    if (err === COMMIT) {
+      status = 1;
+    } else if (err === EXT) {
+      status = 2;
+    } else {
+      return {
+        fault: errorToFault(err),
+        clear: null,
+        declare: null,
+        done: false,
+        returnValue: null,
+      };
+    }
+  }
+
+  const _state = state;
+  let fault;
+
+  await Promise.all(
+    Object.entries(_state.extensions).map(async ([k, v]) => {
+      fault = (await runExtension(_state.sessionContext, k, v)) || fault;
+    })
+  );
+
+  if (fault) {
+    return {
+      fault: fault,
+      clear: null,
+      declare: null,
+      done: false,
+      returnValue: null,
+    };
+  }
+
+  if (status === 2) {
+    return run(
+      script,
+      globals,
+      sessionContext,
+      startRevision,
+      maxRevision,
+      extCounter - _state.extCounter
+    );
+  }
+
+  return {
+    fault: null,
+    clear: _state.clear,
+    declare: _state.declarations,
+    done: status === 0,
+    returnValue: ret,
+  };
+}

lib/scheduling.ts

@@ -0,0 +1,67 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as crypto from "crypto";
+import * as later from "later";
+
+function md532(str): number {
+  const digest = crypto.createHash("md5").update(str).digest();
+  return (
+    digest.readUInt32LE(0) ^
+    digest.readUInt32LE(4) ^
+    digest.readUInt32LE(8) ^
+    digest.readUInt32LE(12)
+  );
+}
+
+export function variance(deviceId: string, vrnc: number): number {
+  return (md532(deviceId) >>> 0) % vrnc;
+}
+
+export function interval(
+  timestamp: number,
+  intrvl: number,
+  offset = 0
+): number {
+  return Math.trunc((timestamp + offset) / intrvl) * intrvl - offset;
+}
+
+export function parseCron(cronExp: string): any {
+  const parts = cronExp.trim().split(/\s+/);
+  if (parts.length === 5) parts.unshift("*");
+
+  return later.schedule(later.parse.cron(parts.join(" "), true));
+}
+
+export function cron(
+  timestamp: number,
+  schedule: unknown,
+  offset = 0
+): number[] {
+  // TODO later.js doesn't throw erorr if expression is invalid!
+  const ret = [0, 0];
+
+  const prev = (schedule as any).prev(1, new Date(timestamp + offset));
+  if (prev) ret[0] = prev.setMilliseconds(0) - offset;
+
+  const next = (schedule as any).next(1, new Date(timestamp + offset + 1000));
+  if (next) ret[1] = next.setMilliseconds(0) - offset;
+
+  return ret;
+}

lib/server.coffee

@@ -1,99 +0,0 @@
-###
-# Copyright 2013-2017  Zaid Abdulla
-#
-# This file is part of GenieACS.
-#
-# GenieACS is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# GenieACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
-###
-
-service = process.argv[2]
-if not service?
-  throw new Error('Missing argument cwmp, fs, or nbi')
-
-util = require 'util'
-cluster = require 'cluster'
-logger = require './logger'
-logger.init(service, require('../package.json').version)
-config = require './config'
-db = require './db'
-extensions = require './extensions'
-
-networkInterface = config.get("#{service.toUpperCase()}_INTERFACE")
-port = config.get("#{service.toUpperCase()}_PORT")
-useHttps = config.get("#{service.toUpperCase()}_SSL")
-
-listener = require("./#{service}").listener
-onConnection = require("./#{service}").onConnection
-
-server = null
-
-exit = () ->
-  setTimeout(() ->
-    process.exit(1)
-  , 30000).unref()
-
-  cluster.worker?.disconnect()
-
-  if not server
-    db.disconnect()
-    extensions.killAll()
-    logger.close()
-    return
-
-  server.close(() ->
-    db.disconnect()
-    extensions.killAll()
-    logger.close()
-  )
-
-
-process.on('uncaughtException', (err) ->
-  logger.error({
-    message: 'Uncaught exception'
-    exception: err
-  })
-  exit()
-)
-
-
-if useHttps
-  path = require 'path'
-  fs = require 'fs'
-  httpsKey = path.resolve(config.get('CONFIG_DIR'), "#{service}.key")
-  httpsCert = path.resolve(config.get('CONFIG_DIR'), "#{service}.crt")
-  httpsCa = path.resolve(config.get('CONFIG_DIR'), "#{service}.ca-bundle")
-  options = {
-    key: fs.readFileSync(httpsKey),
-    cert: fs.readFileSync(httpsCert)
-  }
-
-  try
-    # Use intermediate certificates if available
-    options.ca = fs.readFileSync(httpsCa).toString()
-      .match(/\-+BEGIN CERTIFICATE\-+[0-9a-zA-Z\+\-\/\=\s]+?\-+END CERTIFICATE\-+/g)
-
-  server = require('https').createServer(options, listener)
-  server.on('secureConnection', onConnection) if onConnection?
-else
-  server = require('http').createServer(listener)
-  server.on('connection', onConnection) if onConnection?
-
-db.connect((err) ->
-  throw err if err
-  server.listen(port, networkInterface)
-)
-
-
-process.on('SIGINT', exit)
-process.on('SIGTERM', exit)

lib/server.ts

@@ -0,0 +1,101 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as fs from "fs";
+import * as http from "http";
+import * as https from "https";
+import { Socket } from "net";
+import * as path from "path";
+import { ROOT_DIR } from "./config";
+
+let server: http.Server | https.Server;
+let listener: (...args) => void;
+
+function closeServer(timeout, callback): void {
+  if (!server) return void callback();
+
+  setTimeout(() => {
+    if (!callback) return;
+
+    // Ignore HTTP requests from connection that may still be open
+    server.removeListener("request", listener);
+    server.setTimeout(1);
+
+    const cb = callback;
+    callback = null;
+    setTimeout(cb, 1000);
+  }, timeout).unref();
+
+  server.close(() => {
+    if (!callback) return;
+
+    const cb = callback;
+    callback = null;
+    // Allow some time for connection close events to fire
+    setTimeout(cb, 50);
+  });
+}
+
+interface ServerOptions {
+  port?: number;
+  host?: string;
+  ssl?: { key: string; cert: string };
+  timeout?: number;
+  keepAliveTimeout?: number;
+  onConnection?: (socket: Socket) => void;
+}
+
+export function start(
+  options: ServerOptions,
+  _listener: http.RequestListener
+): void {
+  listener = _listener;
+
+  if (options.ssl) {
+    const opts = {
+      key: options.ssl.key
+        .split(":")
+        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
+      cert: options.ssl.cert
+        .split(":")
+        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
+    };
+
+    server = https.createServer(opts, listener);
+    if (options.onConnection)
+      server.on("secureConnection", options.onConnection);
+  } else {
+    server = http.createServer(listener);
+    if (options.onConnection) server.on("connection", options.onConnection);
+  }
+
+  server.timeout = options.timeout || 0;
+  if (options.keepAliveTimeout != null)
+    server.keepAliveTimeout = options.keepAliveTimeout;
+  server.listen({ port: options.port, host: options.host });
+}
+
+export function stop(): Promise<void> {
+  return new Promise((resolve, reject) => {
+    setTimeout(() => {
+      reject(new Error("Could not close server in a timely manner"));
+    }, 30000).unref();
+    closeServer(20000, resolve);
+  });
+}

lib/soap.js

@@ -1,603 +0,0 @@
-/**
- * Copyright 2013-2017  Zaid Abdulla
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-"use strict";
-
-const libxmljs = require("libxmljs");
-
-const config = require("./config");
-
-const SERVER_NAME = `GenieACS/${require("../package.json").version}`;
-
-const NAMESPACES = {
-  "1.0": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    "xsd": "http://www.w3.org/2001/XMLSchema",
-    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
-    "cwmp": "urn:dslforum-org:cwmp-1-0"
-  },
-  "1.1": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    "xsd": "http://www.w3.org/2001/XMLSchema",
-    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
-    "cwmp": "urn:dslforum-org:cwmp-1-1"
-  },
-  "1.2": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    "xsd": "http://www.w3.org/2001/XMLSchema",
-    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
-    "cwmp": "urn:dslforum-org:cwmp-1-2"
-  },
-  "1.3": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    "xsd": "http://www.w3.org/2001/XMLSchema",
-    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
-    "cwmp": "urn:dslforum-org:cwmp-1-2"
-  },
-  "1.4": {
-    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
-    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
-    "xsd": "http://www.w3.org/2001/XMLSchema",
-    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
-    "cwmp": "urn:dslforum-org:cwmp-1-3"
-  }
-};
-
-const LIBXMLJS_PARSE_OPTIONS = {
-  "nocdata": true,
-  "noent": true,
-  "recover": config.get("XML_RECOVER"),
-  "ignore_enc": config.get("XML_IGNORE_ENC")
-};
-
-const LIBXMLJS_SAVE_OPTIONS = {
-  "format": config.get("XML_FORMAT"),
-  "declaration": !config.get("XML_NO_DECL"),
-  "selfCloseEmpty": !config.get("XML_NO_EMPTY")
-};
-
-const XML_IGNORE_NAMESPACE = config.get("XML_IGNORE_NAMESPACE");
-
-if (XML_IGNORE_NAMESPACE) {
-  libxmljs.Element.prototype.__find = libxmljs.Element.prototype.find;
-  libxmljs.Element.prototype.find = function(xpath, namespaces) {
-    const p = xpath.replace(/([^\/:]+:)?([^\/]+)(\/|$)/g, "$2$3").replace(/([a-zA-Z0-9_-]+)([^\/]*)(\/|$)/g, "*[local-name()=\"$1\"]$2$3");
-    return libxmljs.Element.prototype.__find.call(this, p, namespaces);
-  };
-}
-
-let warnings;
-
-
-function parseBool(v) {
-  v = "" + v;
-  if (v === "true" || v === "TRUE" || v === "True" || v === "1")
-    return true;
-  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
-    return false;
-  else
-    return null;
-}
-
-
-function event(xml) {
-  return xml.find("EventStruct/EventCode").map(e => e.text().trim());
-}
-
-
-function parameterInfoList(xml) {
-  return xml.find("ParameterInfoStruct").map(function(e) {
-    let param = e.get("Name").text();
-    let value = e.get("Writable").text();
-    let parsed = parseBool(value);
-
-    if (parsed === null) {
-      warnings.push({
-        message: "Invalid writable attribute",
-        parameter: param
-      });
-      parsed = false;
-    }
-
-    return [param, parsed];
-  });
-}
-
-
-function parameterValueList(xml) {
-  return xml.find("ParameterValueStruct").map(function(e) {
-    const valueElement = e.get("Value");
-    const valueType = valueElement.attr("type").value();
-    let param = e.get("Name").text();
-    let value = valueElement.text();
-    let parsed = value;
-    if (valueType === "xsd:boolean") {
-      parsed = parseBool(value);
-      if (parsed === null) {
-        warnings.push({
-          message: "Invalid value attribute",
-          parameter: param
-        });
-        parsed = value;
-      }
-    }
-    else if (valueType === "xsd:int" || valueType === "xsd:unsignedInt") {
-      parsed = parseInt(value);
-      if (isNaN(parsed)) {
-        warnings.push({
-          message: "Invalid value attribute",
-          parameter: param
-        });
-        parsed = value;
-      }
-    }
-    else if (valueType === "xsd:dateTime") {
-      parsed = Date.parse(value);
-      if (isNaN(parsed)) {
-        warnings.push({
-          message: "Invalid value attribute",
-          parameter: param
-        });
-        parsed = value;
-      }
-    }
-
-   return [param, parsed, valueType];
-  });
-}
-
-
-function GetParameterNames(xml, methodRequest) {
-  const el = xml.node("cwmp:GetParameterNames");
-  el.node("ParameterPath").text(methodRequest.parameterPath);
-  el.node("NextLevel").text(+methodRequest.nextLevel);
-}
-
-
-function GetParameterNamesResponse(xml) {
-  return {
-    name: "GetParameterNamesResponse",
-    parameterList: parameterInfoList(xml.get("ParameterList"))
-  };
-}
-
-
-function GetParameterValues(xml, methodRequest) {
-  const el = xml.node("cwmp:GetParameterValues").node("ParameterNames");
-  el.attr({
-    "soap-enc:arrayType": `xsd:string[${methodRequest.parameterNames.length}]`
-  });
-
-  for (const p of methodRequest.parameterNames)
-    el.node("string").text(p);
-}
-
-
-function GetParameterValuesResponse(xml) {
-  return {
-    name: "GetParameterValuesResponse",
-    parameterList: parameterValueList(xml.get("ParameterList"))
-  };
-}
-
-
-function SetParameterValues(xml, methodRequest) {
-  const el = xml.node("cwmp:SetParameterValues");
-  const paramList = el.node("ParameterList");
-  paramList.attr({
-    "soap-enc:arrayType": `cwmp:ParameterValueStruct[${methodRequest.parameterList.length}]`
-  });
-
-  for (const p of methodRequest.parameterList) {
-    const pvs = paramList.node("ParameterValueStruct");
-    pvs.node("Name").text(p[0]);
-    pvs.node("Value").attr({"xsi:type": p[2]}).text(p[1]);
-  }
-
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-
-function SetParameterValuesResponse(xml) {
-  return {
-    name: "SetParameterValuesResponse",
-    status: parseInt(xml.get("Status").text())
-  };
-}
-
-
-function AddObject(xml, methodRequest) {
-  const el = xml.node("cwmp:AddObject");
-  el.node("ObjectName").text(methodRequest.objectName);
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-
-function AddObjectResponse(xml) {
-  return {
-    name: "AddObjectResponse",
-    instanceNumber: parseInt(xml.get("InstanceNumber").text()),
-    status: parseInt(xml.get("Status").text())
-  };
-}
-
-
-function DeleteObject(xml, methodRequest) {
-  const el = xml.node("cwmp:DeleteObject");
-  el.node("ObjectName").text(methodRequest.objectName);
-  el.node("ParameterKey").text(methodRequest.parameterKey || "");
-}
-
-
-function DeleteObjectResponse(xml) {
-  return {
-    name: "DeleteObjectResponse",
-    status: parseInt(xml.get("Status").text())
-  };
-}
-
-
-function Reboot(xml, methodRequest) {
-  const el = xml.node("cwmp:Reboot");
-  el.node("CommandKey").text(methodRequest.commandKey || "");
-}
-
-
-function RebootResponse(xml) {
-  return {
-    name: "RebootResponse"
-  };
-}
-
-
-function FactoryReset(xml, methodRequest) {
-  xml.node("cwmp:FactoryReset");
-}
-
-
-function FactoryResetResponse(xml, methodRequest) {
-  return {
-    name: "FactoryResetResponse"
-  };
-}
-
-
-function Download(xml, methodRequest) {
-  const el = xml.node("cwmp:Download");
-  el.node("CommandKey").text(methodRequest.commandKey || "");
-  el.node("FileType").text(methodRequest.fileType);
-  el.node("URL").text(methodRequest.url);
-  el.node("Username").text(methodRequest.username || "");
-  el.node("Password").text(methodRequest.password || "");
-  el.node("FileSize").text(methodRequest.fileSize || "0");
-  el.node("TargetFileName").text(methodRequest.targetFileName || "");
-  el.node("DelaySeconds").text(methodRequest.delaySeconds || "0");
-  el.node("SuccessURL").text(methodRequest.successUrl || "");
-  el.node("FailureURL").text(methodRequest.failureUrl || "");
-}
-
-
-function DownloadResponse(xml) {
-  const res = {
-    name: "DownloadResponse",
-    status: parseInt(xml.get("Status").text())
-  };
-
-  if (res.status === 0) {
-    res.startTime = Date.parse(xml.get("StartTime").text());
-    res.completeTime = Date.parse(xml.get("CompleteTime").text());
-  }
-
-  return res;
-}
-
-
-function Inform(xml) {
-  return {
-    name: "Inform",
-    parameterList: parameterValueList(xml.get("ParameterList")),
-    deviceId: {
-      "Manufacturer": xml.get("DeviceId/Manufacturer").text(),
-      "OUI": xml.get("DeviceId/OUI").text(),
-      "ProductClass": xml.get("DeviceId/ProductClass").text(),
-      "SerialNumber": xml.get("DeviceId/SerialNumber").text()
-    },
-    event: event(xml.get("Event")),
-    retryCount: parseInt(xml.get("RetryCount").text())
-  };
-}
-
-
-function InformResponse(xml, methodResponse) {
-  xml.node("cwmp:InformResponse").node("MaxEnvelopes").text(1);
-}
-
-
-function GetRPCMethods(xml) {
-  return {name: "GetRPCMethods"};
-}
-
-
-function GetRPCMethodsResponse(xml, methodResponse) {
-  const el = xml.node("cwmp:GetRPCMethodsResponse").node("MethodList");
-  el.attr({
-    "soap-enc:arrayType": `xsd:string[${methodResponse.methodList.length}]`
-  });
-
-  for (const m of methodResponse.methodList)
-    el.node("string").text(m);
-}
-
-
-function TransferComplete(xml) {
-  return {
-    name: "TransferComplete",
-    commandKey: xml.get("CommandKey").text(),
-    faultStruct: faultStruct(xml.get("FaultStruct")),
-    startTime: Date.parse(xml.get("StartTime").text()),
-    completeTime: Date.parse(xml.get("CompleteTime").text())
-  };
-}
-
-
-function TransferCompleteResponse(xml, methodResponse) {
-  xml.node("cwmp:TransferCompleteResponse").text("");
-}
-
-
-function RequestDownload(xml) {
-  return {
-    name: "RequestDownload",
-    fileType: xml.get("FileType").text()
-  };
-}
-
-
-function RequestDownloadResponse(xml, methodResponse) {
-  xml.node("cwmp:RequestDownloadResponse").text("");
-}
-
-
-function faultStruct(xml) {
-  const f = {
-    faultCode: xml.get("FaultCode").text(),
-    faultString: xml.get("FaultString").text()
-  };
-
-  for (const e of xml.find("SetParameterValuesFault")) {
-    if (!f.setParameterValuesFault)
-      f.setParameterValuesFault = [];
-
-    f.setParameterValuesFault.push({
-      parameterName: e.get("ParameterName").text(),
-      faultCode: e.get("FaultCode").text(),
-      faultString: e.get("FaultString").text()
-    });
-  }
-
-  return f;
-}
-
-
-function fault(xml, cwmpVersion) {
-  return {
-    faultCode: xml.get("faultcode").text(),
-    faultString: xml.get("faultstring").text(),
-    detail: faultStruct(xml.get("detail/cwmp:Fault", NAMESPACES[cwmpVersion]))
-  };
-}
-
-
-function request(httpRequest, cwmpVersion, warn) {
-  warnings = warn;
-
-  const rpc = {
-    cwmpVersion: cwmpVersion
-  };
-
-  const data = httpRequest.getBody();
-
-  if (!data.length)
-    return rpc;
-
-  const xml = libxmljs.parseXml(data, LIBXMLJS_PARSE_OPTIONS);
-
-  let methodElement;
-
-  if (rpc.cwmpVersion) {
-    methodElement = xml.get("/soap-env:Envelope/soap-env:Body/cwmp:*", NAMESPACES[rpc.cwmpVersion]);
-  }
-  else {
-    // cwmpVersion not passed, thus it's an inform request
-    methodElement = xml.get("/soap-env:Envelope/soap-env:Body/*", NAMESPACES["1.0"]);
-    switch (methodElement.namespace().href()) {
-      case "urn:dslforum-org:cwmp-1-0":
-        rpc.cwmpVersion = "1.0";
-        break;
-      case "urn:dslforum-org:cwmp-1-1":
-        rpc.cwmpVersion = "1.1";
-        break;
-      case "urn:dslforum-org:cwmp-1-2":
-        const timeoutElement = xml.get("/soap-env:Envelope/soap-env:Header/cwmp:sessionTimeout", NAMESPACES["1.2"]);
-        if (timeoutElement) {
-          rpc.sessionTimeout = parseInt(timeoutElement.text());
-          rpc.cwmpVersion = "1.3";
-        }
-        else {
-          rpc.cwmpVersion = "1.2";
-        }
-        break;
-      case "urn:dslforum-org:cwmp-1-3":
-        rpc.cwmpVersion = "1.4";
-        break;
-      default:
-        throw new Error("Unrecognized CWMP version");
-    }
-
-    if (!rpc.sessionTimeout) {
-      const timeoutElement = xml.get("/soap-env:Envelope/soap-env:Header/cwmp:sessionTimeout", NAMESPACES[rpc.cwmpVersion]);
-      if (timeoutElement)
-        rpc.sessionTimeout = parseInt(timeoutElement.text());
-    }
-  }
-
-  rpc.id = xml.get("/soap-env:Envelope/soap-env:Header/cwmp:ID", NAMESPACES[rpc.cwmpVersion]).text();
-
-  if (methodElement && !(XML_IGNORE_NAMESPACE && methodElement.name() === "Fault")) {
-    switch (methodElement.name()) {
-      case "Inform":
-        rpc.cpeRequest = Inform(methodElement);
-        break;
-      case "GetRPCMethods":
-        rpc.cpeRequest = GetRPCMethods(methodElement);
-        break;
-      case "TransferComplete":
-        rpc.cpeRequest = TransferComplete(methodElement);
-        break;
-      case "RequestDownload":
-        rpc.cpeRequest = RequestDownload(methodElement);
-        break;
-      case "GetParameterNamesResponse":
-        rpc.cpeResponse = GetParameterNamesResponse(methodElement);
-        break;
-      case "GetParameterValuesResponse":
-        rpc.cpeResponse = GetParameterValuesResponse(methodElement);
-        break;
-      case "SetParameterValuesResponse":
-        rpc.cpeResponse = SetParameterValuesResponse(methodElement);
-        break;
-      case "AddObjectResponse":
-        rpc.cpeResponse = AddObjectResponse(methodElement);
-        break;
-      case "DeleteObjectResponse":
-        rpc.cpeResponse = DeleteObjectResponse(methodElement);
-        break;
-      case "RebootResponse":
-        rpc.cpeResponse = RebootResponse(methodElement);
-        break;
-      case "FactoryResetResponse":
-        rpc.cpeResponse = FactoryResetResponse(methodElement);
-        break;
-      case "DownloadResponse":
-        rpc.cpeResponse = DownloadResponse(methodElement);
-        break;
-      default:
-        throw new Error(`8000 Method not supported ${methodElement.name()}`);
-    }
-  }
-  else {
-    const faultElement = xml.get("/soap-env:Envelope/soap-env:Body/soap-env:Fault", NAMESPACES[rpc.cwmpVersion]);
-    rpc.cpeFault = fault(faultElement, rpc.cwmpVersion);
-  }
-
-  return rpc;
-}
-
-
-function createSoapEnv(cwmpVersion) {
-  const xml = libxmljs.Document();
-  const env = xml.node("soap-env:Envelope");
-
-  for (const prefix of Object.keys(NAMESPACES[cwmpVersion]))
-    env.defineNamespace(prefix, NAMESPACES[cwmpVersion][prefix]);
-
-  return env;
-}
-
-
-function response(rpc) {
-  const headers = {
-    "Server": SERVER_NAME,
-    "SOAPServer": SERVER_NAME
-  };
-
-  if (!rpc)
-    return {code: 204, headers: headers, data: new Buffer(0)};
-
-  const env = createSoapEnv(rpc.cwmpVersion);
-  const header = env.node("soap-env:Header");
-  header.node("cwmp:ID").attr({
-    "soap-env:mustUnderstand": 1
-  }).text(rpc.id);
-
-  const body = env.node("soap-env:Body");
-
-  if (rpc.acsResponse) {
-    switch (rpc.acsResponse.name) {
-      case "InformResponse":
-        InformResponse(body, rpc.acsResponse);
-        break;
-      case "GetRPCMethodsResponse":
-        GetRPCMethodsResponse(body, rpc.acsResponse);
-        break;
-      case "TransferCompleteResponse":
-        TransferCompleteResponse(body, rpc.acsResponse);
-        break;
-      case "RequestDownloadResponse":
-        RequestDownloadResponse(body, rpc.acsResponse);
-        break;
-      default:
-        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
-    }
-  }
-  else if (rpc.acsRequest) {
-    switch (rpc.acsRequest.name) {
-      case "GetParameterNames":
-        GetParameterNames(body, rpc.acsRequest);
-        break;
-      case "GetParameterValues":
-        GetParameterValues(body, rpc.acsRequest);
-        break;
-      case "SetParameterValues":
-        SetParameterValues(body, rpc.acsRequest);
-        break;
-      case "AddObject":
-        AddObject(body, rpc.acsRequest);
-        break;
-      case "DeleteObject":
-        DeleteObject(body, rpc.acsRequest);
-        break;
-      case "Reboot":
-        Reboot(body, rpc.acsRequest);
-        break;
-      case "FactoryReset":
-        FactoryReset(body, rpc.acsRequest);
-        break;
-      case "Download":
-        Download(body, rpc.acsRequest);
-        break;
-      default:
-        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
-    }
-  }
-
-  headers["Content-Type"] = "text/xml; charset=\"utf-8\"";
-  return {
-    code: 200,
-    headers: headers,
-    data: new Buffer(env.doc().toString(LIBXMLJS_SAVE_OPTIONS))
-  };
-}
-
-
-exports.request = request;
-exports.response = response;

lib/types.ts

@@ -0,0 +1,597 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Path from "./common/path";
+import PathSet from "./common/path-set";
+import VersionedMap from "./versioned-map";
+import InstanceSet from "./instance-set";
+import { IncomingMessage, ServerResponse } from "http";
+import { Script } from "vm";
+
+export type Expression = string | number | boolean | null | any[];
+
+export interface Fault {
+  code: string;
+  message: string;
+  detail?:
+    | FaultStruct
+    | {
+        name: string;
+        message: string;
+        stack?: string;
+      };
+  timestamp?: number;
+}
+
+export interface SessionFault extends Fault {
+  timestamp: number;
+  provisions: string[][];
+  retryNow?: boolean;
+  precondition?: boolean;
+  retries?: number;
+  expiry?: number;
+}
+
+export interface Attributes {
+  object?: [number, 1 | 0];
+  writable?: [number, 1 | 0];
+  value?: [number, [string | number | boolean, string]];
+  notification?: [number, number];
+  accessList?: [number, string[]];
+}
+
+export interface AttributeTimestamps {
+  object?: number;
+  writable?: number;
+  value?: number;
+  notification?: number;
+  accessList?: number;
+}
+
+export interface AttributeValues {
+  object?: boolean;
+  writable?: boolean;
+  value?: [string | number | boolean, string?];
+  notification?: number;
+  accessList?: string[];
+}
+
+export interface DeviceData {
+  paths: PathSet;
+  timestamps: VersionedMap<Path, number>;
+  attributes: VersionedMap<Path, Attributes>;
+  trackers: Map<Path, { [name: string]: number }>;
+  changes: Set<string>;
+}
+
+export type VirtualParameterDeclaration = [
+  Path,
+  {
+    path?: number;
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  }?,
+  {
+    path?: [number, number];
+    object?: boolean;
+    writable?: boolean;
+    value?: [string | number | boolean, string?];
+    notification?: number;
+    accessList?: string[];
+  }?
+];
+
+export interface SyncState {
+  refreshAttributes: {
+    exist: Set<Path>;
+    object: Set<Path>;
+    writable: Set<Path>;
+    value: Set<Path>;
+    notification: Set<Path>;
+    accessList: Set<Path>;
+  };
+  spv: Map<Path, [string | number | boolean, string]>;
+  spa: Map<Path, { notification: number; accessList: string[] }>;
+  gpn: Set<Path>;
+  gpnPatterns: Map<Path, number>;
+  tags: Map<Path, boolean>;
+  virtualParameterDeclarations: VirtualParameterDeclaration[][];
+  instancesToDelete: Map<Path, Set<Path>>;
+  instancesToCreate: Map<Path, InstanceSet>;
+  downloadsToDelete: Set<Path>;
+  downloadsToCreate: InstanceSet;
+  downloadsValues: Map<Path, string | number>;
+  downloadsDownload: Map<Path, number>;
+  duStatesToDelete: Set<Path>;
+  duStatesToCreate: InstanceSet;
+  duStatesValues: Map<Path, string | number>;
+  duStatesChangeDUState: Map<Path, number>;
+  reboot: number;
+  factoryReset: number;
+}
+
+export interface SessionContext {
+  sessionId?: string;
+  timestamp: number;
+  deviceId: string;
+  deviceData: DeviceData;
+  cwmpVersion: string;
+  timeout: number;
+  provisions: any[];
+  channels: { [channel: string]: number };
+  virtualParameters: [
+    string,
+    AttributeTimestamps,
+    AttributeValues,
+    AttributeTimestamps,
+    AttributeValues
+  ][][];
+  revisions: number[];
+  rpcCount: number;
+  iteration: number;
+  cycle: number;
+  extensionsCache: any;
+  declarations: Declaration[][];
+  faults?: { [channel: string]: SessionFault };
+  retries?: { [channel: string]: number };
+  cacheSnapshot?: string;
+  httpResponse?: ServerResponse;
+  httpRequest?: IncomingMessage;
+  faultsTouched?: { [channel: string]: boolean };
+  presetCycles?: number;
+  new?: boolean;
+  debug?: boolean;
+  state: number;
+  authState: number;
+  tasks?: Task[];
+  operations?: { [commandKey: string]: Operation };
+  cacheUntil?: number;
+  syncState?: SyncState;
+  lastActivity?: number;
+  extendLock?: number;
+  rpcRequest?: AcsRequest;
+  operationsTouched?: { [commandKey: string]: 1 | 0 };
+  provisionsRet?: any[];
+  doneTasks?: string[];
+}
+
+export interface Task {
+  _id?: string;
+  name: string;
+  parameterNames?: string[];
+  parameterValues?: [string, string | number | boolean, string?][];
+  objectName?: string;
+  fileType?: string;
+  fileName?: string;
+  targetFileName?: string;
+  expiry?: number;
+  provisions?: (string | number | boolean)[][];
+}
+
+interface OperationBase {
+  name: string;
+  timestamp: number;
+  provisions: string[][];
+  channels: { [channel: string]: number };
+  retries: { [channel: string]: number };
+  args: any;
+}
+
+export interface DownloadOperation extends OperationBase {
+  name: "Download";
+  args: {
+    instance: string;
+    fileType: string;
+    fileName: string;
+    targetFileName: string;
+  };
+}
+
+export interface ChangeDUStateOperation extends OperationBase {
+  name: "ChangeDUState";
+  args: {
+    operations: ChangeDUState["operations"];
+  };
+}
+
+export type Operation = DownloadOperation | ChangeDUStateOperation;
+
+export type AcsRequest =
+  | GetParameterNames
+  | GetParameterValues
+  | GetParameterAttributes
+  | SetParameterValues
+  | SetParameterAttributes
+  | AddObject
+  | DeleteObject
+  | FactoryReset
+  | Reboot
+  | Download
+  | ChangeDUState;
+
+export interface GetParameterNames {
+  name: "GetParameterNames";
+  parameterPath: string;
+  nextLevel: boolean;
+}
+
+export interface GetParameterValues {
+  name: "GetParameterValues";
+  parameterNames: string[];
+}
+
+export interface GetParameterAttributes {
+  name: "GetParameterAttributes";
+  parameterNames: string[];
+}
+
+export interface SetParameterValues {
+  name: "SetParameterValues";
+  parameterList: [string, boolean | number | string, string][];
+  parameterKey?: string;
+  DATETIME_MILLISECONDS?: boolean;
+  BOOLEAN_LITERAL?: boolean;
+}
+
+export interface SetParameterAttributes {
+  name: "SetParameterAttributes";
+  parameterList: [string, number, string[]][];
+}
+
+export interface AddObject {
+  name: "AddObject";
+  objectName: string;
+  parameterKey?: string;
+  instanceValues: Record<string, string>;
+}
+
+export interface DeleteObject {
+  name: "DeleteObject";
+  objectName: string;
+  parameterKey?: string;
+}
+
+export interface FactoryReset {
+  name: "FactoryReset";
+  commandKey?: string;
+}
+
+export interface Reboot {
+  name: "Reboot";
+  commandKey?: string;
+}
+
+export interface Download {
+  name: "Download";
+  commandKey: string;
+  instance: string;
+  fileType: string;
+  fileName?: string;
+  url?: string;
+  username?: string;
+  password?: string;
+  fileSize?: number;
+  targetFileName?: string;
+  delaySecods?: number;
+  successUrl?: string;
+  failureUrl?: string;
+}
+
+export interface ChangeDUState {
+  name: "ChangeDUState";
+  commandKey: string;
+  operations: {
+    operationType: "install" | "update" | "uninstall";
+    instance: string;
+    url: string;
+    uuid: string;
+    version: string;
+    executionEnvRef: string;
+    username: string;
+    password: string;
+  }[];
+}
+
+export interface SpvFault {
+  parameterName: string;
+  faultCode: string;
+  faultString: string;
+}
+
+export interface FaultStruct {
+  faultCode: string;
+  faultString: string;
+  setParameterValuesFault?: SpvFault[];
+}
+
+export interface CpeFault {
+  faultCode: "Client" | "Server";
+  faultString: "CWMP fault";
+  detail?: FaultStruct;
+}
+
+export type CpeResponse =
+  | GetParameterNamesResponse
+  | GetParameterValuesResponse
+  | GetParameterAttributesResponse
+  | SetParameterValuesResponse
+  | SetParameterAttributesResponse
+  | AddObjectResponse
+  | DeleteObjectResponse
+  | RebootResponse
+  | FactoryResetResponse
+  | DownloadResponse
+  | ChangeDUStateResponse;
+
+export interface GetParameterNamesResponse {
+  name: "GetParameterNamesResponse";
+  parameterList: [Path, boolean, boolean][];
+}
+
+export interface GetParameterValuesResponse {
+  name: "GetParameterValuesResponse";
+  parameterList: [Path, string | number | boolean, string][];
+}
+
+export interface GetParameterAttributesResponse {
+  name: "GetParameterAttributesResponse";
+  parameterList: [Path, number, string[]][];
+}
+
+export interface SetParameterValuesResponse {
+  name: "SetParameterValuesResponse";
+  status: number;
+}
+
+export interface SetParameterAttributesResponse {
+  name: "SetParameterAttributesResponse";
+}
+
+export interface AddObjectResponse {
+  name: "AddObjectResponse";
+  instanceNumber: string;
+  status: number;
+}
+
+export interface DeleteObjectResponse {
+  name: "DeleteObjectResponse";
+  status: number;
+}
+
+export interface ChangeDUStateResponse {
+  name: "ChangeDUStateResponse";
+}
+
+export interface RebootResponse {
+  name: "RebootResponse";
+}
+
+export interface FactoryResetResponse {
+  name: "FactoryResetResponse";
+}
+
+export interface DownloadResponse {
+  name: "DownloadResponse";
+  status: number;
+  startTime?: number;
+  completeTime?: number;
+}
+
+export type CpeRequest =
+  | InformRequest
+  | TransferCompleteRequest
+  | GetRPCMethodsRequest
+  | RequestDownloadRequest
+  | DUStateChangeCompleteRequest;
+
+export interface InformRequest {
+  name: "Inform";
+  deviceId: {
+    Manufacturer: string;
+    OUI: string;
+    ProductClass?: string;
+    SerialNumber: string;
+  };
+  event: string[];
+  retryCount: number;
+  parameterList: [Path, string | number | boolean, string][];
+}
+
+export interface TransferCompleteRequest {
+  name: "TransferComplete";
+  commandKey?: string;
+  faultStruct?: FaultStruct;
+  startTime?: number;
+  completeTime?: number;
+}
+
+export interface DUStateChangeCompleteRequest {
+  name: "DUStateChangeComplete";
+  commandKey: string;
+  results: {
+    uuid: string;
+    deploymentUnitRef: string;
+    version: string;
+    currentState: string;
+    resolved: boolean;
+    executionUnitRefList: string;
+    fault?: { faultCode: string; faultString: string };
+    startTime?: number;
+    completeTime?: number;
+  }[];
+}
+
+export interface GetRPCMethodsRequest {
+  name: "GetRPCMethods";
+}
+
+export interface RequestDownloadRequest {
+  name: "RequestDownload";
+  fileType: string;
+}
+
+export type AcsResponse =
+  | InformResponse
+  | GetRPCMethodsResponse
+  | TransferCompleteResponse
+  | RequestDownloadResponse
+  | DUStateChangeCompleteResponse;
+
+export interface InformResponse {
+  name: "InformResponse";
+}
+
+export interface GetRPCMethodsResponse {
+  name: "GetRPCMethodsResponse";
+  methodList: string[];
+}
+
+export interface TransferCompleteResponse {
+  name: "TransferCompleteResponse";
+}
+
+export interface RequestDownloadResponse {
+  name: "RequestDownloadResponse";
+}
+export interface DUStateChangeCompleteResponse {
+  name: "DUStateChangeCompleteResponse";
+}
+
+export interface QueryOptions {
+  projection?: any;
+  skip?: number;
+  limit?: number;
+  sort?: {
+    [param: string]: number;
+  };
+}
+
+export interface Declaration {
+  path: Path;
+  pathGet: number;
+  pathSet?: number | [number, number];
+  attrGet?: {
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  };
+  attrSet?: {
+    object?: boolean;
+    writable?: boolean;
+    value?: [string | number | boolean, string?];
+    notification?: number;
+    accessList?: string[];
+  };
+  defer: boolean;
+}
+
+export type Clear = [
+  Path,
+  number,
+  {
+    object?: number;
+    writable?: number;
+    value?: number;
+    notification?: number;
+    accessList?: number;
+  }?,
+  number?
+];
+
+export interface Preset {
+  name: string;
+  channel: string;
+  schedule?: { md5: string; duration: number; schedule: any };
+  events?: { [event: string]: boolean };
+  precondition?: Expression;
+  provisions: [string, ...Expression[]][];
+}
+
+export interface Provisions {
+  [name: string]: { md5: string; script: Script };
+}
+
+export interface VirtualParameters {
+  [name: string]: { md5: string; script: Script };
+}
+
+export interface Files {
+  [name: string]: { length: number; md5: string; contentType: string };
+}
+
+export interface Users {
+  [name: string]: { password: string; salt: string; roles: string[] };
+}
+
+export interface Permissions {
+  [role: string]: {
+    [access: number]: {
+      [resource: string]: {
+        access: number;
+        filter: Expression;
+        validate?: Expression;
+      };
+    };
+  };
+}
+
+export type PermissionSet = {
+  [resource: string]: {
+    access: number;
+    validate?: Expression;
+    filter: Expression;
+  };
+}[];
+
+export interface Config {
+  [name: string]: Expression;
+}
+
+export interface UiConfig {
+  filters: Record<string, unknown>;
+  device: Record<string, unknown>;
+  index: Record<string, unknown>;
+  overview: {
+    charts?: Record<string, unknown>;
+    groups?: Record<string, unknown>;
+  };
+  pageSize?: Expression;
+}
+
+export interface SoapMessage {
+  id: string;
+  cwmpVersion: string;
+  sessionTimeout: number;
+  cpeRequest?: CpeRequest;
+  cpeFault?: CpeFault;
+  cpeResponse?: CpeResponse;
+}
+
+export interface ScriptResult {
+  fault: Fault;
+  clear: Clear[];
+  declare: Declaration[];
+  done: boolean;
+  returnValue: any;
+}

lib/ui.ts

@@ -0,0 +1,275 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { constants } from "zlib";
+import Koa from "koa";
+import Router from "koa-router";
+import * as jwt from "jsonwebtoken";
+import koaStatic from "koa-static";
+import koaCompress from "koa-compress";
+import koaBodyParser from "koa-bodyparser";
+import koaJwt from "koa-jwt";
+import * as config from "./config";
+import api from "./ui/api";
+import Authorizer from "./common/authorizer";
+import * as logger from "./logger";
+import * as localCache from "./local-cache";
+import { PermissionSet } from "./types";
+import { authLocal } from "./ui/api-functions";
+import * as init from "./init";
+import { version as VERSION } from "../package.json";
+import memoize from "./common/memoize";
+
+declare module "koa" {
+  interface Request {
+    body: any;
+  }
+}
+
+const koa = new Koa();
+const router = new Router();
+
+const JWT_SECRET = "" + config.get("UI_JWT_SECRET");
+const JWT_COOKIE = "genieacs-ui-jwt";
+
+const getAuthorizer = memoize(
+  (snapshot: string, rolesStr: string): Authorizer => {
+    const roles: string[] = JSON.parse(rolesStr);
+    const allPermissions = localCache.getPermissions(snapshot);
+    const permissionSets: PermissionSet[] = roles.map((r) =>
+      Object.values(allPermissions[r] || {})
+    );
+    return new Authorizer(permissionSets);
+  }
+);
+
+koa.on("error", async (err, ctx) => {
+  setTimeout(() => {
+    // Ignored errors resulting from aborted requests
+    if (ctx && ctx.req.aborted) return;
+    throw err;
+  });
+});
+
+koa.use(async (ctx, next) => {
+  const configSnapshot = await localCache.getCurrentSnapshot();
+  ctx.state.configSnapshot = configSnapshot;
+  ctx.set("X-Config-Snapshot", configSnapshot);
+  ctx.set("GenieACS-Version", VERSION);
+  return next();
+});
+
+koa.use(
+  koaJwt({
+    secret: JWT_SECRET,
+    passthrough: true,
+    cookie: JWT_COOKIE,
+    isRevoked: async (ctx, token) => {
+      if (token["authMethod"] === "local") {
+        return !localCache.getUsers(ctx.state.configSnapshot)[
+          token["username"]
+        ];
+      }
+
+      return true;
+    },
+  })
+);
+
+koa.use(async (ctx, next) => {
+  let roles: string[] = [];
+
+  if (ctx.state.user && ctx.state.user.username) {
+    let user;
+    if (ctx.state.user.authMethod === "local") {
+      user = localCache.getUsers(ctx.state.configSnapshot)[
+        ctx.state.user.username
+      ];
+    } else {
+      throw new Error("Invalid auth method");
+    }
+    roles = user.roles || [];
+  }
+
+  ctx.state.authorizer = getAuthorizer(
+    ctx.state.configSnapshot,
+    JSON.stringify(roles)
+  );
+
+  return next();
+});
+
+router.post("/login", async (ctx) => {
+  if (!JWT_SECRET) {
+    ctx.status = 500;
+    ctx.body = "UI_JWT_SECRET is not set";
+    logger.error({ message: "UI_JWT_SECRET is not set" });
+    return;
+  }
+
+  const username = ctx.request.body.username;
+  const password = ctx.request.body.password;
+
+  const log = {
+    message: "Log in",
+    context: ctx,
+    username: username,
+    method: null,
+  };
+
+  function success(authMethod): void {
+    log.method = authMethod;
+    const token = jwt.sign({ username, authMethod }, JWT_SECRET);
+    ctx.cookies.set(JWT_COOKIE, token, { sameSite: "lax" });
+    ctx.body = JSON.stringify(token);
+    logger.accessInfo(log);
+  }
+
+  function failure(): void {
+    ctx.status = 400;
+    ctx.body = "Incorrect username or password";
+    log.message += " failed";
+    logger.accessWarn(log);
+  }
+
+  if (await authLocal(ctx.state.configSnapshot, username, password))
+    return void success("local");
+
+  failure();
+});
+
+router.post("/logout", async (ctx) => {
+  ctx.cookies.set(JWT_COOKIE); // Delete cookie
+  ctx.body = "";
+
+  logger.accessInfo({
+    message: "Log out",
+    context: ctx,
+  });
+});
+
+koa.use(async (ctx, next) => {
+  if (ctx.request.type === "application/octet-stream")
+    ctx.disableBodyParser = true;
+
+  return next();
+});
+
+koa.use(koaBodyParser());
+router.use("/api", api.routes(), api.allowedMethods());
+
+router.get("/status", (ctx) => {
+  ctx.body = "OK";
+});
+
+router.get("/init", async (ctx) => {
+  const status = await init.getStatus();
+  if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
+    if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("permissions", 3))
+      status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("config", 3)) {
+      status["filters"] = false;
+      status["device"] = false;
+      status["index"] = false;
+      status["overview"] = false;
+    }
+    if (!ctx.state.authorizer.hasAccess("presets", 3))
+      status["presets"] = false;
+    if (!ctx.state.authorizer.hasAccess("provisions", 3))
+      status["presets"] = false;
+  }
+
+  ctx.body = status;
+});
+
+router.post("/init", async (ctx) => {
+  const status = ctx.request.body;
+  if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
+    if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("permissions", 3))
+      status["users"] = false;
+    if (!ctx.state.authorizer.hasAccess("config", 3)) {
+      status["filters"] = false;
+      status["device"] = false;
+      status["index"] = false;
+      status["overview"] = false;
+    }
+    if (!ctx.state.authorizer.hasAccess("presets", 3))
+      status["presets"] = false;
+    if (!ctx.state.authorizer.hasAccess("provisions", 3))
+      status["presets"] = false;
+  }
+  await init.seed(status);
+  ctx.body = "";
+});
+
+router.get("/", async (ctx) => {
+  const permissionSets: PermissionSet[] = ctx.state.authorizer.getPermissionSets();
+
+  let wizard = "";
+  if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
+    wizard = '<script>window.location.hash = "#!/wizard";</script>';
+
+  ctx.body = `
+  <html>
+    <head>
+      <title>GenieACS</title>
+      <link rel="shortcut icon" type="image/png" href="favicon.png" />
+      <link rel="stylesheet" href="app.css">
+    </head>
+    <body>
+    <noscript>GenieACS UI requires JavaScript to work. Please enable JavaScript in your browser.</noscript>
+      <script>
+        window.clientConfig = ${JSON.stringify({
+          ui: localCache.getUiConfig(ctx.state.configSnapshot),
+        })};
+        window.configSnapshot = ${JSON.stringify(ctx.state.configSnapshot)};
+        window.genieacsVersion = ${JSON.stringify(VERSION)};
+        window.username = ${JSON.stringify(
+          ctx.state.user ? ctx.state.user.username : ""
+        )};
+        window.permissionSets = ${JSON.stringify(permissionSets)};
+      </script>
+      <script type="module" src="app.js"></script>${wizard} 
+    </body>
+  </html>
+  `;
+});
+
+koa.use(
+  koaCompress({
+    gzip: {
+      flush: constants.Z_SYNC_FLUSH,
+    },
+    deflate: {
+      flush: constants.Z_SYNC_FLUSH,
+    },
+    br: {
+      params: {
+        [constants.BROTLI_PARAM_QUALITY]: 5,
+      },
+    },
+  })
+);
+
+koa.use(router.routes());
+koa.use(koaStatic(config.ROOT_DIR + "/public"));
+
+export const listener = koa.callback();

lib/ui/api-functions.ts

@@ -0,0 +1,211 @@
+/**
+ * Copyright 2013-2019  GenieACS Inc.
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { ObjectID } from "mongodb";
+import * as db from "./db";
+import { del } from "../cache";
+import { getUsers } from "../local-cache";
+import { hashPassword } from "../auth";
+import {
+  insertTasks,
+  awaitSessionEnd,
+  awaitSessionStart,
+  connectionRequest,
+  deleteDevice,
+} from "../api-functions";
+import { Task } from "../types";
+
+async function deleteFault(id): Promise<void> {
+  const deviceId = id.split(":", 1)[0];
+  const channel = id.slice(deviceId.length + 1);
+
+  await Promise.all([
+    db.deleteFault(id),
+    channel.startsWith("task_")
+      ? db.deleteTask(new ObjectID(channel.slice(5)))
+      : null,
+  ]);
+
+  await del(`${deviceId}_tasks_faults_operations`);
+}
+
+export async function deleteResource(
+  resource: string,
+  id: string
+): Promise<void> {
+  switch (resource) {
+    case "devices":
+      await deleteDevice(id);
+      break;
+
+    case "files":
+      await db.deleteFile(id);
+      break;
+
+    case "faults":
+      await deleteFault(id);
+      break;
+
+    case "provisions":
+      await db.deleteProvision(id);
+      break;
+
+    case "presets":
+      await db.deletePreset(id);
+      break;
+
+    case "virtualParameters":
+      await db.deleteVirtualParameter(id);
+      break;
+
+    case "config":
+      await db.deleteConfig(id);
+      break;
+
+    case "permissions":
+      await db.deletePermission(id);
+      break;
+
+    case "users":
+      await db.deleteUser(id);
+      break;
+  }
+
+  await del("presets_hash");
+}
+
+export async function postTasks(
+  deviceId: string,
+  tasks: Task[],
+  timeout: number,
+  device: Record<string, { value?: [boolean | number | string, string] }>
+): Promise<{ connectionRequest: string; tasks: any[] }> {
+  for (const task of tasks) {
+    delete task._id;
+    task["device"] = deviceId;
+  }
+
+  tasks = await insertTasks(tasks);
+  const statuses = tasks.map((t) => {
+    return { _id: t._id, status: "pending" };
+  });
+  const lastInform = Date.now();
+  const notInSession = await awaitSessionEnd(deviceId, 30000);
+  if (!notInSession) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: "Session took too long to complete",
+      tasks: statuses,
+    };
+  }
+
+  await del(`${deviceId}_tasks_faults_operations`);
+
+  const status = await connectionRequest(deviceId, device);
+
+  if (status) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: status,
+      tasks: statuses,
+    };
+  }
+
+  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
+  if (!sessionStarted) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: "No contact from CPE",
+      tasks: statuses,
+    };
+  }
+
+  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
+  if (!sessionEnded) {
+    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+    return {
+      connectionRequest: "Session took too long to complete",
+      tasks: statuses,
+    };
+  }
+
+  const promises = [];
+  for (const s of statuses) {
+    promises.push(db.query("tasks", ["=", ["PARAM", "_id"], s._id]));
+    promises.push(
+      db.query("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
+    );
+  }
+
+  const res = await Promise.all(promises);
+  for (const [i, r] of statuses.entries()) {
+    if (res[i * 2].length === 0) {
+      r.status = "done";
+    } else if (res[i * 2 + 1].length === 1) {
+      r.status = "fault";
+      r["fault"] = res[i * 2 + 1][0];
+    }
+  }
+
+  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
+
+  return { connectionRequest: "OK", tasks: statuses };
+}
+
+export async function putResource(
+  resource: string,
+  id: string,
+  data: Record<string, unknown>
+): Promise<void> {
+  if (resource === "presets") {
+    await db.putPreset(id, data);
+  } else if (resource === "provisions") {
+    await db.putProvision(id, data);
+  } else if (resource === "virtualParameters") {
+    await db.putVirtualParameter(id, data);
+  } else if (resource === "config") {
+    await db.putConfig(id, data);
+  } else if (resource === "permissions") {
+    await db.putPermission(id, data);
+  } else if (resource === "users") {
+    delete data["password"];
+    delete data["salt"];
+    await db.putUser(id, data);
+  }
+
+  await del("presets_hash");
+}
+
+export function authLocal(
+  snapshot: string,
+  username: string,
+  password: string
+): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    const users = getUsers(snapshot);
+    const user = users[username];
+    if (!user || !user.password) return void resolve(null);
+    hashPassword(password, user.salt)
+      .then((hash) => {
+        if (hash === user.password) resolve(true);
+        else resolve(false);
+      })
+      .catch(reject);
+  });
+}