1.1.3

Alpine linux does not have -n option in copy binary.
To avoid a problem on configure, I replaced the command with an "if".

.eslintrc.json

@@ -1,38 +0,0 @@
-{
-  "env": {
-    "es6": true,
-    "node": true
-  },
-  "parser": "@typescript-eslint/parser",
-  "parserOptions": {
-    "project": "./tsconfig.json"
-  },
-  "plugins": ["@typescript-eslint"],
-  "extends": [
-    "eslint:recommended",
-    "plugin:@typescript-eslint/recommended",
-    "prettier"
-  ],
-  "rules": {
-    "no-shadow": ["error", { "allow": ["err"] }],
-    "handle-callback-err": "error",
-    "prefer-arrow-callback": "error",
-    "curly": ["error", "multi-or-nest", "consistent"],
-    "no-buffer-constructor": "error",
-    "prefer-const": ["error", { "destructuring": "all" }],
-    "eqeqeq": ["error", "always", { "null": "ignore" }],
-    "no-return-await": "error",
-    "@typescript-eslint/no-explicit-any": 0,
-    "@typescript-eslint/no-use-before-define": [
-      "error",
-      { "functions": false }
-    ],
-    "@typescript-eslint/explicit-function-return-type": [
-      "error",
-      { "allowExpressions": true }
-    ],
-    "@typescript-eslint/no-floating-promises": "error",
-    "require-atomic-updates": 0,
-    "no-prototype-builtins": 0
-  }
-}

.gitignore

@@ -1,4 +1,27 @@
 *~
+lib/*.map
+lib/api-functions.js
+lib/auth.js
+lib/common.js
+lib/config.js
+lib/cwmp.js
+lib/db.js
+lib/fs.js
+lib/nbi.js
+lib/query.js
+lib/server.js
+lib/session.js
 node_modules
-dist
-docs/_build
+debug
+config/auth.js
+config/config.json
+config/cwmp.key
+config/cwmp.crt
+config/cwmp.ca-bundle
+config/nbi.key
+config/nbi.crt
+config/nbi.ca-bundle
+config/fs.key
+config/fs.crt
+config/fs.ca-bundle
+config/ext

.npmignore

@@ -0,0 +1,21 @@
+*~
+.npmignore
+
+config/auth.js
+config/config.json
+config/cwmp.key
+config/cwmp.crt
+config/cwmp.ca-bundle
+config/nbi.key
+config/nbi.crt
+config/nbi.ca-bundle
+config/fs.key
+config/fs.crt
+config/fs.ca-bundle
+config/ext
+
+debug/*
+!debug/.gitkeep
+
+lib/*.coffee
+lib/*.map

.prettierignore

@@ -1,2 +0,0 @@
-dist
-npm-shrinkwrap.json

CHANGELOG.md

@@ -1,134 +1,5 @@
 # Change Log
 
-## 1.2.5 (2021-03-12)
-
-- Support specifying custom types when uploading files.
-- Fix JS compatibility issue with Safari browser.
-
-## 1.2.4 (2021-02-24)
-
-- The data model state of a CPE is no longer forgotten after unsuccessful
-  session termination (e.g. timeout). This addresses a number of undesired side
-  effects that arise when a CPE does not terminate the session properly.
-- Executing tasks that take a long time to complete (e.g. refreshing the entire
-  data model) no longer shows a timeout error while the task is still being
-  processed.
-- New function `ROUND()` available to expressions. It works similar to the
-  function by the same name in SQLite and PostgreSQL.
-- Log access events for `genieacs-nbi` service.
-- Pipe stdout/stderr from extension scripts to the `genieacs-cwmp` process log.
-- Parameter values of type `xsd:dateTime` are now displayed in the UI and CSV
-  downloads as a date string rather than a numeric value.
-- Add file download link in the files listing page.
-- Display spinner loading animation throughout the UI.
-- Display GenieACS version and build number underneath the logo.
-- New option to specify how many parameters are displayed at a time in the
-  all-parameters component. Simply set `limit` property in the component config.
-- Reduce overly strong Brotli compression level which was causing significant
-  page load slowdown when Brotli is used.
-- Retire dump-data-model tool. `genieacs-sim` can now use a CSV file as its data
-  model.
-- Reduce the number of concurrent database connections from each process.
-- Remove dependency on 'hostInfo' MongoDB command which is a privileged action.
-  It is now possible to use shared MongoDB instances with limited privileged.
-- Fix bug in NBI where querying files returns 404 error.
-- Fix ping not working for devices with an IPv6 address.
-- Fix an elusive memory leak in `genieacs-fs` that slowly eats up memory and can
-  go unnoticed for long periods of time.
-- Fix a rare edge case where a `declare()` call to set a parameter value may not
-  work as intended if the parameter was originally received as part of the
-  Inform message.
-- A number of other fixes and stability improvements.
-
-## 1.2.3 (2020-10-26)
-
-- New config option 'cwmp.skipWritableCheck' for when some CPEs incorrectly
-  report writable parameters as non-writable. When set to true, the scripts will
-  no longer respect the 'writable' attribute of the CPE parameters and will send
-  a SetParamteerValues, AddObject, or DeleteObject request anyway.
-- Tags no longer restrict what characters are allowed. Any character other than
-  alphanumeric characters, hyphen, or underscore is now encoded in the data
-  model (i.e. Tags.\<tag>) using its hex value preceded by "0x".
-- Ask for a confirmation before closing a pop-up dialog with unsaved changes.
-- Better XML validation to avoid crashes caused by invalid CPE requests.
-- Fix confusing 404 error message when the user attempts to modify a resource
-  when they don't have the necessary permissions.
-- Fix a rare issue where genieacs-cwmp stops accepting new connections after
-  running for a few weeks.
-- Fix exception when IS NULL operator is used in certain situations.
-
-## 1.2.2 (2020-10-03)
-
-- Added button to push files to selected devices from device listing page.
-- A few minor UI improvements.
-- Fix exception that can happen and persist after a Download request.
-- Fix validation bug preventing running refreshObject task on data model root.
-- Fix invalid arguments fault in refresh preset configuration when upgrading
-  from v1.1.
-
-## 1.2.1 (2020-09-08)
-
-- Fix bug causing faults to not be displayed in the UI.
-- Fix bug where deleting objects does not get reflected immediately in the UI.
-- Improve conversion between filters written in the expression format and
-  MongoDB queries. There should now be fewer edge cases where the two are not
-  equisatisfiable.
-
-## 1.2.0 (2020-09-01)
-
-- Support GetParameterAttributes and SetParameterAttributes TR-069 methods.
-- Support CASE statement and COALESCE function in expressions.
-- Provision arguments can now be a list of expressions that are dynamically
-  evaluated.
-- Support Forwarded HTTP header to display in the logs the correct IP of CPEs
-  behind a reverse proxy. Must be configured using FORWARDED_HEADER option.
-- Config expressions can now access all available device parameters, not only
-  serial number, product class, and OUI.
-- Use relative URLs throughout the UI to allow serving from a subdirectory using
-  a reverse proxy.
-- Make Date.parse() and Date.UTC() available to provision scripts.
-- libxmljs has been entirely removed in favor of our bespoke XML parser.
-- Removed the config option CWMP_KEEP_ALIVE_TIMEOUT. SESSION_TIMEOUT is now used
-  to determine the TCP connection timeout.
-- The all-parameters component now limits the number of parameters displayed for
-  better performance.
-- The process genieacs-cwmp is now much less likely to throw exceptions as a
-  result of invalid requests from CPE.
-- A large number of bug fixes and stability improvements.
-
-## 1.2.0-beta.0 (2019-07-30)
-
-- A brand new UI superseding genieacs-gui.
-- New initialization wizard on first run.
-- New expression/query language used in search filters and preset preconditions.
-- CPE -> ACS authentication is now supported.
-- New config option (CWMP_KEEP_ALIVE_TIMEOUT) to specify how long to wait for a
-  reply from the CPE before closing the TCP connection.
-- Debug logging has been reimplemented utilizing YAML format for logs.
-- Handle 9005 faults (Invalid Parameter Name) gracefully by attempting to
-  rediscover the path of the missing parameter recursively.
-- declare() statements not followed by an explicit commit() are now deferred
-  until all currently active scripts have been executed.
-- FS_HOSTNAME now defaults to the server's hostname or IP.
-- The API now validates the structure of task objects before saving.
-- New XML parser implementation for better performance. You can revert to the
-  old parser by enabling the config option XML_LIBXMLJS. Requires Node.js v11 or
-  v10.
-- Performance optimizations. While performance has improved for the majority of
-  use cases, there may be situations where performance has degraded. It's
-  recommended to revisit your hardware requirements.
-- Connection request authentication no longer uses 'auth.js' file. Instead, the
-  connection request authentication behavior can now be customized using an
-  'expression'.
-- The config file (config.json) has been deprecated. System configuration (e.g.
-  listen ports, worker count) are now recommended to be passed as environment.
-  variables. Other general configuration options are stored in the database so
-  as to not require service restart for changes to take effect.
-- Optional redis dependency has been removed completely.
-- Tags now allow only alphanumeric characters and underscore.
-- Supported versions of Node.js and MongoDB are 10.x and up and 2.6 and up
-  respectively.
-
 ## 1.1.3 (2018-10-23)
 
 - New config option (MAX_COMMIT_ITERATIONS) to avoid max commit iterations
@@ -154,9 +25,8 @@
 - Parameter values that cannot be parsed according to the reported type now show
 a warning message.
 - Virtual parameter scripts now use the variable 'args' instead of the special
-  'TIMESTAMPS' and 'VALUES' variables. The content of the args array is:
-  {declare timestamps}, {declare values}, {current timestamps}, {current
-  values}.
+'TIMESTAMPS' and 'VALUES' variables. The content of the args array is: {declare
+timestamps}, {declare values}, {current timestamps}, {current values}.
 - Virtual parameter value types are now inferred from the JavaScript type if the
 returned value attribute is not a value-type pair.
 - Show a fault when a virtual parameter script doesn't return the required
@@ -198,13 +68,12 @@
 - Virtual parameters are user-defined parameters whose values are evaluated from
 a custom script.
 - Extensions are sandboxed Node.js scripts that are accessible from provision
-  and virtual parameter scripts to facilitate integration with external
-  entities.
-- Support for UDP/STUN based connection requests for reaching devices behind NAT
-  (TR-069 Annex G).
+and virtual parameter scripts to facilitate integration with external entities.
+- Support for UDP/STUN based connection requests for reaching devices behind
+NAT (TR-069 Annex G).
 - Presets can now be scheduled using a cron-like expression.
 - Presets can now be tied to specific device events (e.g. boot).
-- Presets precondition queries no longer support "\$or" or other MongoDB logical
+- Presets precondition queries no longer support "$or" or other MongoDB logical
 operators.
 - Faults are no longer a part of tasks but are now first class objects.
 - Presets are now assigned to channels. A fault in one channel only blocks
@@ -221,11 +90,11 @@
 - GenieACS will no longer fetch the entire device data model upon first contact
 but will instead only fetch the parameters it needs to fulfill the presets.
 - Logs have been overhauled and split into two streams: process log (stderr) and
-  access log (stdout). Also added config options to dump logs to files rather
-  than standard streams.
+access log (stdout). Also added config options to dump logs to files rather than
+standard streams.
 - Connection request authentication credentials are picked up from the device
-  data model if available. config/auth.js is still supported as a fallback and
-  now supports an optional callback argument.
+data model if available. config/auth.js is still supported as a fallback and now
+supports an optional callback argument.
 - Custom commands have been removed. Use virtual parameters and/or extensions.
 - Aliases and value normalizers (config/parameters.json) have been removed. Use
 virtual parameters.

CONTRIBUTING.md

@@ -1,21 +1,22 @@
 # Have a question?
 
-Please ask in the [forum] instead of opening a Github Issue. The Github issue
-tracker is strictly for bug reports.
+Please ask in the [Users mailing list][mailing-lists] instead of opening a
+Github Issue. The Github issue tracker is strictly for bug reports.
 
 # Issues
 
-We use GitHub issues to track bugs. Please ensure your bug description is clear
-and has sufficient instructions to be able to reproduce the issue.
+We use GitHub issues to track bugs. Please ensure your bug description is
+clear and has sufficient instructions to be able to reproduce the issue.
 
-If you face interoperability issues with your CPE, more often than not it's a
-device related issue. Please consult the [forum] before submitting an Issue.
+If you face interoperability issues with your device, more often than not it's
+a device related issue. Please consult the [Users mailing list][mailing-lists]
+before submitting an Issue.
 
 # Pull Requests
 
 Pull requests are welcome. If you have a fix for a bug, go ahead and create a
-pull request. For other kinds of enhancements, please discuss in the [forum]
-first.
+pull request. For other kinds of enhancements, please discuss in the [Users
+mailing list][mailing-lists] first.
 
 Please follow the guidelines [here][commit-guidelines] when committing your
 changes.
@@ -26,14 +27,13 @@ By submitting a pull request to this repository, you acknowledge that, while
 maintaining copyright, you grant Zaid Abdulla a perpetual, worldwide, non-
 exclusive, no-charge, royalty-free, irrevocable license to reproduce, prepare
 derivative works of, publicly display, publicly perform, sublicense, and
-distribute your contributions and such derivative works under the AGPLv3 license
-or any other license terms, including, but not limited to, proprietary or
-commercial license terms.
+distribute your contributions and such derivative works under the AGPLv3
+license or any other license terms, including, but not limited to, proprietary
+or commercial license terms.
 
-You confirm that you own or have rights to distribute and sublicense the source
-code contained therein, and that your content does not infringe upon the
-intellectual property rights of a third party.
+You confirm that you own or have rights to distribute and sublicense the
+source code contained therein, and that your content does not infringe upon
+the intellectual property rights of a third party.
 
-[commit-guidelines]:
-  http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
-[forum]: https://forum.genieacs.com
+[commit-guidelines]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
+[mailing-lists]: http://lists.genieacs.com

README.md

@@ -1,10 +1,18 @@
 # GenieACS
 
-GenieACS is a high performance Auto Configuration Server (ACS) for remote
-management of TR-069 enabled devices. It utilizes a declarative and fault
-tolerant configuration engine for automating complex provisioning scenarios at
-scale. It's battle-tested to handle hundreds of thousands and potentially
-millions of concurrent devices.
+GenieACS is a fast and lightweight TR-069 auto configuration server (ACS). This
+is the core back end component. A GUI front end is available at
+https://github.com/genieacs/genieacs-gui.
+
+## Features
+
+- **Battle-tested at scale**: Can scale to manage hundreds of thousands and
+potentially millions of devices
+- **Powerful provisioning system**: Scriptable configuration to handle complex
+device provisioning scenarios and other automated operations such as firmware
+upgrade
+- **Extensive API**: A rich HTTP-based API allows easy integration with other
+systems
 
 ## Quick Start
 
@@ -12,65 +20,61 @@ Install [Node.js](http://nodejs.org/) and [MongoDB](http://www.mongodb.org/).
 Refer to their corresponding documentation for installation instructions. The
 supported versions are:
 
-- Node.js: 12.3+
-- MongoDB: 3.6+
+- Node.js: 6.x and 8.x (8.x recommended)
+- MongoDB: 2.6 through 3.4
 
-Install GenieACS from NPM:
+Install build tools and libxml2 development files from your system's package
+manager.
 
-    sudo npm install -g genieacs
+Then install GenieACS using NPM:
 
-To build from source instead, clone this repo or download the source archive
-then _cd_ into the source directory then run:
+    npm install -g genieacs
 
+Alternatively, you can install from source by cloning the git repository:
+
+    git clone https://github.com/genieacs/genieacs.git
+    cd genieacs
+    git checkout $(git tag -l v1.1.* --sort=-v:refname | head -n 1)
     npm install
-    npm run build
+    npm run compile
 
-Finally, run the following services (found under `./dist/bin/` if building from
-source):
+Before proceeding, find and review the file "config.json" in "config" directory
+where GenieACS is downloaded.
 
-### genieacs-cwmp
+Finally, run the following (from bin directory if installing from source):
 
-This is the service that the CPEs will communicate with. It listens on port 7547
-by default. Configure the ACS URL in your devices accordingly.
+    genieacs-cwmp
 
-You may optionally use [genieacs-sim](https://github.com/genieacs/genieacs-sim)
-as a dummy TR-069 simulator if you don't have a CPE at hand.
+This is the service that the CPEs will communicate with. It listens to port 7547
+by default (see config/config.json). Configure the ACS URL of your devices
+accordingly.
 
-### genieacs-nbi
+    genieacs-nbi
 
 This is the northbound interface module. It exposes a REST API on port 7557 by
-default. This one is only required if you have an external system integrating
-with GenieACS using this API.
+default. This must be running for the GUI front end to work.
 
-### genieacs-fs
+    genieacs-fs
 
 This is the file server from which the CPEs will download firmware images and
-such. It listens on port 7567 by default.
+such.
 
-### genieacs-ui
+Note: For production deployment make sure to run those as background services.
 
-This serves the web based user interface. It listens on port 3000 by default.
-You must pass _--ui-jwt-secret_ argument to supply the secret key used for
-signing browser cookies:
-
-    genieacs-ui --ui-jwt-secret secret
-
-The UI has plenty of configuration options. When you open GenieACS's UI in a
-browser you'll be greeted with a database initialization wizard to help you
-populate some initial configuration.
-
-Visit [docs.genieacs.com](https://docs.genieacs.com) for more documentation and
-a complete installation guide for production deployments.
+For further details about installation and configuration, refer to the
+[wiki section](https://github.com/genieacs/genieacs/wiki).
 
 ## Support
 
-The [forum](https://forum.genieacs.com) is a good place to get guidance and help
-from the community. Head on over and join the conversation!
+The [Users mailing list](http://lists.genieacs.com) is a good place to get
+guidance and help from the community. Head on over and join the conversation!
+In addition, the [wiki](https://github.com/genieacs/genieacs/wiki) provides useful
+documentation and tips from GenieACS users.
 
-For commercial support options, please visit
-[genieacs.com](https://genieacs.com/support/).
+For commercial support options and professional services, please visit
+[genieacs.com](https://genieacs.com).
 
 ## License
 
-Copyright 2013-2020 GenieACS Inc. GenieACS is released under the
+Copyright 2013-2018 GenieACS Inc. GenieACS is released under the
 [AGPLv3 license terms](https://raw.githubusercontent.com/genieacs/genieacs/master/LICENSE).

ava.config.js

@@ -1,4 +0,0 @@
-export default {
-  extensions: ["ts"],
-  require: ["esm", "ts-node/register/transpile-only"],
-};

bin/genieacs-cwmp

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+var path = require('path');
+var fs = require('fs');
+
+var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
+process.chdir(dir);
+
+var cluster = require(path.resolve(dir, 'lib/cluster'));
+cluster.start('cwmp');

bin/genieacs-cwmp.ts

@@ -1,122 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import * as cwmp from "../lib/cwmp";
-import * as db from "../lib/db";
-import * as extensions from "../lib/extensions";
-import { version as VERSION } from "../package.json";
-
-logger.init("cwmp", VERSION);
-
-const SERVICE_ADDRESS = config.get("CWMP_INTERFACE") as string;
-const SERVICE_PORT = config.get("CWMP_PORT") as number;
-
-function exitWorkerGracefully(): void {
-  setTimeout(exitWorkerUngracefully, 5000).unref();
-  Promise.all([
-    db.disconnect(),
-    extensions.killAll(),
-    cluster.worker.disconnect(),
-  ]).catch(exitWorkerUngracefully);
-}
-
-function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
-    process.exit(1);
-  });
-}
-
-if (!cluster.worker) {
-  const WORKER_COUNT = config.get("CWMP_WORKER_PROCESSES") as number;
-
-  logger.info({
-    message: `genieacs-cwmp starting`,
-    pid: process.pid,
-    version: VERSION,
-  });
-
-  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
-
-  process.on("SIGINT", () => {
-    logger.info({
-      message: "Received signal SIGINT, exiting",
-      pid: process.pid,
-    });
-
-    cluster.stop();
-  });
-
-  process.on("SIGTERM", () => {
-    logger.info({
-      message: "Received signal SIGTERM, exiting",
-      pid: process.pid,
-    });
-
-    cluster.stop();
-  });
-} else {
-  const key = config.get("CWMP_SSL_KEY") as string;
-  const cert = config.get("CWMP_SSL_CERT") as string;
-
-  const options = {
-    port: SERVICE_PORT,
-    host: SERVICE_ADDRESS,
-    ssl: key && cert ? { key, cert } : null,
-    onConnection: cwmp.onConnection,
-    timeout: 30000,
-    keepAliveTimeout: 0,
-  };
-
-  process.on("uncaughtException", (err) => {
-    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
-    logger.error({
-      message: "Uncaught exception",
-      exception: err,
-      pid: process.pid,
-    });
-    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-  });
-
-  const initPromise = db
-    .connect()
-    .then(() => {
-      server.start(options, cwmp.listener);
-    })
-    .catch((err) => {
-      setTimeout(() => {
-        throw err;
-      });
-    });
-
-  process.on("SIGINT", () => {
-    initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-    });
-  });
-
-  process.on("SIGTERM", () => {
-    initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-    });
-  });
-}

bin/genieacs-fs

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+var path = require('path');
+var fs = require('fs');
+
+var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
+process.chdir(dir);
+
+var cluster = require(path.resolve(dir, 'lib/cluster'));
+cluster.start('fs');

bin/genieacs-fs.ts

@@ -1,124 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/fs";
-import * as db from "../lib/db";
-import { version as VERSION } from "../package.json";
-
-logger.init("fs", VERSION);
-
-const SERVICE_ADDRESS = config.get("FS_INTERFACE") as string;
-const SERVICE_PORT = config.get("FS_PORT") as number;
-
-function exitWorkerGracefully(): void {
-  setTimeout(exitWorkerUngracefully, 5000).unref();
-  Promise.all([db.disconnect(), cluster.worker.disconnect()]).catch(
-    exitWorkerUngracefully
-  );
-}
-
-function exitWorkerUngracefully(): void {
-  process.exit(1);
-}
-
-if (!cluster.worker) {
-  const WORKER_COUNT = config.get("FS_WORKER_PROCESSES") as number;
-
-  logger.info({
-    message: `genieacs-fs starting`,
-    pid: process.pid,
-    version: VERSION,
-  });
-
-  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
-
-  process.on("SIGINT", () => {
-    logger.info({
-      message: "Received signal SIGINT, exiting",
-      pid: process.pid,
-    });
-
-    cluster.stop();
-  });
-
-  process.on("SIGTERM", () => {
-    logger.info({
-      message: "Received signal SIGTERM, exiting",
-      pid: process.pid,
-    });
-
-    cluster.stop();
-  });
-} else {
-  const key = config.get("FS_SSL_KEY") as string;
-  const cert = config.get("FS_SSL_CERT") as string;
-  const options = {
-    port: SERVICE_PORT,
-    host: SERVICE_ADDRESS,
-    ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
-  };
-
-  let stopping = false;
-
-  process.on("uncaughtException", (err) => {
-    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
-    logger.error({
-      message: "Uncaught exception",
-      exception: err,
-      pid: process.pid,
-    });
-    stopping = true;
-    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-  });
-
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    void listener(req, res);
-  };
-
-  const initPromise = db
-    .connect()
-    .then(() => {
-      server.start(options, _listener);
-    })
-    .catch((err) => {
-      setTimeout(() => {
-        throw err;
-      });
-    });
-
-  process.on("SIGINT", () => {
-    stopping = true;
-    initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-    });
-  });
-
-  process.on("SIGTERM", () => {
-    stopping = true;
-    initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-    });
-  });
-}

bin/genieacs-nbi

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+var path = require('path');
+var fs = require('fs');
+
+var dir = path.resolve(path.dirname(fs.realpathSync(__filename)), '..');
+process.chdir(dir);
+
+var cluster = require(path.resolve(dir, 'lib/cluster'));
+cluster.start('nbi');

bin/genieacs-nbi.ts

@@ -1,129 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/nbi";
-import * as db from "../lib/db";
-import * as extensions from "../lib/extensions";
-import { version as VERSION } from "../package.json";
-
-logger.init("nbi", VERSION);
-
-const SERVICE_ADDRESS = config.get("NBI_INTERFACE") as string;
-const SERVICE_PORT = config.get("NBI_PORT") as number;
-
-function exitWorkerGracefully(): void {
-  setTimeout(exitWorkerUngracefully, 5000).unref();
-  Promise.all([
-    db.disconnect(),
-    extensions.killAll(),
-    cluster.worker.disconnect(),
-  ]).catch(exitWorkerUngracefully);
-}
-
-function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
-    process.exit(1);
-  });
-}
-
-if (!cluster.worker) {
-  const WORKER_COUNT = config.get("NBI_WORKER_PROCESSES") as number;
-
-  logger.info({
-    message: `genieacs-nbi starting`,
-    pid: process.pid,
-    version: VERSION,
-  });
-
-  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
-
-  process.on("SIGINT", () => {
-    logger.info({
-      message: "Received signal SIGINT, exiting",
-      pid: process.pid,
-    });
-
-    cluster.stop();
-  });
-
-  process.on("SIGTERM", () => {
-    logger.info({
-      message: "Received signal SIGTERM, exiting",
-      pid: process.pid,
-    });
-
-    cluster.stop();
-  });
-} else {
-  const key = config.get("NBI_SSL_KEY") as string;
-  const cert = config.get("NBI_SSL_CERT") as string;
-  const options = {
-    port: SERVICE_PORT,
-    host: SERVICE_ADDRESS,
-    ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
-  };
-
-  let stopping = false;
-
-  process.on("uncaughtException", (err) => {
-    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
-    logger.error({
-      message: "Uncaught exception",
-      exception: err,
-      pid: process.pid,
-    });
-    stopping = true;
-    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-  });
-
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
-  };
-
-  const initPromise = db
-    .connect()
-    .then(() => {
-      server.start(options, _listener);
-    })
-    .catch((err) => {
-      setTimeout(() => {
-        throw err;
-      });
-    });
-
-  process.on("SIGINT", () => {
-    stopping = true;
-    initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-    });
-  });
-
-  process.on("SIGTERM", () => {
-    stopping = true;
-    initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-    });
-  });
-}

bin/genieacs-ui.ts

@@ -1,129 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as config from "../lib/config";
-import * as logger from "../lib/logger";
-import * as cluster from "../lib/cluster";
-import * as server from "../lib/server";
-import { listener } from "../lib/ui";
-import * as extensions from "../lib/extensions";
-import * as db from "../lib/db";
-import { version as VERSION } from "../package.json";
-
-logger.init("ui", VERSION);
-
-const SERVICE_ADDRESS = config.get("UI_INTERFACE") as string;
-const SERVICE_PORT = config.get("UI_PORT") as number;
-
-function exitWorkerGracefully(): void {
-  setTimeout(exitWorkerUngracefully, 5000).unref();
-  Promise.all([
-    db.disconnect(),
-    extensions.killAll(),
-    cluster.worker.disconnect(),
-  ]).catch(exitWorkerUngracefully);
-}
-
-function exitWorkerUngracefully(): void {
-  extensions.killAll().finally(() => {
-    process.exit(1);
-  });
-}
-
-if (!cluster.worker) {
-  const WORKER_COUNT = config.get("UI_WORKER_PROCESSES") as number;
-
-  logger.info({
-    message: `genieacs-ui starting`,
-    pid: process.pid,
-    version: VERSION,
-  });
-
-  cluster.start(WORKER_COUNT, SERVICE_PORT, SERVICE_ADDRESS);
-
-  process.on("SIGINT", () => {
-    logger.info({
-      message: "Received signal SIGINT, exiting",
-      pid: process.pid,
-    });
-
-    cluster.stop();
-  });
-
-  process.on("SIGTERM", () => {
-    logger.info({
-      message: "Received signal SIGTERM, exiting",
-      pid: process.pid,
-    });
-
-    cluster.stop();
-  });
-} else {
-  const key = config.get("UI_SSL_KEY") as string;
-  const cert = config.get("UI_SSL_CERT") as string;
-  const options = {
-    port: SERVICE_PORT,
-    host: SERVICE_ADDRESS,
-    ssl: key && cert ? { key, cert } : null,
-    timeout: 30000,
-  };
-
-  let stopping = false;
-
-  process.on("uncaughtException", (err) => {
-    if ((err as NodeJS.ErrnoException).code === "ERR_IPC_DISCONNECTED") return;
-    logger.error({
-      message: "Uncaught exception",
-      exception: err,
-      pid: process.pid,
-    });
-    stopping = true;
-    server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-  });
-
-  const _listener = (req, res): void => {
-    if (stopping) res.setHeader("Connection", "close");
-    listener(req, res);
-  };
-
-  const initPromise = db
-    .connect()
-    .then(() => {
-      server.start(options, _listener);
-    })
-    .catch((err) => {
-      setTimeout(() => {
-        throw err;
-      });
-    });
-
-  process.on("SIGINT", () => {
-    stopping = true;
-    initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-    });
-  });
-
-  process.on("SIGTERM", () => {
-    stopping = true;
-    initPromise.finally(() => {
-      server.stop().then(exitWorkerGracefully).catch(exitWorkerUngracefully);
-    });
-  });
-}

build/build.ts

@@ -1,335 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as path from "path";
-import * as fs from "fs";
-import { rollup, WarningHandler } from "rollup";
-import rollupJson from "@rollup/plugin-json";
-import typescript from "@rollup/plugin-typescript";
-import { terser } from "rollup-plugin-terser";
-import nodeResolve from "@rollup/plugin-node-resolve";
-import commonjs from "@rollup/plugin-commonjs";
-import postcss from "postcss";
-import postcssImport from "postcss-import";
-import postcssPresetEnv from "postcss-preset-env";
-import cssnano from "cssnano";
-import { optimize, extendDefaultPlugins } from "svgo";
-import * as xmlParser from "../lib/xml-parser";
-
-const MODE = process.env["NODE_ENV"] || "production";
-
-const BUILD_METADATA = new Date()
-  .toISOString()
-  .split(".")[0]
-  .replace(/[^0-9]/g, "");
-
-const INPUT_DIR = path.resolve(__dirname, "..");
-const OUTPUT_DIR = path.resolve(__dirname, "../dist");
-
-const builtins = [
-  "path",
-  "fs",
-  "cluster",
-  "os",
-  "tls",
-  "http",
-  "https",
-  "zlib",
-  "crypto",
-  "util",
-  "vm",
-  "querystring",
-  "child_process",
-  "dgram",
-  "url",
-  "readline",
-  "stream",
-];
-
-function rmDirSync(dirPath): void {
-  if (!fs.existsSync(dirPath)) return;
-  const files = fs.readdirSync(dirPath);
-
-  for (const file of files) {
-    const filePath = `${dirPath}/${file}`;
-    if (fs.statSync(filePath).isFile()) fs.unlinkSync(filePath);
-    else rmDirSync(filePath);
-  }
-  fs.rmdirSync(dirPath);
-}
-
-function stripDevDeps(deps): void {
-  if (!deps["dependencies"]) return;
-  for (const [k, v] of Object.entries(deps["dependencies"])) {
-    if (v["dev"]) delete deps["dependencies"][k];
-    else stripDevDeps(v);
-  }
-  if (!Object.keys(deps["dependencies"]).length) delete deps["dependencies"];
-}
-
-function xmlTostring(xml): string {
-  const children = [];
-  for (const c of xml.children || []) children.push(xmlTostring(c));
-
-  return xml.name === "root" && xml.bodyIndex === 0
-    ? children.join("")
-    : `<${xml.name} ${xml.attrs}>${children.join("")}</${xml.name}>`;
-}
-
-function generateSymbol(id: string, svgStr: string): string {
-  const xml = xmlParser.parseXml(svgStr);
-  const svg = xml.children[0];
-  const svgAttrs = xmlParser.parseAttrs(svg.attrs);
-  let viewBox = "";
-  for (const a of svgAttrs) {
-    if (a.name === "viewBox") {
-      viewBox = `viewBox="${a.value}"`;
-      break;
-    }
-  }
-  const symbolBody = xml.children[0].children
-    .map((c) => xmlTostring(c))
-    .join("");
-  return `<symbol id="icon-${id}" ${viewBox}>${symbolBody}</symbol>`;
-}
-
-async function init(): Promise<string[]> {
-  // Delete any old output directory
-  rmDirSync(OUTPUT_DIR);
-
-  // Create output directory layout
-  fs.mkdirSync(OUTPUT_DIR);
-  fs.mkdirSync(OUTPUT_DIR + "/bin");
-  fs.mkdirSync(OUTPUT_DIR + "/public");
-
-  // Create package.json
-  const packageJson = JSON.parse(
-    fs.readFileSync(path.resolve(INPUT_DIR, "package.json")).toString()
-  );
-  delete packageJson["devDependencies"];
-  delete packageJson["private"];
-  packageJson["scripts"] = {
-    install: packageJson["scripts"].install,
-    configure: packageJson["scripts"].configure,
-  };
-  packageJson["version"] = `${packageJson["version"]}+${BUILD_METADATA}`;
-  fs.writeFileSync(
-    path.resolve(OUTPUT_DIR, "package.json"),
-    JSON.stringify(packageJson, null, 2)
-  );
-
-  // Create npm-shrinkwrap.json
-  const npmShrinkwrapJson = JSON.parse(
-    fs.readFileSync(path.resolve(INPUT_DIR, "npm-shrinkwrap.json")).toString()
-  );
-  npmShrinkwrapJson["version"] = packageJson["version"];
-  stripDevDeps(npmShrinkwrapJson);
-  fs.writeFileSync(
-    path.resolve(OUTPUT_DIR, "npm-shrinkwrap.json"),
-    JSON.stringify(npmShrinkwrapJson, null, 2)
-  );
-  return Object.keys(packageJson["dependencies"]);
-}
-
-async function copyStatic(): Promise<void> {
-  const files = [
-    "LICENSE",
-    "README.md",
-    "CHANGELOG.md",
-    "public/logo.svg",
-    "public/favicon.png",
-  ];
-
-  for (const file of files) {
-    fs.copyFileSync(
-      path.resolve(INPUT_DIR, file),
-      path.resolve(OUTPUT_DIR, file)
-    );
-  }
-}
-
-async function generateCss(): Promise<void> {
-  const cssInPath = path.resolve(INPUT_DIR, "ui/css/app.css");
-  const cssOutPath = path.resolve(OUTPUT_DIR, "public/app.css");
-  const cssIn = fs.readFileSync(cssInPath);
-  const cssOut = await postcss([
-    postcssImport,
-    postcssPresetEnv({
-      stage: 3,
-      features: {
-        "nesting-rules": true,
-        "color-mod-function": true,
-      },
-    }),
-    cssnano,
-  ]).process(cssIn, { from: cssInPath, to: cssOutPath });
-  fs.writeFileSync(cssOutPath, cssOut.css);
-}
-
-async function generateBackendJs(externals: string[]): Promise<void> {
-  for (const bin of [
-    "genieacs-cwmp",
-    "genieacs-ext",
-    "genieacs-nbi",
-    "genieacs-fs",
-    "genieacs-ui",
-  ]) {
-    const inputFile = path.resolve(INPUT_DIR, `bin/${bin}.ts`);
-    const outputFile = path.resolve(OUTPUT_DIR, `bin/${bin}`);
-    const bundle = await rollup({
-      input: inputFile,
-      external: [...builtins, ...externals],
-      acorn: {
-        allowHashBang: true,
-      },
-      treeshake: {
-        propertyReadSideEffects: false,
-        moduleSideEffects: false,
-      },
-      plugins: [
-        rollupJson({ preferConst: true }),
-        {
-          name: "",
-          resolveId: (importee, importer) => {
-            if (importee.endsWith("/package.json")) {
-              const p = path.resolve(path.dirname(importer), importee);
-              if (p === path.resolve(INPUT_DIR, "package.json"))
-                return path.resolve(OUTPUT_DIR, "package.json");
-            }
-            return null;
-          },
-        },
-        typescript({
-          tsconfig: "./tsconfig.json",
-          include: [`bin/${bin}.ts`, "lib/**/*.ts"],
-        }),
-        MODE === "production" ? terser() : null,
-      ],
-    });
-
-    await bundle.write({
-      format: "cjs",
-      preferConst: true,
-      sourcemap: "inline",
-      sourcemapExcludeSources: true,
-      banner: "#!/usr/bin/env node",
-      file: outputFile,
-    });
-
-    // Mark as executable
-    const mode = fs.statSync(outputFile).mode;
-    fs.chmodSync(outputFile, mode | 73);
-  }
-}
-
-async function generateFrontendJs(externals: string[]): Promise<void> {
-  const inputFile = path.resolve(INPUT_DIR, "ui/app.ts");
-  const outputDir = path.resolve(OUTPUT_DIR, "public");
-
-  const inlineDeps = ["parsimmon", "espresso-iisojs"];
-  const bundle = await rollup({
-    input: inputFile,
-    external: [
-      ...builtins,
-      ...externals.filter((e) => !inlineDeps.includes(e)),
-    ],
-    plugins: [
-      rollupJson({ preferConst: true }),
-      {
-        name: "",
-        resolveId: function (importee, importer) {
-          if (importee.endsWith("/package.json")) {
-            const p = path.resolve(path.dirname(importer), importee);
-            if (p === path.resolve(INPUT_DIR, "package.json"))
-              return path.resolve(OUTPUT_DIR, "package.json");
-          }
-          return null;
-        },
-      },
-      typescript({ tsconfig: "./tsconfig.json" }),
-      nodeResolve(),
-      commonjs(),
-      MODE === "production" ? terser() : null,
-    ],
-    preserveEntrySignatures: false,
-    treeshake: {
-      propertyReadSideEffects: false,
-      moduleSideEffects: false,
-    },
-    onwarn: ((warning, warn) => {
-      // Ignore circular dependency warnings
-      if (warning.code !== "CIRCULAR_DEPENDENCY") warn(warning);
-    }) as WarningHandler,
-  });
-
-  await bundle.write({
-    manualChunks: (id) => {
-      if (id.includes("node_modules/codemirror")) return "codemirror";
-      else if (id.includes("node_modules/yaml")) return "yaml";
-      return "app";
-    },
-    preferConst: true,
-    format: "es",
-    sourcemap: true,
-    sourcemapExcludeSources: true,
-    dir: outputDir,
-  });
-}
-
-async function generateIconsSprite(): Promise<void> {
-  const symbols = [];
-  const iconsDir = path.resolve(INPUT_DIR, "ui/icons");
-  for (const file of fs.readdirSync(iconsDir)) {
-    const id = path.parse(file).name;
-    const filePath = path.join(iconsDir, file);
-    const { data } = await optimize(fs.readFileSync(filePath).toString(), {
-      plugins: extendDefaultPlugins([
-        {
-          name: "removeViewBox",
-          active: false,
-        },
-      ]),
-    });
-    symbols.push(generateSymbol(id, data));
-  }
-  fs.writeFileSync(
-    path.resolve(OUTPUT_DIR, "public/icons.svg"),
-    `<svg xmlns="http://www.w3.org/2000/svg">${symbols.join("")}</svg>`
-  );
-}
-
-init()
-  .then((externals) => {
-    Promise.all([
-      copyStatic(),
-      generateCss(),
-      generateIconsSprite(),
-      generateBackendJs(externals),
-      generateFrontendJs(externals),
-    ])
-      .then(() => {
-        // Ignore
-      })
-      .catch((err) => {
-        process.stderr.write(err.stack + "\n");
-      });
-  })
-  .catch((err) => {
-    process.stderr.write(err.stack + "\n");
-  });

build/spellcheck-dict.pws

@@ -1,167 +0,0 @@
-personal_ws-1.1 en 0
-genieacs
-GenieACS
-javascript
-sudo
-config
-cwmp
-CPE
-systemctl
-nbi
-CWMP
-NBI
-ACS
-SSL
-xsd
-args
-DeviceID
-GENIEACS
-env
-param
-arg
-AUTH
-igd
-TCP
-toctree
-auth
-EnvironmentFile
-JSON
-json
-parameterValues
-setParameterValues
-stderr
-stdout
-attr
-cfg
-CPEs
-ExecStart
-faq
-FileType
-GenieACS's
-JWT
-maxdepth
-npm
-objectName
-refreshObject
-SerialNumber
-systemd
-usr
-WantedBy
-yaml
-chown
-Config
-getParameterValues
-https
-InternetGatewayDevice
-mkdir
-OUI
-sql
-TODO
-UDP
-addObject
-AGPLv
-const
-cpe
-cron
-deleteObject
-FactoryReset
-factoryReset
-FileName
-fileType
-Github
-hostname
-HTTPS
-jwt
-latlong
-NPM
-oui
-parameterNames
-ProductClass
-productClass
-repo
-SSID
-sublicense
-TLS
-WANIPConnection
-WPA
-YourProvisionName
-abc
-Abdulla
-accessors
-AddObject
-AddressingType
-api
-APIs
-chmod
-cwmp's
-dateext
-DATETIME
-dateTime
-datetime
-declaratively
-delaycompress
-DeleteObject
-deviceId
-DHCP
-dir
-distro
-equisatisfiable
-ExternalIPAddress
-failover
-favor
-func
-GetParameterAttributes
-getPassword
-gte
-gui
-hexBinary
-Hils
-HOSTNAME
-init
-journalctl
-journald
-LastFileName
-LastFileType
-lastInform
-LIBXMLJS
-libxmljs
-literalinclude
-lockfile
-logrotate
-lte
-mipsbe
-normalizers
-pre
-quickstart
-README
-redis
-reimplemented
-RequestDownload
-RESTful
-rollout
-sandboxed
-scalable
-SetParameterAttributes
-SetParamteerValues
-SIGINT
-SIGTERM
-skipWritableCheck
-stringify
-subdirectory
-TargetFileName
-useradd
-VirtualParameters
-vparam
-xml
-YAML
-Zaid
-cacheExpire
-iss
-http
-statusCode
-rawData
-pos
-Brotli
-CSV
-hostInfo
-IPv

build/spellcheck.sh

@@ -1,12 +0,0 @@
-#! /bin/sh
-
-cd "$(dirname "$0")"
-
-FILES=`ls ../docs/*.rst ../docs/*.js ../*.md`
-
-for FILE in $FILES
-do
-    echo $FILE
-    cat $FILE | aspell list --lang=en --add-extra-dicts=./spellcheck-dict.pws --ignore 2
-    echo
-done

config/auth-sample.js

@@ -0,0 +1,7 @@
+"use strict";
+
+function connectionRequest(deviceId, url, username, password, callback) {
+  return callback(username || deviceId, password || "");
+}
+
+exports.connectionRequest = connectionRequest;

config/config-sample.json

@@ -0,0 +1,12 @@
+{
+  "MONGODB_CONNECTION_URL" : "mongodb://127.0.0.1/genieacs",
+  "CWMP_INTERFACE" : "0.0.0.0",
+  "CWMP_PORT" : 7547,
+  "CWMP_SSL" : false,
+  "NBI_INTERFACE" : "0.0.0.0",
+  "NBI_PORT" : 7557,
+  "FS_INTERFACE" : "0.0.0.0",
+  "FS_PORT" : 7567,
+  "FS_HOSTNAME" : "acs.example.com",
+  "DEBUG" : false
+}

config/ext-sample.js

@@ -0,0 +1,34 @@
+// This is an exmaple GenieACS extension to get the current latitude/longitude
+// of the International Space Station. Why, you ask? Because why not.
+// To install, copy this file to config/ext/iss.js.
+
+"use strict";
+
+const http = require("http");
+
+let cache = null;
+let cacheExpire = 0;
+
+function latlong(args, callback) {
+  if (Date.now() < cacheExpire)
+    return callback(null, cache);
+
+  http.get("http://api.open-notify.org/iss-now.json", (res) => {
+    if (res.statusCode !== 200)
+      return callback(new Error(`Request failed (status code: ${res.statusCode})`));
+
+    let rawData = "";
+    res.on("data", (chunk) => rawData += chunk);
+
+    res.on("end", () => {
+      let pos = JSON.parse(rawData)["iss_position"];
+      cache = [+pos["latitude"], +pos["longitude"]];
+      cacheExpire = Date.now() + 10000;
+      callback(null, cache);
+    });
+  }).on("error", (err) => {
+    callback(err);
+  });
+}
+
+exports.latlong = latlong;

docs/administration-faq.rst

@@ -1,64 +0,0 @@
-.. _administration-faq:
-
-Administration FAQ
-==================
-
-.. _administration-faq-duplicate-log-entries:
-
-Duplicate log entries when using :func:`log()` function
--------------------------------------------------------
-
-Because GenieACS uses a full fledged scripting language for device
-configuration, the only way to guarantee that it has satisfied the 'desired
-state' is by repeatedly executing the script until there's no more
-discrepancies with the current device state. Though it may seem like this will
-cause duplicate requests going to the device, this isn't actually the case
-because device configuration are stated declaratively and that the scripts
-themselves are pure functions in the context of a session (e.g. Date.now()
-always returns the same value within the session).
-
-To illustrate with an example, consider the following script:
-
-.. code:: javascript
-
-  log("Executing script");
-  declare("Device.param", null, {value: 1});
-  commit();
-  declare("Device.param", null, {value: 2});
-
-This will set the value of the 'Device.param' to 1, then to 2. Then as the
-script is run again the value is set back to 1 and so on. A stable state will
-never be reached so GenieACS will execute the script a few times until it gives
-up and throws a fault. This is an edge case that should be avoided. A more
-typical case is where the script is run once or twice. Essentially if an
-execution doesn't result in any request to the CPE or a change in the data
-model then a stable state is deemed to have been reached.
-
-Configurations not pushed to device after factory reset
----------------------------------------------------------
-
-After a device is reset to its factory default state, the cached data model in
-GenieACS's database needs to be invalidated to force rediscovery. Ensure the
-following lines are called on ``0 BOOTSTRAP`` event:
-
-.. code:: javascript
-
-  const now = Date.now();
-
-  // Clear cached data model to force a refresh
-  clear("Device", now);
-  clear("InternetGatewayDevice", now);
-
-
-Most device parameters are missing
-----------------------------------
-
-For performance reasons (server, client, and network), GenieACS by default only
-fetches parts of the data model that are necessary to satisfy the declarations
-in your provision scripts. Create declarations for any parameters you need
-fetched by default.
-
-If you're unsure and want to explore the available parameters exposed by the
-device, refresh the root parameter (e.g. ``InternetGatewayDevice``) from
-GenieACS's UI. You typically only need to do that one time for a given CPE
-model.

docs/api-reference.rst

@@ -1,538 +0,0 @@
-API Reference
-=============
-
-GenieACS exposes a rich RESTful API through its NBI component. This document
-serves as a reference for the available APIs.
-
-This API makes use of MongoDB's query language in some of its endpoints. Refer
-to MongoDB's documentation for details.
-
-.. note::
-
-  The examples below use ``curl`` command for simplicity and ease of testing.
-  Query parameters are URL-encoded, but the original pre-encoding values are
-  shown for reference. These examples assume genieacs-nbi is running locally
-  and listening on the default NBI port (7557).
-
-.. warning::
-
-  A common pitfall is not properly percent-encoding special characters in the
-  device ID or query in the URL.
-
-Endpoints
----------
-
-GET /\<collection\>/?query=\<query\>
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Search for records in the database (e.g. devices, tasks, presets, files).
-Returns a JSON representation of all items in the given collection that match
-the search criteria.
-
-*collection*: The data collection to search. Could be one of: tasks, devices,
-presets, objects.
-
-*query*: Search query. Refer to MongoDB queries for reference.
-
-Examples
-^^^^^^^^
-
-- Find a device by its ID:
-
-.. code:: javascript
-
-  query = {"_id": "202BC1-BM632w-000000"}
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D'
-
-- Find a device by its MAC address:
-
-.. code:: javascript
-
-  query = {
-    "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress": "20:2B:C1:E0:06:65"
-  }
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/?query=%7B%22InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress%22%3A%2220:2B:C1:E0:06:65%22%7D'
-
-- Search for devices that have not initiated an inform in the last 7 days.
-
-.. code:: javascript
-
-  query = {
-    "_lastInform": {
-      "$lt" : "2017-12-11 13:16:23 +0000"
-    }
-  }
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/?query=%7B%22_lastInform%22%3A%7B%22%24lt%22%3A%222017-12-11%2013%3A16%3A23%20%2B0000%22%7D%7D'
-
-- Show pending tasks for a given device:
-
-.. code:: javascript
-
-  query = {"device": "202BC1-BM632w-000000"}
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/tasks/?query=%7B%22device%22%3A%22202BC1-BM632w-000000%22%7D'
-
-- Return specific parameters for a given device:
-
-.. code:: javascript
-
-  query = {"_id": "202BC1-BM632w-000000"}
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices?query=%7B%22_id%22%3A%22202BC1-BM632w-000000%22%7D&projection=InternetGatewayDevice.DeviceInfo.ModelName,InternetGatewayDevice.DeviceInfo.Manufacturer'
-
-The ``projection`` URL param is a comma-separated list of the parameters to receive.
-
-POST /devices/\<device_id\>/tasks?[connection_request]
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Enqueue task(s) and optionally trigger a connection request to the device.
-Refer to :ref:`tasks` section for information about the task object format.
-Returns status code 200 if the tasks have been successfully executed, and 202
-if the tasks have been queued to be executed at the next inform.
-
-*device_id*: The ID of the device.
-
-*connection_request*: Indicates that a connection request will be triggered to
-execute the tasks immediately. Otherwise, the tasks will be queued and be
-processed at the next inform.
-
-The response body is the task object as it is inserted in the database. The
-object will include ``_id`` property which you can use to look up the task
-later.
-
-Examples
-^^^^^^^^
-
-- Refresh all device parameters now:
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
-  -X POST \
-  --data '{"name": "refreshObject", "objectName": ""}'
-
-- Change WiFi SSID and password:
-
-.. code:: javascript
-
-  {
-    "name": "setParameterValues",
-    "parameterValues": [
-      ["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],
-      ["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]
-    ]
-  }
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tasks?connection_request' \
-  -X POST \
-  --data '{"name":"setParameterValues", "parameterValues": [["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID", "GenieACS", "xsd:string"],["InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.PreSharedKey.1.PreSharedKey", "hello world", "xsd:string"]]}'
-
-POST /tasks/\<task_id\>/retry
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Retry a faulty task at the next inform.
-
-*task_id*: The ID of the task as returned by 'GET /tasks' request.
-
-Example
-^^^^^^^
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/tasks/5403908ef28ea3a25c138adc/retry' -X POST
-
-DELETE /tasks/\<task_id\>
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Delete the given task.
-
-*task_id*: The ID of the task as returned by 'GET /tasks' request.
-
-Example
-^^^^^^^
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/tasks/5403908ef28ea3a25c138adc' -X DELETE
-
-DELETE /faults/\<fault_id\>
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Delete the given fault.
-
-*fault_id*: The ID of the fault as returned by 'GET /faults' request. The ID
-format is "\<device_id\>:\<channel\>".
-
-Example
-^^^^^^^
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/faults/202BC1-BM632w-000000:default' -X DELETE
-
-DELETE /devices/\<device_id\>
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Delete the given device from the database.
-
-Example
-^^^^^^^
-
-.. code:: bash
-
-  curl -X DELETE -i 'http://localhost:7557/devices/202BC1-BM632w-000001'
-
-.. note::
-
-  Note that the device will be registered again when/if it contacts the ACS
-  again (e.g. on the next periodic inform).
-
-POST /devices/\<device_id\>/tags/\<tag\>
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Assign a tag to a device. Has no effect if such tag already exists.
-
-*device_id*: The ID of the device.
-
-*tag*: The tag to be assigned.
-
-Example
-^^^^^^^
-
-Assign the tag "testing" to a device:
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X POST
-
-DELETE /devices/\<device_id\>/tags/\<tag\>
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Remove a tag from a device.
-
-*device_id*: The ID of the device.
-
-*tag*: The tag to be removed.
-
-Example
-^^^^^^^
-
-Remove the tag "testing" from a device:
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/202BC1-BM632w-000000/tags/testing' -X DELETE
-
-PUT /presets/\<preset_name\>
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Create or update a preset. Returns status code 200 if the preset has been
-added/updated successfully. The body of the request is a JSON representation of
-the preset. Refer to :ref:`presets` section below for details about its format.
-
-*preset_name*: The name of the preset.
-
-Example
-^^^^^^^
-
-Create a preset to set 5 minutes inform interval for all devices tagged with
-"test":
-
-.. code:: javascript
-
-  query = {
-    "weight": 0,
-    "precondition": "{\"_tags\": \"test\"}"
-    "configurations": [
-      {
-        "type": "value",
-        "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable",
-        "value": "true"
-      },
-      {
-        "type": "value",
-        "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval",
-        "value": "300"
-      }
-    ]
-  }
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/presets/inform' \
-  -X PUT \
-  --data '{"weight": 0, "precondition": "{\"_tags\": \"test\"}", "configurations": [{"type": "value", "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable", "value": "true"}, {"type": "value", "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval", "value": "300"}]}'
-
-DELETE /presets/\<preset_name\>
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. code:: bash
-
-	curl -i 'http://localhost:7557/presets/inform' -X DELETE
-
-PUT /files/\<file_name\>
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-Upload a new file or overwrite an existing one. Returns status code 200 if the
-file has been added/updated successfully. The file content should be sent as
-the request body.
-
-*file_name*: The name of the uploaded file.
-
-The following file metadata may be sent as request headers:
-
-- ``fileType``: For firmware images it should be "1 Firmware Upgrade Image".
-  Other common types are "2 Web Content" and "3 Vendor Configuration File".
-
-- ``oui``: The OUI of the device model that this file belongs to.
-
-- ``productClass``: The product class of the device.
-
-- ``version``: In case of firmware images, this refer to the firmware version.
-
-Example
-^^^^^^^
-
-Upload a firmware image file:
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/files/new_firmware_v1.0.bin' \
-  -X PUT \
-  --data-binary @"./new_firmware_v1.0.bin" \
-  --header "fileType: 1 Firmware Upgrade Image" \
-  --header "oui: 123456" \
-  --header "productClass: ABC" \
-  --header "version: 1.0"
-
-DELETE /files/\<file_name\>
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Delete a previously uploaded file:
-
-.. code:: bash
-
-	curl -i 'http://localhost:7557/files/new_firmware_v1.0.bin' -X DELETE
-
-GET /files/
-~~~~~~~~~~~
-
-Gets all previously uploaded files.
-
-GET /files/?query={"filename":"\<filename\>"}
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Find files using a query.
-
-.. _tasks:
-
-Tasks
------
-
-Find the different available tasks and their object structure.
-
-``getParameterValues``
-~~~~~~~~~~~~~~~~~~~~~~
-
-.. code:: javascript
-
-  query = {
-    "name": "getParameterValues",
-    "parameterNames": [
-      "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnectionNumberOfEntries",
-      "InternetGatewayDevice.Time.NTPServer1", "InternetGatewayDevice.Time.Status"
-    ]
-  }
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/00236a-96318REF-SR360NA0A4%252D0003196/tasks?timeout=3000&connection_request' \
-  -X POST \
-  --data '{"name": "getParameterValues", "parameterNames": ["InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnectionNumberOfEntries", "InternetGatewayDevice.Time.NTPServer1", "InternetGatewayDevice.Time.Status"] }'
-
-You may request a single or multiple parameters at once.
-
-After the task has been executed successfully you can then fetch the CPE object
-and read the parameters from the JSON object.
-
-.. code:: javascript
-
-  query = {"_id": "00236a-96318REF-SR360NA0A4%2D0003196"}
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/?query=%7B%22_id%22%3A%2200236a-96318REF-SR360NA0A4%252D0003196%22%7D'
-
-``refreshObject``
-~~~~~~~~~~~~~~~~~
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
-  -X POST \
-  --data '{"name": "refreshObject", "objectName": "InternetGatewayDevice.WANDevice.1.WANConnectionDevice"}'
-
-``setParameterValues``
-~~~~~~~~~~~~~~~~~~~~~~
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
-  -X POST \
-  --data '{"name": "setParameterValues", "parameterValues": [["InternetGatewayDevice.ManagementServer.UpgradesManaged",false]]}'
-
-Multiple values can be set at once by adding multiple arrays to the
-parameterValues key. For example:
-
-.. code:: javascript
-
-  {
-    name: "setParameterValues",
-    parameterValues: [["InternetGatewayDevice.ManagementServer.UpgradesManaged", false], ["InternetGatewayDevice.Time.Enable", true], ["InternetGatewayDevice.Time.NTPServer1", "pool.ntp.org"]]
-  }
-
-``addObject``
-~~~~~~~~~~~~~
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
-  -X POST \
-  --data '{"name":"addObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection"}'
-
-``deleteObject``
-~~~~~~~~~~~~~~~~
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
-  -X POST \
-  --data '{"name":"deleteObject","objectName":"InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANPPPConnection.1"}'
-
-``reboot``
-~~~~~~~~~~
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
-  -X POST \
-  --data '{"name": "reboot"}'
-
-``factoryReset``
-~~~~~~~~~~~~~~~~
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
-  -X POST \
-  --data '{"name": "factoryReset"}'
-
-``download``
-~~~~~~~~~~~~
-
-.. code:: bash
-
-  curl -i 'http://localhost:7557/devices/00236a-SR552n-SR552NA084%252D0003269/tasks?timeout=3000&connection_request' \
-  -X POST \
-  --data '{"name": "download", "file": "mipsbe-6-42-lite.xml"}'
-
-.. _presets:
-
-Presets
--------
-
-Presets assign a set of configuration or a Provision script to devices based on
-a precondition (search filter), schedule (cron expression), and events.
-
-Precondition
-~~~~~~~~~~~~
-
-The ``precondition`` property is a JSON string representation of the search
-filter to test if the preset applies to a given device. Examples preconditions
-are:
-
-- ``{"param": "value"}``
-- ``{"param": value", "param2": {"$ne": "value2"}}``
-
-Other operators that can be used are ``$gt``, ``$lt``, ``$gte`` and ``$lte``.
-
-Configuration
-~~~~~~~~~~~~~
-
-The configuration property is an array containing the different configurations
-to be applied to a device, as shown below:
-
-.. code:: javascript
-
-  [
-    {
-      "type": "value",
-      "name": "InternetGatewayDevice.ManagementServer.PeriodicInformEnable",
-      "value": "true"
-    },
-    {
-      "type": "value",
-      "name": "InternetGatewayDevice.ManagementServer.PeriodicInformInterval",
-      "value": "300"
-    },
-    {
-      "type": "delete_object",
-      "name": "object_parent",
-      "object": "object_name"
-    },
-    {
-      "type": "add_object",
-      "name": "object_parent",
-      "object": "object_name"
-    },
-    {
-      "type": "provision",
-      "name": "YourProvisionName"
-    },
-  ] 
-
-The configuration type ``provision`` triggers a Provision script. In the
-example above, the provision named "YourProvisionName" will be executed.
-
-Provisions
-----------
-
-Create a provision
-~~~~~~~~~~~~~~~~~~
-
-The Provision's JavaScript code is the body of the HTTP PUT request.
-
-.. code:: bash
-
-  curl -X PUT -i 'http://localhost:7557/provisions/mynewprovision' --data 'log("Provision started at " + now);'
-
-Delete a provision
-~~~~~~~~~~~~~~~~~~
-
-.. code:: bash
-
-  curl -X DELETE -i 'http://localhost:7557/provisions/mynewprovision'
-
-Get provisions
-~~~~~~~~~~~~~~
-
-Get all provisions:
-
-.. code:: bash
-
-  curl -X GET -i 'http://localhost:7557/provisions/'

docs/conf.py

@@ -1,65 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# http://www.sphinx-doc.org/en/master/config
-
-# -- Path setup --------------------------------------------------------------
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
-
-import json
-
-# -- Project information -----------------------------------------------------
-
-project = 'GenieACS Documentation'
-copyright = '2019, GenieACS Inc.'
-author = 'GenieACS Inc.'
-
-# The full version, including alpha/beta/rc tags
-release = json.load(open("../package.json"))["version"]
-
-
-# -- General configuration ---------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
-]
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
-
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-# html_theme = 'alabaster'
-html_theme = 'sphinx_rtd_theme'
-html_logo = "logo.svg"
-html_theme_options = {
-  "logo_only": True
-}
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-master_doc = 'index'
-
-highlight_language = 'javascript'

docs/cpe-authentication.rst

@@ -1,72 +0,0 @@
-.. _cpe-authentication:
-
-CPE Authentication
-==================
-
-CPE to ACS
-----------
-
-.. note::
-
-  By default GenieACS will accept any incoming connection via HTTP/HTTPS and
-  respond to it.
-
-The following parameters are used to set and get (password is redacted but
-can be set) the username/password used to authenticate against the ACS:
-
-Username: ``Device.ManagementServer.Username`` or ``InternetGatewayDevice.ManagementServer.Username``
-
-Password: ``Device.ManagementServer.Password`` or ``InternetGatewayDevice.ManagementServer.Password``
-
-Enable CPE to ACS Authentication
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-CPE to ACS authentication can be configured in the web interface by using the
-`Config` option in the `Admin` tab.
-
-Go to the `Admin` -> `Config` page and click on `New config` button at the
-bottom of the page. This will open pop-up which requires you to fill in a key
-and value. The key should be ``cwmp.auth``. The value accepts a boolean.
-Setting the value to ``true`` makes it so that GenieACS accepts any incoming
-connection, setting it to ``false`` makes GenieACS deny all incoming
-connections. This can be further configured using the ``AUTH()`` and ``EXT()``
-functions.
-
-The ``AUTH()`` function
-~~~~~~~~~~~~~~~~~~~~~~~
-
-The ``AUTH()`` function accepts two parameters, username and password. It
-checks the given username and password with the incoming request to determine
-whether to return true or false.
-
-Basic usage of the ``AUTH()`` function could be as follows:
-
-.. code:: sql
-
-   AUTH("fixed-username", "fixed-password")
-
-This will only accept incoming request who authenticate with
-"fixed-username" and "fixed-password".
-
-The various device parameters can be referenced from within the ``cwmp.auth``
-expression. For example:
-
-.. code:: sql
-
-   AUTH(Device.ManagementServer.Username, Device.ManagementServer.Password)
-
-The ``EXT()`` function
-~~~~~~~~~~~~~~~~~~~~~~
-
-The ``EXT()`` function makes it possible to call an :ref:`extension
-<extensions>` script from the auth expression. This can be used to fetch
-the credentials from an external source:
-
-.. code:: sql
-
-   AUTH(DeviceID.SerialNumber, EXT("authenticate", "getPassword", DeviceID.SerialNumber))
-
-ACS to CPE
-----------
-
-TODO

docs/environment-variables.rst

@@ -1,217 +0,0 @@
-.. _environment-variables:
-
-Environment Variables
-=====================
-
-Configuring GenieACS services can be done through the following environment
-variables:
-
-.. attention::
-
-  All GenieACS environment variables must be prefixed with ``GENIEACS_``.
-
-MONGODB_CONNECTION_URL
-  MongoDB connection string.
-
-  Default: ``mongodb://127.0.0.1/genieacs``
-
-EXT_DIR
-  The directory from which to look up extension scripts.
-
-  Default: ``<installation dir>/config/ext``
-
-EXT_TIMEOUT
-  Timeout (in milliseconds) to allow for calls to extensions to return a
-  response.
-
-  Default: ``3000``
-
-DEBUG_FILE
-  File to dump CPE debug log.
-
-  Default: unset
-
-DEBUG_FORMAT
-  Debug log format. Valid values are 'yaml' and 'json'.
-
-  Default: ``yaml``
-
-LOG_FORMAT
-  The format used for the log entries in ``CWMP_LOG_FILE``, ``NBI_LOG_FILE``,
-  ``FS_LOG_FILE``, and ``UI_LOG_FILE``. Possible values are ``simple`` and
-  ``json``.
-
-  Default: ``simple``
-
-ACCESS_LOG_FORMAT
-  The format used for the log entries in ``CWMP_ACCESS_LOG_FILE``,
-  ``NBI_ACCESS_LOG_FILE``, ``FS_ACCESS_LOG_FILE``, and ``UI_ACCESS_LOG_FILE``.
-  Possible values are ``simple`` and ``json``.
-
-  Default: ``simple``
-
-CWMP_WORKER_PROCESSES
-  The number of worker processes to spawn for genieacs-cwmp. A value of 0 means
-  as many as there are CPU cores available.
-
-  Default: ``0``
-
-CWMP_PORT
-  The TCP port that genieacs-cwmp listens on.
-
-  Default: ``7547``
-
-CWMP_INTERFACE
-  The network interface that genieacs-cwmp binds to.
-
-  Default: ``0.0.0.0``
-
-CWMP_SSL_CERT
-  Path to certificate file. If omitted, non-secure HTTP will be used.
-
-  Default: unset
-
-CWMP_SSL_KEY
-  Path to certificate key file. If omitted, non-secure HTTP will be used.
-
-  Default: unset
-
-CWMP_LOG_FILE
-  File to log process related events for genieacs-cwmp. If omitted, logs will
-  go to stderr.
-
-  Default: unset
-
-CWMP_ACCESS_LOG_FILE
-  File to log incoming requests for genieacs-cwmp. If omitted, logs will go to
-  stdout.
-
-  Default: unset
-
-NBI_WORKER_PROCESSES
-  The number of worker processes to spawn for genieacs-nbi. A value of 0 means
-  as many as there are CPU cores available.
-
-  Default: ``0``
-
-NBI_PORT
-  The TCP port that genieacs-nbi listens on.
-
-  Default: ``7557``
-
-NBI_INTERFACE
-  The network interface that genieacs-nbi binds to.
-
-  Default: ``0.0.0.0``
-
-NBI_SSL_CERT
-  Path to certificate file. If omitted, non-secure HTTP will be used.
-
-  Default: unset
-
-NBI_SSL_KEY
-  Path to certificate key file. If omitted, non-secure HTTP will be used.
-
-  Default: unset
-
-NBI_LOG_FILE
-  File to log process related events for genieacs-nbi. If omitted, logs will go
-  to stderr.
-
-  Default: unset
-
-NBI_ACCESS_LOG_FILE
-  File to log incoming requests for genieacs-nbi. If omitted, logs will go to
-  stdout.
-
-  Default: unset
-
-FS_WORKER_PROCESSES
-  The number of worker processes to spawn for genieacs-fs. A value of 0 means
-  as many as there are CPU cores available.
-
-  Default: ``0``
-
-FS_PORT
-  The TCP port that genieacs-fs listens on.
-
-  Default: ``7567``
-
-FS_INTERFACE
-  The network interface that genieacs-fs binds to.
-
-  Default: ``0.0.0.0``
-
-FS_SSL_CERT
-  Path to certificate file. If omitted, non-secure HTTP will be used.
-
-  Default: unset
-
-FS_SSL_KEY
-  Path to certificate key file. If omitted, non-secure HTTP will be used.
-
-  Default: unset
-
-FS_LOG_FILE
-  File to log process related events for genieacs-fs. If omitted, logs will go
-  to stderr.
-
-  Default: unset
-
-FS_ACCESS_LOG_FILE
-  File to log incoming requests for genieacs-fs. If omitted, logs will go to
-  stdout.
-
-  Default: unset
-
-FS_URL_PREFIX
-  The URL prefix (e.g. 'https://example.com:7657/') to use when generating the
-  file URL for TR-069 Download requests. Set this if genieacs-fs and
-  genieacs-cwmp are behind a proxy or running on different servers.
-
-  Default: auto generated based on the hostname from the ACS URL, FS_PORT
-  config, and whether or not SSL is enabled for genieacs-fs.
-
-UI_WORKER_PROCESSES
-  The number of worker processes to spawn for genieacs-ui. A value of 0 means
-  as many as there are CPU cores available.
-
-  Default: ``0``
-
-UI_PORT
-  The TCP port that genieacs-ui listens on.
-
-  Default: ``3000``
-
-UI_INTERFACE
-  The network interface that genieacs-ui binds to.
-
-  Default: ``0.0.0.0``
-
-UI_SSL_CERT
-  Path to certificate file. If omitted, non-secure HTTP will be used.
-
-  Default: unset
-
-UI_SSL_KEY
-  Path to certificate key file. If omitted, non-secure HTTP will be used.
-
-  Default: unset
-
-UI_LOG_FILE
-  File to log process related events for genieacs-ui. If omitted, logs will go
-  to stderr.
-
-  Default: unset
-
-UI_ACCESS_LOG_FILE
-  File to log incoming requests for genieacs-ui. If omitted, logs will go to
-  stdout.
-
-  Default: unset
-
-UI_JWT_SECRET
-  The key used for signing JWT tokens that are stored in browser cookies. The
-  string can be up to 64 characters in length.
-
-  Default: unset

docs/ext-sample.js

@@ -1,37 +0,0 @@
-// This is an example GenieACS extension to get the current latitude/longitude
-// of the International Space Station. Why, you ask? Because why not.
-// To install, copy this file to config/ext/iss.js.
-
-"use strict";
-
-const http = require("http");
-
-let cache = null;
-let cacheExpire = 0;
-
-function latlong(args, callback) {
-  if (Date.now() < cacheExpire) return callback(null, cache);
-
-  http
-    .get("http://api.open-notify.org/iss-now.json", (res) => {
-      if (res.statusCode !== 200)
-        return callback(
-          new Error(`Request failed (status code: ${res.statusCode})`)
-        );
-
-      let rawData = "";
-      res.on("data", (chunk) => (rawData += chunk));
-
-      res.on("end", () => {
-        let pos = JSON.parse(rawData)["iss_position"];
-        cache = [+pos["latitude"], +pos["longitude"]];
-        cacheExpire = Date.now() + 10000;
-        callback(null, cache);
-      });
-    })
-    .on("error", (err) => {
-      callback(err);
-    });
-}
-
-exports.latlong = latlong;

docs/extensions.rst

@@ -1,33 +0,0 @@
-.. _extensions:
-
-Extensions
-==========
-
-Given that :ref:`provisions` and :ref:`virtual-parameters` are executed in a
-sandbox environment, it is not possible to interact with external sources or
-execute any action that requires OS, file system, or network access. Extensions
-exist to bridge that gap.
-
-Extensions are fully-privileged Node.js modules and as such have access to
-standard Node libraries and 3rd party packages. Functions exposed by the
-extension can be called from Provision scripts using the ``ext()`` function. A
-typical use case for extensions is fetching credentials from a database to have
-that pushed to the device during provisioning.
-
-By default, the extension JS code must be placed under ``config/ext``
-directory. You may need to create that directory if it doesn't already exist.
-
-The example extension below fetches data from an external REST API and returns
-that to the caller:
-
-.. literalinclude:: ext-sample.js
-  :language: javascript
-
-To call this extension from a Provision or a Virtual Parameter script:
-
-.. code:: javascript
-
-  // The arguments "arg1" and "arg2" are passed to the latlong. Though they are
-  // unused in this particular example.
-  const res = ext("ext-sample", "latlong", "arg1", "arg2");
-  log(JSON.stringify(res));

docs/https.rst

@@ -1,6 +0,0 @@
-.. _https:
-
-HTTPS
-=====
-
-TODO

docs/index.rst

@@ -1,56 +0,0 @@
-.. GenieACS documentation master file, created by
-   sphinx-quickstart on Wed Jun  5 13:47:06 2019.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
-Welcome to GenieACS's documentation!
-====================================
-
-.. toctree::
-  :caption: Table of Contents
-
-.. raw:: latex
-
-   \part{Installation}
-
-.. toctree::
-  :maxdepth: 1
-  :caption: Installation
-
-  installation-guide
-  environment-variables
-
-.. raw:: latex
-
-   \part{Administration}
-
-.. toctree::
-  :maxdepth: 1
-  :caption: Administration
-
-  provisions
-  virtual-parameters
-  administration-faq
-
-.. raw:: latex
-
-   \part{Integration}
-
-.. toctree::
-  :maxdepth: 1
-  :caption: Integration
-
-  extensions
-  api-reference
-
-.. raw:: latex
-
-   \part{Security}
-
-.. toctree::
-  :maxdepth: 1
-  :caption: Security
-
-  https
-  cpe-authentication
-  roles-and-permissions

docs/installation-guide.rst

@@ -1,233 +0,0 @@
-Installation Guide
-==================
-
-This guide is for installing GenieACS on a single server on any Linux distro
-that uses *systemd* as its init system.
-
-The various GenieACS services are independent of each other and may be
-installed on different servers. You may also run multiple instances of each in
-a load-balancing/failover setup.
-
-.. attention::
-
-  For production deployments make sure to configure TLS and change
-  ``UI_JWT_SECRET`` to a unique and secure string. Refer to :ref:`https`
-  section for how to enable TLS to encrypt traffic.
-
-Prerequisites
--------------
-
-.. topic:: Node.js
-
-  GenieACS requires Node.js 10.x and up. Refer to https://nodejs.org/ for
-  instructions.
-
-.. topic:: MongoDB
-
-  GenieACS requires MongoDB 3.6 and up. Refer to https://www.mongodb.com/ for
-  instructions.
-
-Install GenieACS
--------------------
-
-.. topic:: Installing from NPM:
-
-  .. parsed-literal::
-
-    sudo npm install -g genieacs@\ |release|
-
-.. topic:: Installing from source
-
-  If you prefer installing from source, such as when running a GenieACS copy
-  with custom patches, refer to README.md file in the source package. Adjust
-  the next steps below accordingly.
-
-Configure systemd
------------------
-
-.. topic:: Create a system user to run GenieACS daemons
-
-  .. code:: bash
-
-    sudo useradd --system --no-create-home --user-group genieacs
-
-.. topic:: Create directory to save extensions and environment file
-
-  We'll use :file:`/opt/genieacs/ext/` directory to store extension scripts (if any).
-
-  .. code:: bash
-    
-    mkdir /opt/genieacs
-    mkdir /opt/genieacs/ext
-    chown genieacs:genieacs /opt/genieacs/ext
-
-  Create the file :file:`/opt/genieacs/genieacs.env` to hold our configuration
-  options which we pass to GenieACS as environment variables. See
-  :ref:`environment-variables` section for a list of all available
-  configuration options.
-
-  .. code:: bash
-
-    GENIEACS_CWMP_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-cwmp-access.log
-    GENIEACS_NBI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-nbi-access.log
-    GENIEACS_FS_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-fs-access.log
-    GENIEACS_UI_ACCESS_LOG_FILE=/var/log/genieacs/genieacs-ui-access.log
-    GENIEACS_DEBUG_FILE=/var/log/genieacs/genieacs-debug.yaml
-    NODE_OPTIONS=--enable-source-maps
-    GENIEACS_EXT_DIR=/opt/genieacs/ext
-    GENIEACS_UI_JWT_SECRET=secret
-  
-  Set file ownership and permissions:
-
-  .. code:: bash
-
-    sudo chown genieacs:genieacs /opt/genieacs/genieacs.env
-    sudo chmod 600 /opt/genieacs/genieacs.env
-
-.. topic:: Create logs directory
-
-  .. code:: bash
-    
-    mkdir /var/log/genieacs
-    chown genieacs:genieacs /var/log/genieacs
-
-.. topic:: Create systemd unit files
-
-  Create a systemd unit file for each of the four GenieACS services. Note that
-  we're using EnvironmentFile directive to read the environment variables from
-  the file we created earlier.
-
-  Each service has two streams of logs: access log and process log. Access logs
-  are configured here to be dumped in a log file under
-  :file:`/var/log/genieacs/` while process logs go to *journald*. Use
-  ``journalctl`` command to view process logs.
-
-  .. attention::
-
-    If the command :command:`systemctl edit --force --full` fails, you can
-    create the unit file manually.
-
-  1. Run the following command to create ``genieacs-cwmp`` service:
-  
-    .. code:: bash
-
-      sudo systemctl edit --force --full genieacs-cwmp
-    
-    Then paste the following in the editor and save:
-
-    .. code:: cfg
-
-      [Unit]
-      Description=GenieACS CWMP
-      After=network.target
-
-      [Service]
-      User=genieacs
-      EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=/usr/bin/genieacs-cwmp
-
-      [Install]
-      WantedBy=default.target
-
-  2. Run the following command to create ``genieacs-nbi`` service:
-  
-    .. code:: bash
-
-      sudo systemctl edit --force --full genieacs-nbi
-    
-    Then paste the following in the editor and save:
-
-    .. code:: cfg
-
-      [Unit]
-      Description=GenieACS NBI
-      After=network.target
-
-      [Service]
-      User=genieacs
-      EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=/usr/bin/genieacs-nbi
-
-      [Install]
-      WantedBy=default.target
-
-  3. Run the following command to create ``genieacs-fs`` service:
-  
-    .. code:: bash
-
-      sudo systemctl edit --force --full genieacs-fs
-    
-    Then paste the following in the editor and save:
-
-    .. code:: cfg
-
-      [Unit]
-      Description=GenieACS FS
-      After=network.target
-
-      [Service]
-      User=genieacs
-      EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=/usr/bin/genieacs-fs
-
-      [Install]
-      WantedBy=default.target
-
-  4. Run the following command to create ``genieacs-ui`` service:
-  
-    .. code:: bash
-
-      sudo systemctl edit --force --full genieacs-ui
-    
-    Then paste the following in the editor and save:
-
-    .. code:: cfg
-
-      [Unit]
-      Description=GenieACS UI
-      After=network.target
-
-      [Service]
-      User=genieacs
-      EnvironmentFile=/opt/genieacs/genieacs.env
-      ExecStart=/usr/bin/genieacs-ui
-
-      [Install]
-      WantedBy=default.target
-
-.. topic:: Configure log file rotation using logrotate
-
-  Save the following as :file:`/etc/logrotate.d/genieacs`
-
-  .. code::
-  
-    /var/log/genieacs/*.log /var/log/genieacs/*.yaml {
-        daily
-        rotate 30
-        compress
-        delaycompress
-        dateext
-    }
-
-.. topic:: Enable and start services
-
-  .. code:: bash
-
-    sudo systemctl enable genieacs-cwmp
-    sudo systemctl start genieacs-cwmp
-    sudo systemctl status genieacs-cwmp
-
-    sudo systemctl enable genieacs-nbi
-    sudo systemctl start genieacs-nbi
-    sudo systemctl status genieacs-nbi
-
-    sudo systemctl enable genieacs-fs
-    sudo systemctl start genieacs-fs
-    sudo systemctl status genieacs-fs
-
-    sudo systemctl enable genieacs-ui
-    sudo systemctl start genieacs-ui
-    sudo systemctl status genieacs-ui
-
-  Review the status message for each to verify that the services are running
-  successfully.

docs/logo.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="310" height="64" viewBox="0 0 82.021 16.933"><path d="M7.341 9.892l.687 1.307c-.305.53-.718.99-1.238 1.378-.851.503-1.543.421-2.076-.244 1.03-.466 1.907-1.28 2.627-2.44M4.676 5.97a2.144 2.144 0 0 1 .237 1.228 8.358 8.358 0 0 1-.518 2.21 7.386 7.386 0 0 0-.424 1.708 2.38 2.38 0 0 0 .168 1.205c.163.393.457.73.883 1.01-.458.138-.859.175-1.202.112-.6-.114-.992-.476-1.175-1.084a3.571 3.571 0 0 1-.093-1.787 11.742 11.742 0 0 1 .435-1.644 11.606 11.606 0 0 0 .432-1.708c.037-.22.041-.45.01-.687a1.66 1.66 0 0 0-.273-.706 1.582 1.582 0 0 0-.6-.506c.245-.046.487-.083.728-.113a1.45 1.45 0 0 1 .728.123c.295.13.517.343.665.639m7.901-1.802l.222.218-1.847 1.678.555.555 1.678-1.846.218.22-1.843 1.679.521.521L13.76 5.35l.222.218-1.847 1.678.556.556 1.764-1.765.375.368-1.584 1.584h.004l-2.065 2.061-1.253.515-1.795-1.348-.007-.008-.008.008.004.007 1.194 1.592-.398.161-1.75-1.75.444-1.08 1.636 1.228.004.003.011-.007-.003-.004-1.39-1.85.225-.553 1.694-1.689h-.004l1.956-1.952.372.368-1.769 1.768.552.552 1.678-1.843m1.456 9.91c1.55-1.549 2.324-3.42 2.324-5.612 0-2.187-.774-4.058-2.323-5.612C12.48 1.304 10.61.53 8.423.53 6.23.53 4.36 1.305 2.81 2.854c-.95.953-1.609 2.025-1.974 3.213l.158-.086c.788-.415 1.486-.203 2.094.638-.55.113-.992.232-1.325.357a1.923 1.923 0 0 0-.503.289c-.337.268-.55.608-.638 1.02a3.98 3.98 0 0 0 .011 1.761c.285 1.527 1.011 2.87 2.178 4.032 1.549 1.552 3.42 2.328 5.612 2.328 2.19 0 4.06-.776 5.612-2.328" fill="white"/><path d="M47.104 1.767c-.456 0-.816.388-.816.849 0 .46.355.816.816.816a.82.82 0 0 0 .833-.816.85.85 0 0 0-.833-.849zM25.602 3.171c-.205 0-.382.046-.5.13-.118.085-.187.201-.187.4v.981l-.151-.031a9.89 9.89 0 0 0-1.872-.173c-1.288 0-2.148.066-2.657.374a1.264 1.264 0 0 0-.548.676c-.119.313-.177.73-.177 1.28v6.123c0 .982.25 1.508.787 1.824.539.315 1.407.41 2.628.41 1.308 0 2.178-.13 2.701-.484.524-.354.745-.932.745-1.942 0-.821-.147-1.37-.543-1.722-.395-.352-1.075-.528-2.186-.528h-2.787V9.255l.148.03c.478.092 1.094.11 1.89.11 1.292 0 2.148-.112 2.658-.453.51-.34.724-.915.724-1.958V3.7c0-.198-.07-.314-.186-.399a.843.843 0 0 0-.487-.13zm6.112 1.29c-1.226 0-2.038.137-2.526.495-.488.357-.697.945-.697 1.947v2.84c0 .935.23 1.54.743 1.932.512.39 1.337.571 2.544.571 1.392 0 2.17-.058 2.567-.196.199-.069.298-.15.355-.246.058-.097.078-.227.078-.405 0-.208-.04-.378-.117-.485-.076-.108-.186-.171-.396-.171-.794 0-1.057.19-2.408.19-.593 0-1.06-.078-1.395-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.717.717 0 0 0 .539-.213c.13-.138.213-.342.213-.62V6.903c0-1.037-.179-1.623-.654-1.97-.476-.348-1.3-.472-2.617-.472zm8.83 0c-1.303 0-2.117.086-2.59.49-.472.403-.665 1.168-.665 2.623v4.002c0 .215.074.338.197.426.123.088.31.134.524.134.223 0 .412-.046.537-.134.125-.088.198-.211.198-.426v-3.81c0-.424.006-.754.04-1.019.036-.265.102-.47.236-.623.135-.152.328-.236.574-.28.246-.044.552-.053.949-.053.4 0 .712.01.963.053.25.044.447.126.587.278.139.151.211.358.25.623.038.265.046.597.046 1.02v3.81c0 .217.07.34.19.427.122.088.308.134.53.134.215 0 .401-.046.525-.134.123-.088.196-.211.196-.426V7.574c0-1.455-.201-2.219-.682-2.623-.48-.404-1.303-.49-2.605-.49zm12.996 0c-1.227 0-2.038.137-2.527.495-.488.357-.696.945-.696 1.947v2.84c0 .935.23 1.54.742 1.932.513.39 1.338.571 2.545.571 1.392 0 2.17-.058 2.567-.196.2-.069.298-.15.355-.246.058-.097.079-.227.079-.405 0-.208-.04-.378-.117-.485-.077-.108-.188-.171-.398-.171-.793 0-1.056.19-2.406.19-.594 0-1.062-.078-1.396-.255-.335-.177-.53-.473-.53-.84v-.89h4.301a.716.716 0 0 0 .539-.213c.13-.138.213-.342.213-.62V6.903c0-1.037-.18-1.623-.655-1.97-.475-.348-1.3-.472-2.616-.472zm-6.436.13c-.212 0-.398.047-.522.14-.124.09-.198.22-.198.435v6.41c0 .215.073.338.196.426.123.088.31.134.524.134.223 0 .414-.046.54-.134.124-.088.198-.211.198-.426v-6.41c0-.215-.075-.342-.201-.434-.126-.092-.316-.142-.537-.142zM22.892 5.726c.662 0 1.4.065 1.918.146l.105.016v.92c0 .246-.01.453-.058.632a.757.757 0 0 1-.292.441c-.305.216-.798.263-1.673.263h-.123v-.013c-.761 0-1.26-.002-1.55-.16a.637.637 0 0 1-.307-.395c-.049-.17-.057-.37-.057-.625v-.062c0-.227.03-.418.108-.58a.811.811 0 0 1 .373-.373c.337-.171.826-.21 1.556-.21zm8.839.049c.677 0 1.116.054 1.412.237a.775.775 0 0 1 .316.373c.063.154.085.33.085.535v.811h-3.611V6.92c0-.437.135-.755.439-.928.304-.174.739-.217 1.359-.217zm21.825 0c.677 0 1.117.054 1.413.237a.779.779 0 0 1 .318.373c.062.154.084.33.084.535v.811h-3.613V6.92c0-.437.136-.755.44-.928.304-.174.738-.217 1.358-.217zm-32.701 5.867h2.388c.613 0 1.05.045 1.355.226.306.18.443.505.443.95 0 .429-.179.743-.533.907-.354.164-.861.206-1.568.206-.652 0-1.151-.045-1.51-.186a.952.952 0 0 1-.425-.3.789.789 0 0 1-.15-.483v-1.195z" fill="white"/><path d="M62.339 4.462c-1.399 0-2.18.078-2.58.225-.2.073-.3.157-.357.254-.058.097-.079.22-.079.386 0 .199.043.362.122.469.079.107.19.172.392.172.38 0 .616-.048.947-.096a9.457 9.457 0 0 1 1.475-.098c.58 0 .966.043 1.233.192a.685.685 0 0 1 .292.312c.06.13.082.281.082.452v.857h-2.484c-1.134 0-1.763.19-2.11.559-.346.367-.443.95-.443 1.802 0Line truncated

docs/provisions.rst

@@ -1,279 +0,0 @@
-.. _provisions:
-
-Provisions
-==========
-
-A Provision is a piece of JavaScript code that is executed on the server on a
-per-device basis. It enables implementing complex provisioning scenarios and
-other operations such as automated firmware upgrade rollout. Apart from a few
-special functions, the script is essentially a standard ES6 code executed in
-strict mode.
-
-Provisions are mapped to devices using presets. Note that the added performance
-overhead when using Provisions as opposed to simple preset configuration
-entries is relatively small. Anything that can be done via preset
-configurations can be done using a Provision script. In fact, the now
-deprecated configuration format is still supported primarily for backward
-compatibility and it is recommended to use Provision scripts for all
-configuration.
-
-When assigning a Provision script to a preset, you may pass arguments to the
-script. The arguments can be accessed from the script through the global
-``args`` variable.
-
-.. note::
-
-  Provision scripts may get executed multiple times in a given session.
-  Although all data model-mutating operations are idempotent, a script as a
-  whole may not be. It is, therefore, necessary to repeatedly run the script
-  until there are no more side effects and a stable state is reached.
-
-Built-in functions
-------------------
-
-``declare(path, timestamps, values)``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This function is for declaring parameter values to be set, as well as specify
-constraints on how recent you'd like the parameter value (or other attributes)
-to have been refreshed from the device. If the given timestamp is lower than
-the timestamp of the last refresh from the device, then this function will
-return the last known value. Otherwise, the value will be fetched from the
-device before being returned to the caller.
-
-The timestamp argument is an object where the key is the attribute name (e.g.
-``value``, ``object``, ``writable``, ``path``) and the value is an integer
-representing a Unix timestamp.
-
-The values argument is an object similar to the timestamp argument but its
-property values being the parameter values to be set.
-
-The possible attributes in 'timestamps' and 'values' arguments are:
-
-- ``value``: a [<value>, <type>] pair
-
-This attribute is not available for objects or object instances. If the value
-is not a [<value>, <type>] array then it'll assumed to be a value without a
-type and therefore the type will be inferred from the parameter's type.
-
-- ``writable``: boolean
-
-The meaning of this attribute can vary depending on the type of the parameter.
-In the case of regular parameters, it indicates if its value is writable. In
-the case of objects, it's whether or not it's possible to add new object
-instances. In the case of object instances, it indicates whether or not this
-instance can be deleted.
-
-- ``object``: boolean
-
-True if this is an object or object instance, false otherwise.
-
-- ``path``: string
-
-This attribute is special in that it's not a parameter attribute per se, but it
-refers to the presence of parameters matching the given path. For example,
-given the following wildcard path:
-
-``InternetGatewayDevice.LANDevice.1.Hosts.Host.*.MACAddress``
-
-Using a recent timestamp for path in ``declare()`` will result in a sync with
-the device to rediscover all Host instances (``Host.*``). The path attribute
-can also be used to create or delete object instances as described in
-:ref:`path-format` section.
-
-The return value of ``declare()`` is an iterator to access parameters that
-match the given path. Each item in the iterator has the attribute 'path' in
-addition to any other attribute given in the ``declare()`` call. The iterator
-object itself has convenience attribute accessors which come in handy when
-you're expecting a single parameter (e.g. when path does not contain wildcards
-or aliases).
-
-.. code:: javascript
-
-  // Example: Setting the SSID as the last 6 characters of the serial number
-  let serial = declare("Device.DeviceInfo.SerialNumber", {value: 1});
-  declare("Device.LANDevice.1.WLANConfiguration.1.SSID", null, {value: serial.value[0]});
-
-``clear(path, timestamp)``
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This function invalidates the database copy of parameters (and their child
-parameters) that match the given path and have a last refresh timestamp that is
-less than the given timestamp. The most obvious use for this function is to
-invalidate the database copy of the entire data model after the device has been
-factory reset:
-
-.. code:: javascript
-
-  // Example: Clear cached device data model Note
-  // Make sure to apply only on "0 BOOTSTRAP" event
-  clear("Device", Date.now());
-  clear("InternetGatewayDevice", Date.now());
-
-``commit()``
-~~~~~~~~~~~~
-
-This function commits the pending declarations and performs any necessary sync
-with the device. It's usually not required to call this function as it called
-implicitly at the end of the script and when accessing any property of the
-promise-like object returned by the ``declare()`` function. Calling this
-explicitly is only necessary if you want to control the order in which
-parameters are configured.
-
-``ext(file, function, arg1, arg2, ...)``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Execute an extension script and return the result. The first argument is the
-script filename while second argument is the function name within that script.
-Any remaining arguments will be passed to that function. See :ref:`extensions`
-for more details.
-
-``log(message)``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Prints out a string in genieacs-cwmp's access log. It's meant to be used for
-debugging. Note that you may see multiple log entries as the script can be
-executed multiple times in a session. See :ref:`this FAQ
-<administration-faq-duplicate-log-entries>`.
-
-.. _path-format:
-
-Path format
------------
-
-A parameter path may contain a wildcard (``*``) or an alias filter
-(``[name:value]``). A wildcard segment in a parameter path will apply the
-declared configuration to zero or more parameters that match the given path
-where the wildcard segment can be anything.
-
-An alias filter is like a wildcard, but additionally performs filtering on the
-child parameters based on the key-value pairs provided. For example, the
-following path:
-
-``Device.WANDevice.1.WANConnectionDevice.1.WANIPConnection.[AddressingType:DHCP].ExternalIPAddress``
-
-will return a list of ExternalIPAddress parameters (0 or more) where the
-sibling parameter AddressingType is assigned the value "DHCP".
-
-This can be useful when the exact instance numbers may be different from one
-device to another. It is possible to use more than one key-value pair in the
-alias filter. It's also possible to use multiple filters or use a combination
-of filters and wildcards.
-
-Creating/deleting object instances
-----------------------------------
-
-Given the declarative nature of provisions, we cannot explicitly tell the
-device to create or delete an instance under a given object. Instead, we
-specify the number of instances we want there to be, and based on that GenieACS
-will determine whether or not it needs to create or delete instances. For
-example, the following declaration will ensure we have one and only one
-WANIPConnection object:
-
-.. code:: javascript
-
-  // Example: Ensure we have one and only one WANIPConnection object
-  declare("InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.*", null, {path: 1});
-
-Note the wildcard at the end of the parameter path.
-
-It is also possible to use alias filters as the last path segment which will
-ensure that the declared number of instances is satisfied given the alias
-filter:
-
-.. code:: javascript
-
-  // Ensure that *all* other instances are deleted
-  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[]", null, {path: 0});
-
-  // Add the two entries we care about
-  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[SourceIPAddress:192.168.1.0,SourceNetMask:255.255.255.0]",  {path: now}, {path: 1});
-  declare("InternetGatewayDevice.X_BROADCOM_COM_IPAddrAccCtrl.X_BROADCOM_COM_IPAddrAccCtrlListCfg.[SourceIPAddress:172.16.12.0,SourceNetMask:255.255.0.0]", {path: now}, {path: 1});
-
-Special GenieACS parameters
----------------------------
-
-In addition to the parameters exposed in the device's data model through
-TR-069, GenieACS has its own set of special parameters:
-
-``DeviceID``
-~~~~~~~~~~~~
-
-This parameter sub-tree includes the following read-only parameters:
-
-- ``DeviceID.ID``
-- ``DeviceID.SerialNumber``
-- ``DeviceID.ProductClass``
-- ``DeviceID.OUI``
-- ``DeviceID.Manufacturer``
-
-``Tags``
-~~~~~~~~
-
-The ``Tags`` root parameter is used to expose device tags in the data model.
-Tags appear as child parameters that are writable and have boolean value.
-Setting a tag to ``false`` will delete that tag, and setting the value of a
-non-existing tag parameter to ``true`` will create it.
-
-.. code:: javascript
-
-  // Example: Remove "tag1", add "tag2", and read "tag3"
-  declare("Tags.tag1", null, {value: false});
-  declare("Tags.tag2", null, {value: true});
-  let tag3 = declare("Tags.tag3", {value: 1});
-
-``Reboot``
-~~~~~~~~~~
-
-The ``Reboot`` root parameter hold the timestamp of the last reboot command.
-The parameter value is writable and declaring a timestamp value that is larger
-than the current value will trigger a reboot.
-
-.. code:: javascript
-
-  // Example: Reboot the device only if it hasn't been rebooted in the past 300 seconds
-  declare("Reboot", null, {value: Date.now() - (300 * 1000)});
-
-``FactoryReset``
-~~~~~~~~~~~~~~~~
-
-Works like ``Reboot`` parameter but for factory reset.
-
-.. code:: javascript
-
-  // Example: Default the device to factory settings
-  declare("FactoryReset", null, {value: Date.now()});
-
-``Downloads``
-~~~~~~~~~~~~~
-
-The ``Downloads`` sub-tree holds information about the last download
-command(s). A download command is represented as an instance (e.g.
-``Downloads.1``) containing parameters such as ``Download`` (timestamp),
-``LastFileType``, ``LastFileName``. The parameters ``FileType``, ``FileName``,
-``TargetFileName`` and ``Download`` are writable and can be used to trigger a
-new download.
-
-.. code:: javascript
-
-  declare("Downloads.[FileType:1 Firmware Upgrade Image]", {path: 1}, {path: 1});
-  declare("Downloads.[FileType:1 Firmware Upgrade Image].FileName", {value: 1}, {value: "firmware-2017.01.tar"});
-  declare("Downloads.[FileType:1 Firmware Upgrade Image].Download", {value: 1}, {value: Date.now()});
-
-Common file types are:
-
-- ``1 Firmware Upgrade Image``
-- ``2 Web Content``
-- ``3 Vendor Configuration File``
-- ``4 Tone File``
-- ``5 Ringer File``
-
-.. warning::
-
-  Pushing a file to the device is often a service-interrupting operation. It's
-  recommended to only trigger it on certain events such as ``1 BOOT`` or during
-  a predetermined maintenance window).
-
-After the CPE had finished downloading and applying the config file, it will
-send a ``7 TRANSFER COMPLETE`` event. You may use that to trigger a reboot
-after the firmware image or configuration file had been applied.

docs/roles-and-permissions.rst

@@ -1,6 +0,0 @@
-.. _roles-and-permissions:
-
-Roles and Permissions
-=====================
-
-TODO

docs/virtual-parameters.rst

@@ -1,105 +0,0 @@
-.. _virtual-parameters:
-
-Virtual Parameters
-==================
-
-Virtual parameters are user-defined parameters whose values are generated using
-a custom Javascript code. Virtual parameters behave just like regular
-parameters and appear in the data model under ``VirtualParameters.`` path.
-Virtual parameter names cannot contain a period (``.``).
-
-The execution environment for virtual parameters is almost identical to that of
-provisions. See :ref:`provisions` for more details and examples. The only
-differences between the scripts of provisions and virtual parameters are:
-
-- You can't pass custom arguments to virtual parameter scripts. Instead, the
-  variable ``args`` will hold the current vparam timestamps and values as well
-  as the declared timestamps and values. Like this:
-
-.. code:: javascript
-
-  // [<declared attr timestamps, declared attr values>, <current attr timestamps>, <current attr values>]
-  [{path: 1559849387191, value: 1559849387191}, {value: ["new val", "xsd:string"]}, {path: 1559840000000, value: 1559840000000}, {value: ["cur val", "xsd:string"]}]
-
-- Virtual parameter scripts must return an object containing the attributes of
-  this parameter.
-
-.. note::
-
-  Just like a regular parameter, creating a virtual parameter does not
-  automatically add it to the parameter list for a device. It needs to fetched
-  (manually or via a preset) before you can see it in the data model.
-
-Examples
---------
-
-Unified MAC parameter across different device models
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. code:: javascript
-
-  // Example: Unified MAC parameter across different device models
-  let m = "00:00:00:00:00:00";
-  let d = declare("Device.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress", {value: Date.now()});
-  let igd = declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANPPPConnection.*.MACAddress", {value: Date.now()});
-
-  if (d.size) {
-    for (let p of d) {
-      if (p.value[0]) {
-        m = p.value[0];
-        break;
-      }
-    }  
-  }
-  else if (igd.size) {
-    for (let p of igd) {
-      if (p.value[0]) {
-        m = p.value[0];
-        break;
-      }
-    }  
-  }
-
-  return {writable: false, value: [m, "xsd:string"]};
-
-Expose an external value as a virtual parameter
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. code:: javascript
-
-  // Example: Expose an external value as a virtual parameter
-  let serial = declare("DeviceID.SerialNumber", {value: 1});
-  if (args[1].value) {
-    ext("example-ext", "set", serial.value[0], args[1].value[0]);
-    return {writable: true, value: [args[1].value[0], "xsd:string"]};
-  }
-  else {
-    let v = ext("example-ext", "get", serial.value[0]);
-    return {writable: true, value: [v, "xsd:string"]};
-  }
-
-Create an editable virtual parameter for WPA passphrase
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. code:: javascript
-
-  // Example: Create an editable virtual parameter for WPA passphrase
-  let m = "";
-  if (args[1].value) {
-    m = args[1].value[0];
-    declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", null, {value: m});
-    declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", null, {value: m});
-  }
-  else {
-    let d = declare("Device.WiFi.AccessPoint.1.Security.KeyPassphrase", {value: Date.now()});
-    let igd = declare("InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase", {value: Date.now()});
-
-    if (d.size) {
-      m = d.value[0];
-    }
-    else if (igd.size) {
-      m = igd.value[0];  
-    }
-  }
-
-  return {writable: true, value: [m, "xsd:string"]};

lib/api-functions.coffee

@@ -0,0 +1,266 @@
+###
+# Copyright 2013-2017  Zaid Abdulla
+#
+# This file is part of GenieACS.
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
+crypto = require 'crypto'
+dgram = require 'dgram'
+
+config = require './config'
+db = require './db'
+http = require 'http'
+common = require './common'
+util = require 'util'
+URL = require 'url'
+auth = require './auth'
+cache = require './cache'
+
+
+udpConReq = (address, un, key, callback) ->
+  return false if not address
+  [host, port] = address.split(':', 2)
+  if port is undefined
+    port = 80
+  else
+    port = parseInt(port)
+    return false if isNaN(port)
+
+  ts = Math.trunc(Date.now() / 1000)
+  id = Math.trunc(Math.random() * 4294967295)
+  cn = crypto.randomBytes(8).toString('hex')
+  sig = crypto.createHmac('sha1', key).update("#{ts}#{id}#{un}#{cn}").digest('hex')
+  uri = "http://#{address}?ts=#{ts}&id=#{id}&un=#{un}&cn=#{cn}&sig=#{sig}"
+
+  message = Buffer.from("GET #{uri} HTTP/1.1\r\nHost: #{address}\r\n\r\n")
+
+  client = dgram.createSocket({type: 'udp4', reuseAddr: true})
+  UDP_CONNECTION_REQUEST_PORT = config.get('UDP_CONNECTION_REQUEST_PORT')
+  if UDP_CONNECTION_REQUEST_PORT
+    # When a device is NAT'ed, the UDP Connection Request must originate from the same address and port used by the STUN server, in order to traverse the firewall.
+    # This does require that the Genieacs NBI and STUN server are allowed to bind to the same address and port.
+    # The STUN server needs to open its UDP port with the SO_REUSEADDR option, allowing the NBI to also bind to the same port.
+    client.bind({port: UDP_CONNECTION_REQUEST_PORT, exclusive: true})
+
+  count = 3
+  client.send(message, 0, message.length, port, host, f = (err) ->
+    if err or -- count <= 0
+      client.close()
+      return callback(err)
+    client.send(message, 0, message.length, port, host, f)
+  )
+  return true
+
+
+httpConReq = (url, username, password, allowBasicAuth, timeout, callback) ->
+  options = URL.parse(url)
+
+  if options.protocol != 'http:'
+    return callback(new Error('Invalid connection request URL or protocol'))
+
+  # Ensure socket is reused in case of digest authentication
+  options.agent = new http.Agent({maxSockets : 1})
+
+  statusToError = (statusCode) ->
+    switch statusCode
+      when 200, 204
+        null
+      when 401
+        new Error('Incorrect connection request credentials')
+      when 0
+        new Error('Device is offline')
+      else
+        new Error("Unexpected response code from device: #{statusCode}")
+
+  request = http.get(options, (res) ->
+    if res.statusCode == 401 and res.headers['www-authenticate']?
+      authHeader = auth.parseAuthHeader(res.headers['www-authenticate'])
+      if authHeader.method is 'Basic'
+        if not allowBasicAuth
+          request.abort()
+          callback(new Error('Basic HTTP authentication not allowed'))
+          return
+        options.headers = {'Authorization' : auth.basic(username or '', password or '')}
+      else if authHeader.method is 'Digest'
+        options.headers = {
+          'Authorization' : auth.digest(username or '', password or '', options.path, 'GET', null, authHeader)
+        }
+
+      request = http.get(options, (res) ->
+        if res.statusCode == 0
+          # Workaround for some devices unexpectedly closing the connection
+          request = http.get(options, (res) ->
+            callback(statusToError(res.statusCode))
+            res.resume()
+          ).on('error', (err) ->
+            request.abort()
+            callback(statusToError(0))
+          ).on('socket', (socket) ->
+            socket.setTimeout(timeout)
+            socket.on('timeout', () -> request.abort())
+          )
+        else
+          callback(statusToError(res.statusCode))
+        res.resume()
+      ).on('error', (err) ->
+        request.abort()
+        callback(statusToError(0))
+      ).on('socket', (socket) ->
+        socket.setTimeout(timeout)
+        socket.on('timeout', () -> request.abort())
+      )
+    else
+      callback(statusToError(res.statusCode))
+
+    # No listener for data so emit resume
+    res.resume()
+  ).on('error', (err) ->
+    request.abort()
+    callback(statusToError(0))
+  ).on('socket', (socket) ->
+    socket.setTimeout(timeout)
+    socket.on('timeout', () -> request.abort())
+  )
+
+
+connectionRequest = (deviceId, callback) ->
+  CONNECTION_REQUEST_TIMEOUT = config.get('CONNECTION_REQUEST_TIMEOUT', deviceId)
+  CONNECTION_REQUEST_ALLOW_BASIC_AUTH = config.get('CONNECTION_REQUEST_ALLOW_BASIC_AUTH', deviceId)
+  proj = {
+    'Device.ManagementServer.ConnectionRequestURL._value' : 1,
+    'Device.ManagementServer.UDPConnectionRequestAddress._value' : 1,
+    'Device.ManagementServer.ConnectionRequestUsername._value' : 1,
+    'Device.ManagementServer.ConnectionRequestPassword._value' : 1,
+    'InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value' : 1,
+    'InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress._value' : 1,
+    'InternetGatewayDevice.ManagementServer.ConnectionRequestUsername._value' : 1,
+    'InternetGatewayDevice.ManagementServer.ConnectionRequestPassword._value' : 1
+  }
+
+  db.devicesCollection.findOne({_id : deviceId}, proj, (err, device)->
+    return callback(err) if err
+    return callback(new Error("No such device")) if !device
+
+    if device.Device? # TR-181 data model
+      connectionRequestUrl = device.Device.ManagementServer.ConnectionRequestURL._value
+      udpConnectionRequestAddress = device.Device.ManagementServer?.UDPConnectionRequestAddress?._value
+      username = device.Device.ManagementServer.ConnectionRequestUsername?._value
+      password = device.Device.ManagementServer.ConnectionRequestPassword?._value
+    else # TR-098 data model
+      connectionRequestUrl = device.InternetGatewayDevice.ManagementServer.ConnectionRequestURL._value
+      udpConnectionRequestAddress = device.InternetGatewayDevice.ManagementServer?.UDPConnectionRequestAddress?._value
+      username = device.InternetGatewayDevice.ManagementServer.ConnectionRequestUsername?._value
+      password = device.InternetGatewayDevice.ManagementServer.ConnectionRequestPassword?._value
+
+    conReq = () ->
+      udpSent = udpConReq(udpConnectionRequestAddress, username, password, (err) -> throw err if err)
+
+      httpConReq(connectionRequestUrl, username, password, CONNECTION_REQUEST_ALLOW_BASIC_AUTH, CONNECTION_REQUEST_TIMEOUT, (err) ->
+        if udpSent
+          return callback()
+        callback(err)
+      )
+
+    if config.auth?.connectionRequest?
+      # Callback is optional for backward compatibility
+      if config.auth.connectionRequest.length > 4
+        return config.auth.connectionRequest(deviceId, connectionRequestUrl, username, password, (u, p) ->
+          username = u
+          password = p
+          conReq()
+        )
+      [username, password] = config.auth.connectionRequest(deviceId, connectionRequestUrl, username, password)
+    conReq()
+  )
+
+
+watchTask = (deviceId, taskId, timeout, callback) ->
+  setTimeout(() ->
+    db.tasksCollection.findOne({_id : taskId}, {'_id' : 1}, (err, task) ->
+      return callback(err) if err
+
+      if not task
+        return callback(null, 'completed')
+
+      db.faultsCollection.findOne({_id : "#{deviceId}:task_#{taskId}"}, {'_id' : 1}, (err, fault) ->
+        return callback(err) if err
+
+        if fault
+          return callback(null, 'fault')
+
+        if (timeout -= 500) <= 0
+          return callback(null, 'timeout')
+
+        watchTask(deviceId, taskId, timeout, callback)
+      )
+    )
+  , 500)
+
+
+sanitizeTask = (task, callback) ->
+  task.timestamp = new Date(task.timestamp ? Date.now())
+  if task.expiry?
+    if common.typeOf(task.expiry) is common.DATE_TYPE or isNaN(task.expiry)
+      task.expiry = new Date(task.expiry)
+    else
+      task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000)
+
+  callback(task)
+
+
+insertTasks = (tasks, callback) ->
+  if tasks? and common.typeOf(tasks) isnt common.ARRAY_TYPE
+    tasks = [tasks]
+  else if not tasks? or tasks.length == 0
+    return callback(null, tasks or [])
+
+  counter = tasks.length
+
+  for task in tasks
+    sanitizeTask(task, (t) ->
+      if t.uniqueKey?
+        db.tasksCollection.remove({device : t.device, uniqueKey : t.uniqueKey}, (err) ->
+        )
+
+      --counter
+      if counter == 0
+        db.tasksCollection.insert(tasks, (err, _tasks) ->
+          #util.log("#{_task.device}: Added task #{_task.name}(#{_task._id})") for _task in _tasks
+          callback(err, _tasks)
+        )
+    )
+
+
+deleteDevice = (deviceId, callback) ->
+  db.tasksCollection.remove({'device' : deviceId}, (err) ->
+    return callback(err) if err
+    db.devicesCollection.remove({'_id' : deviceId}, (err) ->
+      return callback(err) if err
+      db.faultsCollection.remove({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}, (err) ->
+        return callback(err) if err
+        db.operationsCollection.remove({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}, (err) ->
+          return callback(err) if err
+          cache.del("#{deviceId}_tasks_faults_operations", callback)
+        )
+      )
+    )
+  )
+
+
+exports.connectionRequest = connectionRequest
+exports.watchTask = watchTask
+exports.insertTasks = insertTasks
+exports.deleteDevice = deleteDevice

lib/api-functions.ts

@@ -1,352 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { parse } from "url";
-import * as db from "./db";
-import * as common from "./common";
-import * as cache from "./cache";
-import {
-  getCurrentSnapshot,
-  getConfig,
-  getConfigExpression,
-} from "./local-cache";
-import {
-  httpConnectionRequest,
-  udpConnectionRequest,
-} from "./connection-request";
-import { Expression, Task } from "./types";
-import { flattenDevice } from "./mongodb-functions";
-import { evaluate } from "./common/expression";
-
-export async function connectionRequest(
-  deviceId: string,
-  device?: Record<string, { value?: [boolean | number | string, string] }>
-): Promise<string> {
-  if (!device) {
-    const res = await db.devicesCollection.findOne({ _id: deviceId });
-    if (!res) throw new Error("No such device");
-    device = flattenDevice(res);
-  }
-
-  let connectionRequestUrl, udpConnectionRequestAddress, username, password;
-
-  if (device["InternetGatewayDevice.ManagementServer.ConnectionRequestURL"]) {
-    connectionRequestUrl = (device[
-      "InternetGatewayDevice.ManagementServer.ConnectionRequestURL"
-    ].value || [""])[0];
-    udpConnectionRequestAddress = ((
-      device[
-        "InternetGatewayDevice.ManagementServer.UDPConnectionRequestAddress"
-      ] || {}
-    ).value || [""])[0];
-    username = ((
-      device[
-        "InternetGatewayDevice.ManagementServer.ConnectionRequestUsername"
-      ] || {}
-    ).value || [""])[0];
-    password = ((
-      device[
-        "InternetGatewayDevice.ManagementServer.ConnectionRequestPassword"
-      ] || {}
-    ).value || [""])[0];
-  } else {
-    connectionRequestUrl = (device[
-      "Device.ManagementServer.ConnectionRequestURL"
-    ].value || [""])[0];
-    udpConnectionRequestAddress = ((
-      device["Device.ManagementServer.UDPConnectionRequestAddress"] || {}
-    ).value || [""])[0];
-    username = ((
-      device["Device.ManagementServer.ConnectionRequestUsername"] || {}
-    ).value || [""])[0];
-    password = ((
-      device["Device.ManagementServer.ConnectionRequestPassword"] || {}
-    ).value || [""])[0];
-  }
-
-  const remoteAddress = parse(connectionRequestUrl).host;
-
-  const evalCallback = (exp): Expression => {
-    if (!Array.isArray(exp)) return exp;
-    if (exp[0] === "PARAM" && typeof exp[1] === "string") {
-      let name = exp[1];
-      if (name === "id") name = "DeviceID.ID";
-      else if (name === "serialNumber") name = "DeviceID.SerialNumber";
-      else if (name === "productClass") name = "DeviceID.ProductClass";
-      else if (name === "oui") name = "DeviceID.OUI";
-      else if (name === "remoteAddress") return remoteAddress;
-      else if (name === "username") return username;
-      else if (name === "password") return password;
-
-      const p = device[name];
-      if (p && p.value) return p.value[0];
-    } else if (exp[0] === "FUNC") {
-      if (exp[1] === "REMOTE_ADDRESS") return remoteAddress;
-      else if (exp[1] === "USERNAME") return username;
-      else if (exp[1] === "PASSWORD") return password;
-    }
-    return exp;
-  };
-
-  const snapshot = await getCurrentSnapshot();
-  const now = Date.now();
-  const UDP_CONNECTION_REQUEST_PORT = +getConfig(
-    snapshot,
-    "cwmp.udpConnectionRequestPort",
-    {},
-    now,
-    evalCallback
-  );
-  const CONNECTION_REQUEST_TIMEOUT = +getConfig(
-    snapshot,
-    "cwmp.connectionRequestTimeout",
-    {},
-    now,
-    evalCallback
-  );
-  const CONNECTION_REQUEST_ALLOW_BASIC_AUTH = !!getConfig(
-    snapshot,
-    "cwmp.connectionRequestAllowBasicAuth",
-    {},
-    now,
-    evalCallback
-  );
-  let authExp: Expression = getConfigExpression(
-    snapshot,
-    "cwmp.connectionRequestAuth"
-  );
-
-  if (authExp === undefined) {
-    authExp = [
-      "FUNC",
-      "AUTH",
-      ["PARAM", "username"],
-      ["PARAM", "password"],
-    ] as Expression;
-  }
-
-  authExp = evaluate(authExp, {}, now, evalCallback);
-
-  const debug = !!getConfig(snapshot, "cwmp.debug", {}, now, evalCallback);
-
-  let udpProm = Promise.resolve(false);
-  if (udpConnectionRequestAddress) {
-    try {
-      const u = new URL("udp://" + udpConnectionRequestAddress);
-      udpProm = udpConnectionRequest(
-        u.hostname,
-        parseInt(u.port || "80"),
-        authExp,
-        UDP_CONNECTION_REQUEST_PORT,
-        debug,
-        deviceId
-      ).then(
-        () => true,
-        () => false
-      );
-    } catch (err) {
-      // Ignore invalid address
-    }
-  }
-
-  const status = await httpConnectionRequest(
-    connectionRequestUrl,
-    authExp,
-    CONNECTION_REQUEST_ALLOW_BASIC_AUTH,
-    CONNECTION_REQUEST_TIMEOUT,
-    debug,
-    deviceId
-  );
-
-  if (await udpProm) return "";
-
-  return status;
-}
-
-export async function awaitSessionStart(
-  deviceId: string,
-  lastInform: number,
-  timeout: number
-): Promise<boolean> {
-  const now = Date.now();
-  const device = await db.devicesCollection.findOne(
-    { _id: deviceId },
-    { projection: { _lastInform: 1 } }
-  );
-  const li = (device["_lastInform"] as Date).getTime();
-  if (li > lastInform) return true;
-  const token = await cache.get(`cwmp_session_${deviceId}`);
-  if (token) return true;
-  if (timeout < 500) return false;
-  await new Promise((resolve) => setTimeout(resolve, 500));
-  timeout -= Date.now() - now;
-  return awaitSessionStart(deviceId, lastInform, timeout);
-}
-
-export async function awaitSessionEnd(
-  deviceId: string,
-  timeout: number
-): Promise<boolean> {
-  const now = Date.now();
-  const token = await cache.get(`cwmp_session_${deviceId}`);
-  if (!token) return true;
-  if (timeout < 500) return false;
-  await new Promise((resolve) => setTimeout(resolve, 500));
-  timeout -= Date.now() - now;
-  return awaitSessionEnd(deviceId, timeout);
-}
-
-function sanitizeTask(task): void {
-  task.timestamp = new Date(task.timestamp || Date.now());
-  if (task.expiry) {
-    if (task.expiry instanceof Date || isNaN(task.expiry))
-      task.expiry = new Date(task.expiry);
-    else task.expiry = new Date(task.timestamp.getTime() + +task.expiry * 1000);
-  }
-
-  const validParamValue = (p): boolean => {
-    if (
-      !Array.isArray(p) ||
-      p.length < 2 ||
-      typeof p[0] !== "string" ||
-      !p[0].length ||
-      !["string", "boolean", "number"].includes(typeof p[1]) ||
-      (p[2] != null && typeof p[2] !== "string")
-    )
-      return false;
-    return true;
-  };
-
-  switch (task.name) {
-    case "getParameterValues":
-      if (!Array.isArray(task.parameterNames) || !task.parameterNames.length)
-        throw new Error("Missing 'parameterNames' property");
-      for (const p of task.parameterNames) {
-        if (typeof p !== "string" || !p.length)
-          throw new Error(`Invalid parameter name '${p}'`);
-      }
-      break;
-
-    case "setParameterValues":
-      if (!Array.isArray(task.parameterValues) || !task.parameterValues.length)
-        throw new Error("Missing 'parameterValues' property");
-      for (const p of task.parameterValues) {
-        if (!validParamValue(p))
-          throw new Error(`Invalid parameter value '${p}'`);
-      }
-      break;
-
-    case "refreshObject":
-      if (typeof task.objectName !== "string")
-        throw new Error("Missing 'objectName' property");
-      break;
-
-    case "deleteObject":
-      if (typeof task.objectName !== "string" || !task.objectName.length)
-        throw new Error("Missing 'objectName' property");
-      break;
-
-    case "addObject":
-      if (task.parameterValues != null) {
-        if (!Array.isArray(task.parameterValues))
-          throw new Error("Invalid 'parameterValues' property");
-        for (const p of task.parameterValues) {
-          if (!validParamValue(p))
-            throw new Error(`Invalid parameter value '${p}'`);
-        }
-      }
-      break;
-
-    case "download":
-      // genieacs-gui sends file ID instead of fileName and fileType
-      if (!task.file) {
-        if (typeof task.fileType !== "string" || !task.fileType.length)
-          throw new Error("Missing 'fileType' property");
-
-        if (typeof task.fileName !== "string" || !task.fileName.length)
-          throw new Error("Missing 'fileName' property");
-      }
-
-      if (
-        task.targetFileName != null &&
-        typeof task.targetFileName !== "string"
-      )
-        throw new Error("Invalid 'targetFileName' property");
-      break;
-
-    case "provisions":
-      if (
-        !Array.isArray(task.provisions) ||
-        !task.provisions.every((arr) =>
-          arr.every(
-            (s) =>
-              s == null || ["boolean", "number", "string"].includes(typeof s)
-          )
-        )
-      )
-        throw new Error("Invalid 'provisions' property");
-      break;
-
-    case "reboot":
-      break;
-
-    case "factoryReset":
-      break;
-
-    default:
-      throw new Error("Invalid task name");
-  }
-
-  return task;
-}
-
-export async function insertTasks(tasks: any[]): Promise<Task[]> {
-  if (tasks && !Array.isArray(tasks)) tasks = [tasks];
-  else if (!tasks || tasks.length === 0) return tasks || [];
-
-  for (const task of tasks) {
-    sanitizeTask(task);
-    if (task.uniqueKey) {
-      await db.tasksCollection.deleteOne({
-        device: task.device,
-        uniqueKey: task.uniqueKey,
-      });
-    }
-  }
-  await db.tasksCollection.insertMany(tasks);
-  return tasks;
-}
-
-export async function deleteDevice(deviceId: string): Promise<void> {
-  await Promise.all([
-    db.tasksCollection.deleteMany({ device: deviceId }),
-    db.devicesCollection.deleteOne({ _id: deviceId }),
-    db.faultsCollection.deleteMany({
-      _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
-      },
-    }),
-    db.operationsCollection.deleteMany({
-      _id: {
-        $regex: `^${common.escapeRegExp(deviceId)}\\:`,
-      },
-    }),
-    cache.del(`${deviceId}_tasks_faults_operations`),
-  ]);
-}

lib/auth.coffee

@@ -0,0 +1,92 @@
+###
+# Copyright 2013-2017  Zaid Abdulla
+#
+# This file is part of GenieACS.
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
+crypto = require 'crypto'
+
+
+exports.parseAuthHeader = (authHeader) ->
+  authHeader = authHeader.trim()
+  method = authHeader.split(' ', 1)[0]
+  res = {method: method}
+  parts = authHeader.slice(method.length + 1).split(',')
+
+  while (part = parts.shift())?
+    name = part.split('=', 1)[0]
+    throw new Error('Unable to parse auth header') if name.length == part.length
+    value = part.slice(name.length + 1)
+    if not /^\s*"/.test(value)
+      value = value.trim()
+    else
+      while not /[^\\]"\s*$/.test(value)
+        p = parts.shift()
+        throw new Error('Unable to parse auth header') if not p?
+        value += ',' + p
+
+      try
+        value = JSON.parse(value)
+      catch
+        throw new Error('Unable to parse auth header')
+
+    res[name.trim()] = value
+
+  return res
+
+
+exports.basic = (username, password) ->
+  "Basic #{new Buffer("#{username}:#{password}").toString('base64')}"
+
+
+exports.digest = (username, password, uri, httpMethod, body, authHeader) ->
+  cnonce = '0a4f113b'
+  nc = '00000001'
+
+  if authHeader.qop?
+    if authHeader.qop.indexOf(',') != -1
+      qop = 'auth' # either auth or auth-int. prefer auth
+    else
+      qop = authHeader.qop
+
+  ha1 = crypto.createHash('md5')
+  ha1.update(username).update(':').update(authHeader.realm).update(':').update(password)
+  # TODO support "MD5-sess" algorithm directive
+  ha1 = ha1.digest('hex')
+
+  ha2 = crypto.createHash('md5')
+  ha2.update(httpMethod).update(':').update(uri)
+  if qop == 'auth-int'
+    ha2.update(':').update(body)
+  ha2 = ha2.digest('hex')
+
+  response = crypto.createHash('md5')
+  response.update(ha1).update(':').update(authHeader.nonce)
+
+  if qop?
+    response.update(':').update(nc).update(':').update(cnonce).update(':').update(qop)
+  response.update(':').update(ha2)
+  response = response.digest('hex')
+
+  authString = "Digest username=\"#{username}\""
+  authString += ",realm=\"#{authHeader.realm}\""
+  authString += ",nonce=\"#{authHeader.nonce}\""
+  authString += ",uri=\"#{uri}\""
+  authString += ",algorithm=#{authHeader.algorithm}" if authHeader.algorithm?
+  authString += ",qop=#{qop},nc=#{nc},cnonce=\"#{cnonce}\"" if qop?
+  authString += ",response=\"#{response}\""
+  authString += ",opaque=\"#{authHeader.opaque}\"" if authHeader.opaque?
+  return authString

lib/auth.ts

@@ -1,197 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { createHash, randomBytes, pbkdf2 } from "crypto";
-
-function parseHeaderFeilds(str: string): Record<string, string> {
-  const res = {};
-  const parts = str.split(",");
-
-  let part: string;
-  while ((part = parts.shift()) != null) {
-    const name = part.split("=", 1)[0];
-    if (name.length === part.length) {
-      if (!part.trim()) continue;
-      throw new Error("Unable to parse auth header");
-    }
-
-    let value = part.slice(name.length + 1);
-    if (!/^\s*"/.test(value)) {
-      value = value.trim();
-    } else {
-      while (!/[^\\]"\s*$/.test(value)) {
-        const p = parts.shift();
-        if (p == null) throw new Error("Unable to parse auth header");
-        value += "," + p;
-      }
-
-      try {
-        value = JSON.parse(value);
-      } catch (error) {
-        throw new Error("Unable to parse auth header");
-      }
-    }
-    res[name.trim()] = value;
-  }
-  return res;
-}
-
-export function parseAuthorizationHeader(
-  authHeader: string
-): { method: string } {
-  authHeader = authHeader.trim();
-  const method = authHeader.split(" ", 1)[0];
-  const res = { method: method };
-
-  if (method === "Basic") {
-    // Inspired by https://github.com/jshttp/basic-auth
-    const USER_PASS_REGEX = /^([^:]*):(.*)$/;
-    const creds = USER_PASS_REGEX.exec(
-      Buffer.from(authHeader.slice(method.length + 1), "base64").toString()
-    );
-
-    if (!creds) throw new Error("Unable to parse auth header");
-    res["username"] = creds[1];
-    res["password"] = creds[2];
-  } else if (method === "Digest") {
-    Object.assign(res, parseHeaderFeilds(authHeader.slice(method.length + 1)));
-  }
-
-  return res;
-}
-
-export function parseWwwAuthenticateHeader(
-  authHeader: string
-): Record<string, string> {
-  authHeader = authHeader.trim();
-  const method = authHeader.split(" ", 1)[0];
-  const res = { method: method };
-  Object.assign(res, parseHeaderFeilds(authHeader.slice(method.length + 1)));
-  return res;
-}
-
-export function basic(username: string, password: string): string {
-  return "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
-}
-
-export function digest(
-  username: string | Buffer,
-  realm: string | Buffer,
-  password: string | Buffer,
-  nonce: string | Buffer,
-  httpMethod: string | Buffer,
-  uri: string | Buffer,
-  qop?: string | Buffer,
-  body?: string | Buffer,
-  cnonce?: string | Buffer,
-  nc?: string | Buffer
-): string {
-  const ha1 = createHash("md5");
-  ha1.update(username).update(":").update(realm).update(":").update(password);
-  // TODO support "MD5-sess" algorithm directive
-  const ha1d = ha1.digest("hex");
-
-  const ha2 = createHash("md5");
-  ha2.update(httpMethod).update(":").update(uri);
-
-  if (qop === "auth-int") {
-    const bodyHash = createHash("md5")
-      .update(body || "")
-      .digest("hex");
-    ha2.update(":").update(bodyHash);
-  }
-
-  const ha2d = ha2.digest("hex");
-
-  const hash = createHash("md5");
-  hash.update(ha1d).update(":").update(nonce);
-  if (qop) {
-    hash
-      .update(":")
-      .update(nc)
-      .update(":")
-      .update(cnonce)
-      .update(":")
-      .update(qop);
-  }
-  hash.update(":").update(ha2d);
-
-  return hash.digest("hex");
-}
-
-export function solveDigest(
-  username: string | Buffer,
-  password: string | Buffer,
-  uri: string | Buffer,
-  httpMethod: string | Buffer,
-  body: string | Buffer,
-  authHeader: Record<string, string>
-): string {
-  const cnonce = randomBytes(8).toString("hex");
-  const nc = "00000001";
-
-  let qop;
-  if (authHeader.qop) {
-    if (authHeader.qop.indexOf(",") !== -1) qop = "auth";
-    // Either auth or auth-int, prefer auth
-    else qop = authHeader.qop;
-  }
-
-  const hash = digest(
-    username,
-    authHeader.realm,
-    password,
-    authHeader.nonce,
-    httpMethod,
-    uri,
-    qop,
-    body,
-    cnonce,
-    nc
-  );
-
-  let authString = `Digest username="${username}"`;
-  authString += `,realm="${authHeader.realm}"`;
-  authString += `,nonce="${authHeader.nonce}"`;
-  authString += `,uri="${uri}"`;
-  if (authHeader.algorithm) authString += `,algorithm=${authHeader.algorithm}`;
-  if (qop) authString += `,qop=${qop},nc=${nc},cnonce="${cnonce}"`;
-  authString += `,response="${hash}"`;
-  if (authHeader.opaque) authString += `,opaque="${authHeader.opaque}"`;
-
-  return authString;
-}
-
-export function generateSalt(length: number): Promise<string> {
-  return new Promise((resolve, reject) => {
-    randomBytes(length, (err, rand) => {
-      if (err) return void reject(err);
-      resolve(rand.toString("hex"));
-    });
-  });
-}
-
-export function hashPassword(pass: string, salt: string): Promise<string> {
-  return new Promise((resolve, reject) => {
-    pbkdf2(pass, salt, 10000, 128, "sha512", (err, hash) => {
-      if (err) return void reject(err);
-      resolve(hash.toString("hex"));
-    });
-  });
-}

lib/cache.js

@@ -0,0 +1,257 @@
+/**
+ * Copyright 2013-2017  Zaid Abdulla
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+"use strict";
+
+const config = require("./config");
+
+const MAX_CACHE_TTL = config.get("MAX_CACHE_TTL");
+
+let redisClient;
+let mongoDb;
+let mongoCollection;
+let mongoTimeOffset = 0;
+
+
+function connect(callback) {
+  const REDIS_HOST = config.get("REDIS_HOST");
+
+  if (REDIS_HOST) {
+    const redis = require("redis");
+    const REDIS_PORT = config.get("REDIS_PORT");
+    const REDIS_DB = config.get("REDIS_DB");
+    redisClient = redis.createClient(REDIS_PORT, REDIS_HOST);
+    redisClient.select(REDIS_DB, function(err) {
+      exports.get = redisGet;
+      exports.set = redisSet;
+      exports.del = redisDel;
+      exports.pop = redisPop;
+      exports.lock = redisLock;
+      callback(err);
+    });
+  }
+  else {
+    const mongodb = require("mongodb");
+    const MONGODB_CONNECTION_URL = config.get("MONGODB_CONNECTION_URL");
+
+    mongodb.MongoClient.connect(MONGODB_CONNECTION_URL, function(err, db) {
+      if (err)
+        return callback(err);
+
+      mongoDb = db;
+      mongoCollection = db.collection("cache");
+      mongoCollection.ensureIndex({"expire": 1}, {expireAfterSeconds: 0});
+
+      const now = Date.now();
+      mongoDb.command({hostInfo: 1}, function(err, res) {
+        if (err)
+          return callback(err);
+        mongoTimeOffset = res.system.currentTime.getTime() - now;
+        exports.get = mongoGet;
+        exports.set = mongoSet;
+        exports.del = mongoDel;
+        exports.pop = mongoPop;
+        exports.lock = mongoLock;
+        callback();
+      });
+    });
+  }
+}
+
+
+function disconnect() {
+  if (redisClient)
+    redisClient.quit();
+
+  if (mongoDb)
+    mongoDb.close();
+}
+
+
+function redisGet(key, callback) {
+  if (Array.isArray(key))
+    redisClient.mget(key, callback);
+  else
+    redisClient.get(key, callback);
+}
+
+
+function redisDel(key, callback) {
+  redisClient.del(key, callback);
+}
+
+
+function redisSet(key, value, ttl, callback) {
+  if (!callback && typeof v === "function") {
+    callback = ttl;
+    ttl = null;
+  }
+
+  if (!ttl)
+    ttl = MAX_CACHE_TTL;
+
+  redisClient.setex(key, ttl, value, callback);
+}
+
+
+function redisPop(key, callback) {
+  const POP_SCRIPT = "local v=redis.call(\"get\",KEYS[1]);redis.call(\"del\",KEYS[1]);return v;";
+  redisClient.eval(POP_SCRIPT, 1, key, callback);
+}
+
+
+function redisLock(lockName, ttl, callback) {
+  const UNLOCK_SCRIPT = "if redis.call(\"get\", KEYS[1]) == ARGV[1] then return redis.call(\"del\", KEYS[1]) else return 0 end";
+  const EXTEND_SCRIPT = "if redis.call(\"get\", KEYS[1]) == ARGV[1] then return redis.call(\"pexpire\", KEYS[1], ARGV[2]) else return 0 end";
+  const token = Math.random().toString(36).slice(2);
+
+  function unlockOrExtend(ttl) {
+    if (!ttl) {
+      redisClient.eval(UNLOCK_SCRIPT, 1, lockName, token, function(err, res) {
+        if (err || !res)
+          throw err || new Error("Lock expired");
+      });
+    }
+    else {
+      redisClient.eval(EXTEND_SCRIPT, 1, lockName, token, ttl, function(err, res) {
+        if (err || !res)
+          throw err || new Error("Lock expired");
+      });
+    }
+  }
+
+  redisClient.set(lockName, token, "NX", "EX", ttl, function(err, res) {
+    if (err || res)
+      return callback(err, unlockOrExtend);
+
+    setTimeout(function() {
+      redisLock(lockName, ttl, callback);
+    }, 200);
+  });
+}
+
+
+function mongoGet(key, callback) {
+  let expire = new Date(Date.now() - mongoTimeOffset);
+  if (Array.isArray(key))
+    mongoCollection.find({"_id": {"$in": key}}).toArray(function(err, res) {
+      if (err)
+        return callback(err);
+
+      let indices = {};
+      key.forEach(function(v, i) {
+        indices[v] = i;
+      });
+
+      let values = [];
+      res.forEach(function(r) {
+        if (r["expire"] > expire)
+          values[indices[r["_id"]]] = r["value"];
+      });
+      callback(null, values);
+    });
+  else
+    mongoCollection.findOne({"_id": {"$in": [key]}}, function(err, res) {
+      if (err || !res)
+        return callback(err);
+
+      if (res["expire"] > expire)
+        return callback(null, res["value"]);
+
+      callback();
+    });
+}
+
+
+function mongoDel(key, callback) {
+  if (Array.isArray(key))
+    mongoCollection.remove({"_id": {"$in": key}}, callback);
+  else
+    mongoCollection.remove({"_id": key}, callback);
+}
+
+
+function mongoSet(key, value, ttl, callback) {
+  if (!callback && typeof v === "function") {
+    callback = ttl;
+    ttl = null;
+  }
+
+  if (!ttl)
+    ttl = MAX_CACHE_TTL;
+
+  let expire = new Date((Date.now() - mongoTimeOffset) + (ttl * 1000));
+  mongoCollection.save({"_id": key, "value": value, "expire": expire}, callback);
+}
+
+
+function mongoPop(key, callback) {
+  mongoCollection.findAndModify({"_id": key}, null, null, {remove: true}, function(err, res) {
+    if (err || !res["value"])
+      return callback(err);
+
+    if (res["value"]["expire"] > new Date(Date.now() - mongoTimeOffset))
+      return callback(null, res["value"]["value"]);
+
+    callback();
+  });
+}
+
+
+function mongoLock(lockName, ttl, callback) {
+  const token = Math.random().toString(36).slice(2);
+
+  function unlockOrExtend(ttl) {
+    if (!ttl) {
+      mongoCollection.remove({"_id": lockName, "value": token}, function(err, res) {
+        if (err || res["result"]["n"] !== 1)
+          throw err || new Error("Lock expired");
+      });
+    }
+    else {
+      let expire = new Date((Date.now() - mongoTimeOffset) + (ttl * 1000));
+      mongoCollection.update(
+        {"_id": lockName, "value": token},
+        {"expire": expire},
+        function(err, res) {
+          if (err || res["result"]["n"] !== 1)
+            throw err | new Error("Lock expired");
+        }
+      );
+    }
+  }
+
+  let expireTest = new Date(Date.now() - mongoTimeOffset);
+  let expireSet = new Date((Date.now() - mongoTimeOffset) + (ttl * 1000));
+
+  mongoCollection.update(
+    {"_id": lockName, "expire": {"$lte": expireTest}},
+    {"$set": {"value" : token, "expire": expireSet}},
+    {upsert: true},
+    function(err, res) {
+      if (err && err.code === 11000)
+        return setTimeout(function() {mongoLock(lockName, ttl, callback)}, 200);
+
+      return callback(err, unlockOrExtend);
+    }
+  );
+}
+
+
+exports.connect = connect;
+exports.disconnect = disconnect;

lib/cache.ts

@@ -1,118 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { Collection } from "mongodb";
-import { onConnect } from "./db";
-import * as config from "./config";
-
-const CLOCK_SKEW_TOLERANCE = 30000;
-const MAX_CACHE_TTL = +config.get("MAX_CACHE_TTL");
-
-let cacheCollection: Collection;
-
-onConnect(async (db) => {
-  cacheCollection = db.collection("cache");
-  await cacheCollection.createIndex({ expire: 1 }, { expireAfterSeconds: 0 });
-});
-
-export async function get(key: string): Promise<string> {
-  const res = await cacheCollection.findOne({ _id: key });
-  if (res) return res["value"];
-  return null;
-}
-
-export async function del(key: string): Promise<void> {
-  await cacheCollection.deleteOne({ _id: key });
-}
-
-export async function set(
-  key: string,
-  value: string,
-  ttl: number = MAX_CACHE_TTL
-): Promise<void> {
-  const timestamp = new Date();
-  const expire = new Date(
-    timestamp.getTime() + CLOCK_SKEW_TOLERANCE + ttl * 1000
-  );
-  await cacheCollection.replaceOne(
-    { _id: key },
-    { _id: key, value, expire, timestamp },
-    { upsert: true }
-  );
-}
-
-export async function pop(key: string): Promise<any> {
-  const res = await cacheCollection.findOneAndDelete({ _id: key });
-  if (res && res["value"]) return res["value"]["value"];
-  return null;
-}
-
-export async function acquireLock(
-  lockName: string,
-  ttl: number,
-  timeout = 0,
-  token = Math.random().toString(36).slice(2)
-): Promise<string> {
-  try {
-    const now = Date.now();
-    const r = await cacheCollection.findOneAndUpdate(
-      { _id: lockName, token },
-      {
-        $set: {
-          value: token,
-          expire: new Date(now + ttl + CLOCK_SKEW_TOLERANCE),
-        },
-        $currentDate: { timestamp: true },
-      },
-      { upsert: true, returnOriginal: false }
-    );
-    const v = r.value;
-    if (Math.abs(v["timestamp"].getTime() - now) > CLOCK_SKEW_TOLERANCE)
-      throw new Error("Database clock skew too great");
-  } catch (err) {
-    if (err.code === 11000) {
-      if (timeout > 0) {
-        return new Promise((resolve, reject) => {
-          const w = 50 + Math.random() * 50;
-          setTimeout(() => {
-            acquireLock(lockName, ttl, timeout - w, token).then(
-              resolve,
-              reject
-            );
-          }, w);
-        });
-      }
-      throw new Error("Failed to acquire lock");
-    }
-    throw err;
-  }
-
-  return token;
-}
-
-export async function releaseLock(
-  lockName: string,
-  token: string
-): Promise<void> {
-  const res = await cacheCollection.deleteOne({
-    _id: lockName,
-    value: token,
-  });
-  if (res["result"]["n"] !== 1) throw new Error("Lock expired");
-}

lib/cluster.js

@@ -0,0 +1,177 @@
+/**
+ * Copyright 2013-2017  Zaid Abdulla
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+"use strict";
+
+const cluster = require("cluster");
+const logger = require("./logger");
+const config = require("./config");
+
+const VERSION = require("../package.json").version;
+
+let respawnTimestamp = 0;
+let tooManyCrashesTimestamp = 0;
+
+function getDependencyVersions() {
+  let dependencies = [`node@${process.versions.node}`];
+  try {
+    let npmls = JSON.parse(require("child_process").execSync("npm ls --prod --json --depth 0"));
+
+    for (let k in npmls.dependencies)
+      dependencies.push(`${k}@${npmls.dependencies[k].version}`);
+  } catch (err) {}
+
+  return dependencies.join(",");
+}
+
+
+function getConfig() {
+  let conf = [];
+  const keys = Object.keys(process.env);
+  for (let k of keys)
+    if (k.startsWith("GENIEACS_")) {
+      const name = k.slice(9);
+      const value = process.env[k];
+      const def = config.getDefault(name);
+      if (def == null || value !== `${def}`)
+        conf.push(`${name}=${value}`);
+    }
+
+  return conf.join(",");
+}
+
+
+function restartWorker(worker, code, signal) {
+  let msg = {
+    message: "Worker died",
+    pid: worker.process.pid
+  };
+
+  if (code != null)
+    msg.exitCode = code;
+
+  if (signal != null)
+    msg.signal = signal;
+
+  logger.error(msg);
+
+  const now = Date.now();
+  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
+  if (respawnTimestamp === now) {
+    tooManyCrashesTimestamp = now;
+    cluster.fork();
+    return
+  }
+
+  if (now - tooManyCrashesTimestamp > 60000) {
+    process.exitCode = 1;
+    cluster.removeListener("exit", restartWorker);
+    for (let pid in cluster.workers)
+      cluster.workers[pid].kill();
+
+    logger.error({
+      message: "Too many crashes, exiting",
+      pid: process.pid
+    });
+    return;
+  }
+
+  setTimeout(function() {
+    if (process.exitCode) return;
+    cluster.fork();
+  }, respawnTimestamp - now);
+}
+
+
+function start(service) {
+  logger.init(service, VERSION);
+  const config = require("./config");
+  const SERVICE_ADDRESS = config.get(`${service.toUpperCase()}_INTERFACE`)
+  const SERVICE_PORT = config.get(`${service.toUpperCase()}_PORT`)
+
+  logger.info({
+    message: `GenieACS (genieacs-${service}) starting`,
+    pid: process.pid,
+    version: VERSION,
+    dependencies: getDependencyVersions(),
+    config: getConfig()
+  });
+
+  cluster.on("listening", function(worker, address) {
+    if ((address.addressType === 4 || address.addressType === 6) &&
+        address.address === SERVICE_ADDRESS && address.port === SERVICE_PORT) {
+      logger.info({
+        message: "Worker listening",
+        pid: worker.process.pid,
+        address: address.address,
+        port: address.port
+      });
+    }
+  });
+
+  cluster.on("exit", restartWorker);
+
+  process.on("SIGINT", function() {
+    logger.info({
+      message: "Received signal SIGINT, exiting",
+      pid: process.pid
+    });
+
+    cluster.removeListener("exit", restartWorker);
+  });
+
+  process.on("SIGTERM", function() {
+    logger.info({
+      message: "Received signal SIGTERM, exiting",
+      pid: process.pid
+    });
+
+    cluster.removeListener("exit", restartWorker);
+    for (let pid in cluster.workers)
+      cluster.workers[pid].kill();
+  });
+
+  cluster.setupMaster({
+    exec: "lib/server",
+    args: [service]
+  });
+
+  let workerCount = config.get(`${service.toUpperCase()}_WORKER_PROCESSES`);
+
+  if (!workerCount)
+    workerCount = Math.max(2, require("os").cpus().length);
+
+  for (let i = 0; i < workerCount; ++ i)
+    cluster.fork();
+}
+
+
+process.on("uncaughtException", function(err) {
+  logger.error({
+    message: "Uncaught exception in master process, exiting",
+    exception: err,
+    pid: process.pid
+  });
+  process.exitCode = 1;
+  cluster.removeListener("exit", restartWorker);
+  for (let pid in cluster.workers)
+    cluster.workers[pid].kill();
+});
+
+
+exports.start = start;

lib/cluster.ts

@@ -1,124 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import cluster from "cluster";
-import { cpus } from "os";
-import * as logger from "./logger";
-
-let respawnTimestamp = 0;
-let crashes: number[] = [];
-
-function fork(): cluster.Worker {
-  const w = cluster.fork();
-  w.on("error", (err: NodeJS.ErrnoException) => {
-    // Avoid exception when attempting to kill the process just as it's exiting
-    if (err.code !== "EPIPE") throw err;
-    setTimeout(() => {
-      if (!w.isDead()) throw err;
-    }, 50);
-  });
-  return w;
-}
-
-function restartWorker(worker, code, signal): void {
-  const msg = {
-    message: "Worker died",
-    pid: worker.process.pid,
-    exitCode: null,
-    signal: null,
-  };
-
-  if (code != null) msg.exitCode = code;
-
-  if (signal != null) msg.signal = signal;
-
-  logger.error(msg);
-
-  const now = Date.now();
-  crashes.push(now);
-
-  let min1 = 0,
-    min2 = 0,
-    min3 = 0;
-
-  crashes = crashes.filter((n) => {
-    if (n > now - 60000) ++min1;
-    else if (n > now - 120000) ++min2;
-    else if (n > now - 180000) ++min3;
-    else return false;
-    return true;
-  });
-
-  if (min1 > 5 && min2 > 5 && min3 > 5) {
-    process.exitCode = 1;
-    cluster.removeListener("exit", restartWorker);
-    for (const pid in cluster.workers) cluster.workers[pid].kill();
-
-    logger.error({
-      message: "Too many crashes, exiting",
-      pid: process.pid,
-    });
-    return;
-  }
-
-  respawnTimestamp = Math.max(now, respawnTimestamp + 2000);
-  if (respawnTimestamp === now) {
-    fork();
-    return;
-  }
-
-  setTimeout(() => {
-    if (process.exitCode) return;
-    fork();
-  }, respawnTimestamp - now);
-}
-
-export function start(
-  workerCount: number,
-  servicePort: number,
-  serviceAddress: string
-): void {
-  cluster.on("listening", (worker, address) => {
-    if (
-      (address.addressType === 4 || address.addressType === 6) &&
-      address.address === serviceAddress &&
-      address.port === servicePort
-    ) {
-      logger.info({
-        message: "Worker listening",
-        pid: worker.process.pid,
-        address: address.address,
-        port: address.port,
-      });
-    }
-  });
-
-  cluster.on("exit", restartWorker);
-
-  if (!workerCount) workerCount = Math.max(2, cpus().length);
-
-  for (let i = 0; i < workerCount; ++i) fork();
-}
-
-export function stop(): void {
-  cluster.removeListener("exit", restartWorker);
-  for (const pid in cluster.workers) cluster.workers[pid].kill();
-}
-
-export const worker = cluster.worker;

lib/common.coffee

@@ -0,0 +1,229 @@
+###
+# Copyright 2013-2017  Zaid Abdulla
+#
+# This file is part of GenieACS.
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
+buffer = require 'buffer'
+
+UNDEFINED_TYPE = '[object Undefined]'
+NULL_TYPE = '[object Null]'
+BOOLEAN_TYPE = '[object Boolean]'
+NUMBER_TYPE = '[object Number]'
+STRING_TYPE = '[object String]'
+ARRAY_TYPE = '[object Array]'
+OBJECT_TYPE = '[object Object]'
+REGEXP_TYPE = '[object RegExp]'
+DATE_TYPE = '[object Date]'
+
+
+typeOf = (obj) ->
+  Object.prototype.toString.call(obj)
+
+
+exports.generateDeviceId = (deviceIdStruct) ->
+  # Percent escaping function. Escapes everything except alphanumerics and underscore
+  esc = (str) ->
+    str.replace(/[^A-Za-z0-9_]/g, (chr) ->
+      buf = new buffer.Buffer(chr, 'utf8')
+      rep = ''
+      rep += "%#{b.toString(16).toUpperCase()}" for b in buf
+      return rep
+    )
+
+  # Guaranteeing globally unique id as defined in TR-069
+  if deviceIdStruct['ProductClass']
+    return "#{esc(deviceIdStruct['OUI'])}-#{esc(deviceIdStruct['ProductClass'])}-#{esc(deviceIdStruct['SerialNumber'])}"
+
+  return "#{esc(deviceIdStruct['OUI'])}-#{esc(deviceIdStruct['SerialNumber'])}"
+
+
+exports.getParamValueFromPath = (obj, path) ->
+  pp = path.split('.')
+  ref = obj
+  try
+    for p in pp
+      ref = ref[p]
+    return ref
+  catch err
+    return undefined
+
+
+pathOverlap = (a, b, start) ->
+  if a.length == b.length
+    res = 3
+  else if a.length > b.length
+    res = 2
+  else
+    res = 1
+
+  for i in [(start ? 0)...Math.min(a.length, b.length)] by 1
+    if a[i]?
+      if not b[i]?
+        res &= 2
+      else if a[i] != b[i]
+        return 0
+    else if b[i]?
+      res &= 1
+
+    return 0 if not res
+
+  return res
+
+
+parseAlias = (pattern, start, res) ->
+  aliases = []
+  i = start
+  while i < pattern.length and pattern[i] != ']'
+    alias = []
+    i = j = parsePath(pattern, i, alias) + 1
+
+    while pattern[j] not in [']', ',']
+      if pattern[j] == '"' and i == j
+        ++ j
+        while pattern[j] != '"' or pattern[j - 1] == '\\'
+          if ++ j >= pattern.length
+            throw new Error('Invalid alias expression')
+      if ++ j >= pattern.length
+        throw new Error('Invalid alias expression')
+    value = pattern[i...j].trim()
+    i = j
+    if value[0] == '"'
+      try
+        value = JSON.parse(value)
+      catch err
+        throw new Error('Invalid alias expression')
+    alias.push(value)
+    aliases.push(alias)
+    ++ i if pattern[i] == ','
+
+  # Need to sort to ensure identical expressions have idential string representation
+  srt = (a, b) ->
+    for j in [0...Math.min(a.length, b.length)] by 2
+      for k in [0...Math.min(a[j].length, b[j].length)] by 1
+        if Array.isArray(a[j][k])
+          if Array.isArray(b[j][k])
+            return srt(a[j][k], b[j][k])
+          else if not b[j][k]?
+            return -1
+          else
+            return 1
+        else if not a[j][k]?
+          if not b[j][k]?
+            return 0
+          else
+            return 1
+        else if not b[j][k]? or Array.isArray(b[j][k])
+            return -1
+        else if a[j][k] > b[j][k]
+          return 1
+        else if a[j][k] < b[j][k]
+          return -1
+
+      if a[j].length > b[j].length
+        return -1
+      else if a[j].length < b[j].length
+        return 1
+
+      if a[j + 1] > b[j + 1]
+        return -1
+      else if a[j + 1] < b[j + 1]
+        return 1
+
+    if a.length > b.length
+      return -1
+    else if a.length < b.length
+      return 1
+
+    return 0
+
+  aliases.sort(srt)
+  res.push([].concat.apply([], aliases))
+  return i
+
+
+parsePath = (pattern, start, res) ->
+  path = []
+  path.wildcard = 0
+  path.alias = 0
+  i = start ? 0
+
+  # Colon separator is needed for parseAlias
+  if i < pattern.length and pattern[i] != ':'
+    while true
+      if pattern[i] == '['
+        path.alias |= 1 << path.length
+        i = parseAlias(pattern, i + 1, path) + 1
+      else
+        j = i
+        while i < pattern.length and pattern[i] != ':' and pattern[i] != '.'
+          ++ i
+        n = pattern.slice(j, i)
+        path.wildcard |= 1 << path.length if n == '*'
+        path.push(n)
+
+      if i >= pattern.length or pattern[i] == ':'
+        break
+      else if pattern[i] != '.'
+        throw new Error('Invalid alias expression')
+
+      ++ i
+
+  Object.freeze(path)
+
+  if not res?
+    return path
+
+  res.push(path)
+  return i
+
+
+addPathMeta = (path) ->
+  return path if path.alias? or path.wildcard?
+
+  path.alias = 0
+  path.wildcard = 0
+
+  for p, i in path
+    if typeOf(p) is ARRAY_TYPE
+      path.alias |= 1 << i
+      for j in [0...p.length] by 2
+        addPathMeta(p[j])
+    else if p == '*'
+      path.wildcard |= 1 << i
+
+  Object.freeze(path)
+  return path
+
+
+# Source: http://stackoverflow.com/a/6969486
+escapeRegExp = (str) ->
+  return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&")
+
+
+exports.UNDEFINED_TYPE = UNDEFINED_TYPE
+exports.NULL_TYPE = NULL_TYPE
+exports.NUMBER_TYPE = NUMBER_TYPE
+exports.STRING_TYPE = STRING_TYPE
+exports.ARRAY_TYPE = ARRAY_TYPE
+exports.OBJECT_TYPE = OBJECT_TYPE
+exports.REGEXP_TYPE = REGEXP_TYPE
+
+exports.typeOf = typeOf
+exports.pathOverlap = pathOverlap
+exports.parsePath = parsePath
+exports.addPathMeta = addPathMeta
+exports.escapeRegExp = escapeRegExp

lib/common.ts

@@ -1,63 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-export function generateDeviceId(
-  deviceIdStruct: Record<string, string>
-): string {
-  // Escapes everything except alphanumerics and underscore
-  function esc(str): string {
-    return str.replace(/[^A-Za-z0-9_]/g, (chr) => {
-      const buf = Buffer.from(chr, "utf8");
-      let rep = "";
-      for (const b of buf) rep += "%" + b.toString(16).toUpperCase();
-      return rep;
-    });
-  }
-
-  // Guaranteeing globally unique id as defined in TR-069
-  if (deviceIdStruct["ProductClass"]) {
-    return (
-      esc(deviceIdStruct["OUI"]) +
-      "-" +
-      esc(deviceIdStruct["ProductClass"]) +
-      "-" +
-      esc(deviceIdStruct["SerialNumber"])
-    );
-  }
-  return esc(deviceIdStruct["OUI"]) + "-" + esc(deviceIdStruct["SerialNumber"]);
-}
-
-// Source: http://stackoverflow.com/a/6969486
-export function escapeRegExp(str: string): string {
-  return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
-}
-
-export function encodeTag(tag: string): string {
-  return encodeURIComponent(tag)
-    .replace(
-      /[!~*'().]/g,
-      (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase()
-    )
-    .replace(/0x(?=[0-9A-Z]{2})/g, "0%78")
-    .replace(/%/g, "0x");
-}
-
-export function decodeTag(tag: string): string {
-  return decodeURIComponent(tag.replace(/0x(?=[0-9A-Z]{2})/g, "%"));
-}

lib/common/authorizer.ts

@@ -1,149 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { PermissionSet, Expression } from "../types";
-import { evaluate, or } from "./expression";
-
-export default class Authorizer {
-  private permissionSets: PermissionSet[];
-  private validatorCache: WeakMap<
-    any,
-    (mutationType, mutation, any) => boolean
-  >;
-  private hasAccessCache: Map<string, boolean>;
-  private getFilterCache: Map<string, Expression>;
-
-  public constructor(permissionSets: PermissionSet[]) {
-    this.permissionSets = permissionSets;
-    this.validatorCache = new WeakMap();
-    this.hasAccessCache = new Map();
-    this.getFilterCache = new Map();
-  }
-
-  public hasAccess(resourceType: string, access: number): boolean {
-    const cacheKey = `${resourceType}-${access}`;
-    if (this.hasAccessCache.has(cacheKey))
-      return this.hasAccessCache.get(cacheKey);
-
-    let has = false;
-    for (const permissionSet of this.permissionSets) {
-      for (const perm of permissionSet) {
-        if (perm[resourceType]) {
-          if (perm[resourceType].access >= access) {
-            has = true;
-            break;
-          }
-        }
-      }
-    }
-
-    this.hasAccessCache.set(cacheKey, has);
-    return has;
-  }
-
-  public getFilter(resourceType: string, access: number): Expression {
-    const cacheKey = `${resourceType}-${access}`;
-    if (this.getFilterCache.has(cacheKey))
-      return this.getFilterCache.get(cacheKey);
-
-    let filter: Expression = null;
-    for (const permissionSet of this.permissionSets) {
-      for (const perm of permissionSet) {
-        if (perm[resourceType]) {
-          if (perm[resourceType].access >= access)
-            filter = or(filter, perm[resourceType].filter);
-        }
-      }
-    }
-
-    this.getFilterCache.set(cacheKey, filter);
-    return filter;
-  }
-
-  public getValidator(
-    resourceType: string,
-    resource: unknown
-  ): (mutationType: string, mutation?: any, args?: any) => boolean {
-    if (this.validatorCache.has(resource))
-      return this.validatorCache.get(resource);
-
-    const validators: Expression[] = [];
-
-    for (const permissionSet of this.permissionSets) {
-      for (const perm of permissionSet) {
-        if (
-          perm[resourceType] &&
-          perm[resourceType].access >= 3 &&
-          perm[resourceType].validate
-        )
-          validators.push(perm[resourceType].validate);
-      }
-    }
-
-    const validator = (
-      mutationType: string,
-      mutation: any,
-      any: any
-    ): boolean => {
-      if (!validators.length) return false;
-
-      const object = {
-        mutationType,
-        mutation,
-        resourceType,
-        object: resource,
-        options: any,
-      };
-
-      const valueFunction = (paramName): any => {
-        const entry = paramName.split(".", 1)[0];
-        paramName = paramName.slice(entry.length + 1);
-        let value = null;
-        if (["mutation", "options"].includes(entry)) {
-          value = object[entry];
-          for (const seg of paramName.split(".")) {
-            // typeof null is "object"
-            if (value != null && typeof value !== "object") value = null;
-            else value = value[seg];
-            if (value == null) break;
-          }
-        } else if (object[entry]) {
-          if (paramName) value = object[entry][paramName];
-          else value = object[entry];
-        }
-
-        return value;
-      };
-
-      const res = evaluate(
-        validators.length > 1 ? ["OR", validators] : validators[0],
-        valueFunction,
-        Date.now()
-      );
-      return !Array.isArray(res) && !!res;
-    };
-
-    this.validatorCache.set(resource, validator);
-    return validator;
-  }
-
-  public getPermissionSets(): PermissionSet[] {
-    return this.permissionSets;
-  }
-}

lib/common/bigint.ts

@@ -1,106 +0,0 @@
-/**
- * Copyright 2013-2020  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-export function and(a: bigint, b: bigint): bigint {
-  return a & b;
-}
-
-export function or(a: bigint, b: bigint): bigint {
-  return a | b;
-}
-
-export function xor(a: bigint, b: bigint): bigint {
-  return a ^ b;
-}
-
-export function not(a: bigint): bigint {
-  return ~a;
-}
-
-export function lshift(a: bigint, b: bigint): bigint {
-  return a << b;
-}
-
-export function rshift(a: bigint, b: bigint): bigint {
-  return a >> b;
-}
-
-export function add(a: bigint, b: bigint): bigint {
-  return a + b;
-}
-
-export function sub(a: bigint, b: bigint): bigint {
-  return a - b;
-}
-
-export function mul(a: bigint, b: bigint): bigint {
-  return a * b;
-}
-
-export function div(a: bigint, b: bigint): bigint {
-  return a / b;
-}
-
-export function exp(a: bigint, b: bigint): bigint {
-  return a ** b;
-}
-
-export function rem(a: bigint, b: bigint): bigint {
-  return a % b;
-}
-
-export function toNumber(a: bigint): number {
-  return Number(a);
-}
-
-export function eq(a: bigint, b: bigint): boolean {
-  return a === b;
-}
-
-export function ne(a: bigint, b: bigint): boolean {
-  return a !== b;
-}
-
-export function lt(a: bigint, b: bigint): boolean {
-  return a < b;
-}
-
-export function lte(a: bigint, b: bigint): boolean {
-  return a <= b;
-}
-
-export function gt(a: bigint, b: bigint): boolean {
-  return a > b;
-}
-
-export function gte(a: bigint, b: bigint): boolean {
-  return a >= b;
-}
-
-export function asUintN(a: number, b: bigint): bigint {
-  return BigInt.asUintN(a, b);
-}
-
-export function asIntN(a: number, b: bigint): bigint {
-  return BigInt.asIntN(a, b);
-}
-
-const _BigInt = BigInt;
-type _bigint = bigint;
-export { _BigInt as BigInt, _bigint as bigint };

lib/common/expression-parser.ts

@@ -1,478 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import parsimmon from "parsimmon";
-import { Expression } from "../types";
-
-// Turn escaped characters into real ones (e.g. "\\n" becomes "\n").
-function interpretEscapes(str): string {
-  const escapes = {
-    b: "\b",
-    f: "\f",
-    n: "\n",
-    r: "\r",
-    t: "\t",
-  };
-  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
-    const type = escape.charAt(0);
-    const hex = escape.slice(1);
-    if (type === "u") return String.fromCharCode(parseInt(hex, 16));
-
-    if (escapes.hasOwnProperty(type)) return escapes[type];
-
-    return type;
-  });
-}
-
-export function map(
-  exp: Expression,
-  callback: (e: Expression) => Expression
-): Expression {
-  if (!Array.isArray(exp)) return callback(exp);
-
-  let clone;
-  for (let i = 1; i < exp.length; ++i) {
-    const sub = map(exp[i], callback);
-    if (sub !== exp[i]) {
-      clone = clone || exp.slice();
-      clone[i] = sub;
-    }
-  }
-
-  return callback(clone || exp);
-}
-
-export async function mapAsync(
-  exp: Expression,
-  callback: (e: Expression) => Promise<Expression>
-): Promise<Expression> {
-  if (!Array.isArray(exp)) return callback(exp);
-
-  let clone;
-  for (let i = 1; i < exp.length; ++i) {
-    const sub = await mapAsync(exp[i], callback);
-    if (sub !== exp[i]) {
-      clone = clone || exp.slice();
-      clone[i] = sub;
-    }
-  }
-
-  return callback(clone || exp);
-}
-
-function binaryLeft(
-  operatorsParser: parsimmon.Parser<string>,
-  nextParser: parsimmon.Parser<Expression>
-): parsimmon.Parser<Expression> {
-  return parsimmon.seqMap(
-    nextParser,
-    parsimmon.seq(operatorsParser, nextParser).many(),
-    (first, rest) =>
-      rest.reduce((acc, ch) => {
-        const [op, another] = ch;
-        if (Array.isArray(acc) && op === acc[0]) return acc.concat([another]);
-        if (Array.isArray(another) && op === another[0])
-          return [op, acc].concat(another.slice(1));
-        return [op, acc, another];
-      }, first)
-  );
-}
-
-const lang = parsimmon.createLanguage({
-  ComparisonOperator: function () {
-    return parsimmon
-      .alt(
-        parsimmon.string(">="),
-        parsimmon.string("<>"),
-        parsimmon.string("<="),
-        parsimmon.string("="),
-        parsimmon.string(">"),
-        parsimmon.string("<")
-      )
-      .skip(parsimmon.optWhitespace);
-  },
-  LikeOperator: function () {
-    return parsimmon
-      .alt(
-        parsimmon.regexp(/like/i).result("LIKE").desc("LIKE"),
-        parsimmon
-          .regexp(/not\s+like/i)
-          .result("NOT LIKE")
-          .desc("NOT LIKE")
-      )
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace);
-  },
-  IsNullOperator: function () {
-    return parsimmon
-      .alt(
-        parsimmon
-          .regexp(/is\s+null/i)
-          .result("IS NULL")
-          .desc("IS NULL"),
-        parsimmon
-          .regexp(/is\s+not\s+null/i)
-          .result("IS NOT NULL")
-          .desc("IS NOT NULL")
-      )
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace);
-  },
-  NotOperator: function () {
-    return parsimmon
-      .regexp(/not/i)
-      .result("NOT")
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace)
-      .desc("NOT");
-  },
-  AndOperator: function () {
-    return parsimmon
-      .regexp(/and/i)
-      .result("AND")
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace)
-      .desc("AND");
-  },
-  OrOperator: function () {
-    return parsimmon
-      .regexp(/or/i)
-      .result("OR")
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace)
-      .desc("OR");
-  },
-  Parameter: function (r) {
-    return parsimmon
-      .alt(
-        parsimmon.regexp(/[a-zA-Z0-9_.*-]+/),
-        r.Expression.wrap(
-          parsimmon.string("{").skip(parsimmon.optWhitespace),
-          parsimmon.string("}")
-        )
-      )
-      .atLeast(1)
-      .map((x) => ["PARAM", x.length > 1 ? ["||"].concat(x) : x[0]])
-      .skip(parsimmon.optWhitespace)
-      .desc("parameter");
-  },
-  StringValueSql: function () {
-    return parsimmon
-      .regexp(/'([^']*)'/, 1)
-      .atLeast(1)
-      .skip(parsimmon.optWhitespace)
-      .map((s) => s.join("'"))
-      .desc("string");
-  },
-  StringValueJs: function () {
-    return parsimmon
-      .regexp(/"((?:\\.|.)*?)"/, 1)
-      .skip(parsimmon.optWhitespace)
-      .map(interpretEscapes)
-      .desc("string");
-  },
-  NumberValue: function () {
-    return parsimmon
-      .regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/)
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace)
-      .map(Number)
-      .desc("number");
-  },
-  BooleanValue: function () {
-    return parsimmon
-      .alt(
-        parsimmon.regexp(/true/i).result(true).desc("TRUE"),
-        parsimmon.regexp(/false/i).result(false).desc("FALSE")
-      )
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace);
-  },
-  NullValue: function () {
-    return parsimmon
-      .regexp(/null/i)
-      .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-      .skip(parsimmon.optWhitespace)
-      .result(null)
-      .desc("NULL");
-  },
-  FuncValue: function (r) {
-    return parsimmon.seqMap(
-      parsimmon
-        .regexp(/([a-zA-Z0-9_]+)/, 1)
-        .skip(parsimmon.optWhitespace)
-        .desc("function"),
-      r.ExpressionList.wrap(
-        parsimmon.string("(").skip(parsimmon.optWhitespace),
-        parsimmon.string(")").skip(parsimmon.optWhitespace)
-      ),
-      (f, args) => ["FUNC", f.toUpperCase()].concat(args)
-    );
-  },
-  WhenPair: function (r) {
-    return parsimmon.seq(
-      parsimmon
-        .regexp(/when/i)
-        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-        .skip(parsimmon.optWhitespace)
-        .desc("WHEN")
-        .then(r.Expression),
-      parsimmon
-        .regexp(/then/i)
-        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-        .skip(parsimmon.optWhitespace)
-        .desc("THEN")
-        .then(r.Expression)
-    );
-  },
-  CaseStatement: function (r) {
-    return parsimmon.seqMap(
-      parsimmon
-        .regexp(/case/i)
-        .result("CASE")
-        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-        .skip(parsimmon.optWhitespace)
-        .desc("CASE"),
-      r.WhenPair.many(),
-      parsimmon
-        .regexp(/else/i)
-        .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-        .skip(parsimmon.optWhitespace)
-        .desc("ELSE")
-        .then(r.Expression)
-        .map((e) => [[true, e]])
-        .fallback(null)
-        .skip(
-          parsimmon
-            .regex(/end/i)
-            .notFollowedBy(parsimmon.regexp(/[a-zA-Z0-9_]/))
-        )
-        .skip(parsimmon.optWhitespace),
-      (...arr) => arr.flat(2)
-    );
-  },
-  Value: function (r) {
-    return parsimmon.alt(
-      r.NullValue,
-      r.BooleanValue,
-      r.NumberValue,
-      r.StringValueSql,
-      r.StringValueJs,
-      r.FuncValue,
-      r.CaseStatement
-    );
-  },
-  ValueExpression: function (r) {
-    return binaryLeft(
-      parsimmon.string("||").skip(parsimmon.optWhitespace),
-      binaryLeft(
-        parsimmon
-          .alt(parsimmon.string("+"), parsimmon.string("-"))
-          .skip(parsimmon.optWhitespace),
-        binaryLeft(
-          parsimmon
-            .alt(parsimmon.string("*"), parsimmon.string("/"))
-            .skip(parsimmon.optWhitespace),
-          parsimmon.alt(
-            r.Value,
-            r.Parameter,
-            r.Expression.wrap(
-              parsimmon.string("(").skip(parsimmon.optWhitespace),
-              parsimmon.string(")").skip(parsimmon.optWhitespace)
-            )
-          )
-        )
-      )
-    );
-  },
-  Comparison: function (r) {
-    return parsimmon.alt(
-      parsimmon.seqMap(r.ValueExpression, r.IsNullOperator, (p, o) => [o, p]),
-      parsimmon.seqMap(
-        r.ValueExpression,
-        r.ComparisonOperator,
-        r.ValueExpression,
-        (p, o, v) => [o, p, v]
-      ),
-      parsimmon.seqMap(
-        r.ValueExpression,
-        r.LikeOperator,
-        r.ValueExpression.skip(
-          parsimmon
-            .regexp(/escape/i)
-            .result("ESCAPE")
-            .skip(parsimmon.whitespace)
-            .desc("ESCAPE")
-        ),
-        r.ValueExpression,
-        (a, b, c, d) => [b, a, c, d]
-      ),
-      parsimmon.seqMap(
-        r.ValueExpression,
-        r.LikeOperator,
-        r.ValueExpression,
-        (a, b, c) => [b, a, c]
-      )
-    );
-  },
-  ExpressionList: function (r) {
-    return r.Expression.sepBy(
-      parsimmon.string(",").skip(parsimmon.optWhitespace)
-    );
-  },
-  Expression: function (r) {
-    function unary(
-      operatorsParser: parsimmon.Parser<string>,
-      nextParser: parsimmon.Parser<Expression>
-    ): parsimmon.Parser<Expression> {
-      return parsimmon.seq(operatorsParser, nextParser).or(nextParser);
-    }
-
-    return binaryLeft(
-      r.OrOperator,
-      binaryLeft(
-        r.AndOperator,
-        unary(r.NotOperator, r.Comparison.or(r.ValueExpression))
-      )
-    ).trim(parsimmon.optWhitespace);
-  },
-});
-
-export function parse(str: string): Expression {
-  if (!str) return null;
-  return lang.Expression.tryParse(str);
-}
-
-export function parseList(str: string): Expression[] {
-  if (!str) return [];
-  return lang.ExpressionList.tryParse(str);
-}
-
-export function stringify(exp: Expression, level = 0): string {
-  if (!Array.isArray(exp)) return JSON.stringify(exp);
-
-  const opLevels = {
-    OR: 10,
-    AND: 11,
-    NOT: 12,
-    "=": 20,
-    "<>": 20,
-    ">": 20,
-    ">=": 20,
-    "<": 20,
-    "<=": 20,
-    LIKE: 20,
-    "NOT LIKE": 20,
-    "IS NULL": 20,
-    "IS NOT NULL": 20,
-    "||": 30,
-    "+": 31,
-    "-": 31,
-    "*": 32,
-    "/": 32,
-  };
-
-  const op = exp[0].toUpperCase();
-
-  function wrap(e): string {
-    if (opLevels[op] <= level) return `(${e})`;
-    else return e;
-  }
-
-  if (op === "FUNC") {
-    return wrap(
-      `${exp[1]}(${exp
-        .slice(2)
-        .map((e) => stringify(e))
-        .join(", ")})`
-    );
-  } else if (op === "PARAM") {
-    if (typeof exp[1] === "string") {
-      return wrap(exp[1]);
-    } else if (Array.isArray(exp[1]) && exp[1][0] === "||") {
-      return wrap(
-        exp[1]
-          .slice(1)
-          .map((p) => {
-            if (typeof p === "string") return p;
-            else return `{${stringify(p)}}`;
-          })
-          .join("")
-      );
-    } else {
-      return wrap(`{${stringify(exp[1])}}`);
-    }
-  } else if (op === "IS NULL" || op === "IS NOT NULL") {
-    return wrap(`${stringify(exp[1], opLevels[op])} ${op}`);
-  } else if (op === "LIKE" || op === "NOT LIKE") {
-    if (exp[3]) {
-      return wrap(
-        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
-          exp[2],
-          opLevels[op]
-        )} ESCAPE ${stringify(exp[3], opLevels[op])}`
-      );
-    } else {
-      return wrap(
-        `${stringify(exp[1], opLevels[op])} ${op} ${stringify(
-          exp[2],
-          opLevels[op]
-        )}`
-      );
-    }
-  } else if (op === "CASE") {
-    const parts: string[] = ["CASE"];
-    for (let i = 1; i < exp.length - 1; i += 2) {
-      if (!Array.isArray(exp[i]) && exp[i]) {
-        if (exp[i + 1] != null) parts.push("ELSE", stringify(exp[i + 1]));
-        break;
-      }
-      parts.push("WHEN", stringify(exp[i]), "THEN", stringify(exp[i + 1]));
-    }
-    parts.push("END");
-    return parts.join(" ");
-  } else if (op in opLevels) {
-    const parts = exp.slice(1).map((e, i) => {
-      return stringify(e, opLevels[exp[0]] + Math.min(i - 1, 0));
-    });
-
-    if (op === "NOT") return wrap(`${op} ${parts[0]}`);
-    else return wrap(parts.join(` ${op} `));
-  } else {
-    throw new Error(`Unrecognized operator ${exp[0]}`);
-  }
-}
-
-export function parseLikePattern(pat: string, esc: string): string[] {
-  const chars = pat.split("");
-
-  for (let i = 0; i < chars.length; ++i) {
-    const c = chars[i];
-    if (c === esc) {
-      chars[i] = chars[i + 1] || "";
-      chars[i + 1] = "";
-    } else if (c === "_") {
-      chars[i] = "\\_";
-    } else if (c === "%") {
-      chars[i] = "\\%";
-      while (chars[i + 1] === "%") chars[++i] = "";
-    }
-  }
-  return chars.filter((c) => c);
-}

lib/common/expression.ts

@@ -1,408 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { map, mapAsync, parseLikePattern } from "./expression-parser";
-import { Expression } from "../types";
-
-const isArray = Array.isArray;
-
-const regExpCache = new WeakMap<any, RegExp>();
-
-const REDUCE_SKIP = {};
-function reduce(exp, callback): Expression {
-  let loop = true;
-  while (loop) {
-    loop = false;
-    for (let i = 2; i < exp.length; ++i) {
-      const res = callback(exp[i - 1], exp[i], i - 2);
-      if (res !== REDUCE_SKIP) {
-        loop = true;
-        exp = exp.slice();
-        exp.splice(i - 1, 2, res);
-      }
-    }
-  }
-  if (exp.length === 2) return exp[1];
-  return exp;
-}
-
-export function likePatternToRegExp(pat: string, esc = "", flags = ""): RegExp {
-  const convChars = {
-    "-": "\\-",
-    "/": "\\/",
-    "\\": "\\/",
-    "^": "\\^",
-    $: "\\$",
-    "*": "\\*",
-    "+": "\\+",
-    "?": "\\?",
-    ".": "\\.",
-    "(": "\\(",
-    ")": "\\)",
-    "|": "\\|",
-    "[": "\\[",
-    "]": "\\]",
-    "{": "\\{",
-    "}": "\\}",
-    "\\%": ".*",
-    "\\_": ".",
-  };
-  let chars = parseLikePattern(pat, esc);
-  if (!chars.length) return new RegExp("^$", flags);
-  chars = chars.map((c) => convChars[c] || c);
-  chars[0] = chars[0] === ".*" ? "" : "^" + chars[0];
-  const l = chars.length - 1;
-  chars[l] = [".*", ""].includes(chars[l]) ? "" : chars[l] + "$";
-  return new RegExp(chars.join(""), flags);
-}
-
-function compare(
-  a: boolean | number | string,
-  b: boolean | number | string
-): number {
-  if (typeof a === "boolean") a = +a;
-  if (typeof b === "boolean") b = +b;
-  if (typeof a !== typeof b) return typeof a === "string" ? 1 : -1;
-  return a > b ? 1 : a < b ? -1 : 0;
-}
-
-function toNumber(a: boolean | number | string): number {
-  switch (typeof a) {
-    case "number":
-      return a;
-    case "boolean":
-      return +a;
-    case "string":
-      return parseFloat(a) || 0;
-  }
-}
-
-function toString(a: boolean | number | string): string {
-  switch (typeof a) {
-    case "string":
-      return a;
-    case "number":
-      return a.toString();
-    case "boolean":
-      return (+a).toString();
-  }
-}
-
-export function evaluateCallback(exp: Expression): Expression {
-  if (!Array.isArray(exp)) return exp;
-  if (exp[0] === "CASE") {
-    for (let i = 1; i < exp.length; i += 2) {
-      if (Array.isArray(exp[i])) return exp;
-      if (exp[i]) return exp[i + 1];
-    }
-    return null;
-  } else if (exp[0] === "FUNC") {
-    if (exp[1] === "COALESCE") {
-      const args: Expression[] = [];
-      for (let i = 2; i < exp.length; ++i) {
-        const e = exp[i];
-        if (e == null) continue;
-        args.push(e);
-        if (!Array.isArray(e)) break;
-      }
-      if (!args.length) return null;
-      if (args.length === 1) return args[0];
-      return ["FUNC", "COALESCE", ...args];
-    } else if (exp[1] === "UPPER") {
-      if (exp[2] == null) return null;
-      if (!isArray(exp[2])) return toString(exp[2]).toUpperCase();
-    } else if (exp[1] === "LOWER") {
-      if (exp[2] == null) return null;
-      if (!isArray(exp[2])) return toString(exp[2]).toLowerCase();
-    } else if (exp[1] === "ROUND") {
-      const n = exp[2];
-      const p = exp.length > 3 ? exp[3] : 0;
-      if (n == null || p == null) return null;
-      if (!isArray(n) && !isArray(p)) {
-        const d = 10 ** p;
-        const m = n * d * (1 + Number.EPSILON);
-        return Math.round(m) / d;
-      }
-    }
-  } else if (exp[0] === "PARAM") {
-    if (exp[1] == null) return null;
-  } else if (exp[0] === "AND") {
-    for (let i = 1; i < exp.length; ++i)
-      if (!Array.isArray(exp[i]) && exp[i] != null && !exp[i]) return false;
-    const args: Expression[] = [];
-    for (let i = 1; i < exp.length; ++i) {
-      const ee = exp[i];
-      if (ee == null) return null;
-      if (Array.isArray(ee)) {
-        if (ee[0] === "AND") args.push(...ee.slice(1));
-        else args.push(ee);
-      }
-    }
-    if (!args.length) return true;
-    if (args.length === 1) args.push(true);
-    return ["AND", ...args];
-  } else if (exp[0] === "OR") {
-    const args: Expression[] = [];
-    for (let i = 1; i < exp.length; ++i) {
-      const ee = exp[i];
-      if (Array.isArray(ee)) {
-        if (ee[0] === "OR") args.push(...ee.slice(1));
-        else args.push(ee);
-      } else if (ee) {
-        return true;
-      }
-    }
-    if (!args.length) return exp.some((ee) => ee == null) ? null : false;
-    if (args.length === 1) args.push(false);
-    return ["OR", ...args];
-  } else if (exp[0] === "NOT") {
-    if (exp[1] == null) return null;
-    if (!isArray(exp[1])) return !exp[1];
-    else if (exp[1][0] === "NOT") return exp[1][1];
-  } else if (exp[0] === "IS NULL") {
-    if (isArray(exp[1])) return exp;
-    else if (exp[1] == null) return true;
-    else return null;
-  } else if (exp[0] === "IS NOT NULL") {
-    if (isArray(exp[1])) return exp;
-    else if (exp[1] != null) return true;
-    else return null;
-  } else if (exp[0] === "LIKE") {
-    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
-    else if (
-      exp[1] == null ||
-      exp[2] == null ||
-      ((exp as any[]).length >= 4 && exp[3] == null)
-    )
-      return null;
-    let r = regExpCache.get(exp as any[]);
-    if (!r) {
-      r = likePatternToRegExp(exp[2], exp[3]);
-      regExpCache.set(exp as any[], r);
-    }
-    return r.test(exp[1]);
-  } else if (exp[0] === "NOT LIKE") {
-    if (isArray(exp[1]) || isArray(exp[2]) || isArray(exp[3])) return exp;
-    else if (
-      exp[1] == null ||
-      exp[2] == null ||
-      ((exp as any[]).length >= 4 && exp[3] == null)
-    )
-      return null;
-    let r = regExpCache.get(exp as any[]);
-    if (!r) {
-      r = likePatternToRegExp(exp[2], exp[3]);
-      regExpCache.set(exp as any[], r);
-    }
-    return !r.test(exp[1]);
-  } else if (exp[0] === "=") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) === 0;
-  } else if (exp[0] === "<>") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) !== 0;
-  } else if (exp[0] === ">") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) > 0;
-  } else if (exp[0] === ">=") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) >= 0;
-  } else if (exp[0] === "<") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) < 0;
-  } else if (exp[0] === "<=") {
-    if (exp[1] == null || exp[2] == null) return null;
-    if (isArray(exp[1]) || isArray(exp[2])) return exp;
-    return compare(exp[1], exp[2]) <= 0;
-  } else if (exp[0] === "*") {
-    return reduce(exp, (a, b) => {
-      if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b)) return toNumber(a) * toNumber(b);
-      return REDUCE_SKIP;
-    });
-  } else if (exp[0] === "/") {
-    return reduce(exp, (a, b, i) => {
-      if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b))
-        return i === 0 ? toNumber(a) / toNumber(b) : toNumber(a) * toNumber(b);
-      return REDUCE_SKIP;
-    });
-  } else if (exp[0] === "+") {
-    return reduce(exp, (a, b) => {
-      if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b)) return toNumber(a) + toNumber(b);
-      return REDUCE_SKIP;
-    });
-  } else if (exp[0] === "-") {
-    return reduce(exp, (a, b, i) => {
-      if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b))
-        return i === 0 ? toNumber(a) - toNumber(b) : toNumber(a) + toNumber(b);
-      return REDUCE_SKIP;
-    });
-  } else if (exp[0] === "||") {
-    return reduce(exp, (a, b) => {
-      if (a == null || b == null) return null;
-      if (!isArray(a) && !isArray(b)) return toString(a) + toString(b);
-      return REDUCE_SKIP;
-    });
-  }
-  return exp;
-}
-
-export function evaluate(
-  exp: Expression,
-  obj: Record<string, unknown> | ((e: string) => Expression),
-  now: number,
-  cb?: (e: Expression) => Expression
-): string | number | boolean | null;
-export function evaluate(
-  exp: Expression,
-  obj?: Record<string, unknown> | ((e: string) => Expression),
-  now?: number,
-  cb?: (e: Expression) => Expression
-): Expression;
-export function evaluate(
-  exp: Expression,
-  obj?: Record<string, unknown> | ((e: string) => Expression),
-  now?: number,
-  cb?: (e: Expression) => Expression
-): Expression {
-  return map(exp, (e) => {
-    if (cb) e = cb(e);
-    if (!isArray(e)) return e;
-
-    if (e[0] === "FUNC" && e[1] === "NOW") {
-      if (now) return now;
-    } else if (e[0] === "PARAM") {
-      if (e[1] == null) return null;
-      if (obj && !isArray(e[1])) {
-        let v;
-        if (typeof obj === "function") v = obj(e[1]);
-        else v = obj[e[1]];
-
-        if (v == null) return null;
-        if (typeof v === "object") v = v.value ? v.value[0] : null;
-        return v;
-      }
-    }
-    return evaluateCallback(e);
-  });
-}
-
-export async function evaluateAsync(
-  exp: Expression,
-  obj: Record<string, unknown>,
-  now: number,
-  cb?: (e: Expression) => Promise<Expression>
-): Promise<string | number | boolean | null>;
-export async function evaluateAsync(
-  exp: Expression,
-  obj?: Record<string, unknown>,
-  now?: number,
-  cb?: (e: Expression) => Promise<Expression>
-): Promise<Expression>;
-export async function evaluateAsync(
-  exp: Expression,
-  obj?: Record<string, unknown>,
-  now?: number,
-  cb?: (e: Expression) => Promise<Expression>
-): Promise<Expression> {
-  return mapAsync(exp, async (e) => {
-    if (cb) e = await cb(e);
-    if (!isArray(e)) return e;
-
-    if (e[0] === "FUNC") {
-      if (e[1] === "NOW") {
-        if (now) return now;
-      } else if (e[1] === "UPPER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toUpperCase();
-      } else if (e[1] === "LOWER") {
-        if (e[2] == null) return null;
-        if (!isArray(e[2])) return `${e[2]}`.toLowerCase();
-      }
-    } else if (e[0] === "PARAM") {
-      if (e[1] == null) return null;
-      if (obj && !isArray(e[1])) {
-        let v = obj[e[1]];
-        if (v == null) return null;
-        if (typeof v === "object") v = v["value"] ? v["value"][0] : null;
-        return v as Expression;
-      }
-    }
-    return evaluateCallback(e);
-  });
-}
-
-export function and(exp1: Expression, exp2: Expression): Expression {
-  if (exp1 != null && !exp1) return false;
-  if (exp2 != null && !exp2) return false;
-  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
-    if (exp1 == null || exp2 == null) return null;
-    return true;
-  }
-  if (!Array.isArray(exp2) && exp1[0] === "AND") return exp1;
-  if (!Array.isArray(exp1) && exp2[0] === "AND") return exp2;
-
-  const res: Expression = ["AND"];
-
-  if (Array.isArray(exp1) && exp1[0] === "AND") res.push(...exp1.slice(1));
-  else res.push(exp1);
-
-  if (Array.isArray(exp2) && exp2[0] === "AND") res.push(...exp2.slice(1));
-  else res.push(exp2);
-
-  return res;
-}
-
-export function or(exp1: Expression, exp2: Expression): Expression {
-  if (!Array.isArray(exp1) && exp1) return true;
-  if (!Array.isArray(exp2) && exp2) return true;
-  if (!Array.isArray(exp1) && !Array.isArray(exp2)) {
-    if (exp1 == null || exp2 == null) return null;
-    return false;
-  }
-  if (!Array.isArray(exp2) && exp1[0] === "OR") return exp1;
-  if (!Array.isArray(exp1) && exp2[0] === "OR") return exp2;
-
-  const res: Expression = ["OR"];
-
-  if (Array.isArray(exp1) && exp1[0] === "OR") res.push(...exp1.slice(1));
-  else res.push(exp1);
-
-  if (Array.isArray(exp2) && exp2[0] === "OR") res.push(...exp2.slice(1));
-  else res.push(exp2);
-
-  return res;
-}
-
-export function extractParams(exp: Expression): Expression[] {
-  const params = new Set<Expression>();
-  map(exp, (e) => {
-    if (isArray(e) && e[0] === "PARAM") params.add(e[1]);
-    return e;
-  });
-  return Array.from(params);
-}

lib/common/memoize.ts

@@ -1,72 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-let cache1 = new Map();
-let cache2 = new Map();
-const keys = new WeakMap();
-
-function getKey(obj): string {
-  if (obj === null) return "null";
-  else if (obj === undefined) return "undefined";
-
-  const t = typeof obj;
-  if (t === "number" || t === "boolean" || t === "string") return `${t}:${obj}`;
-  if (t !== "function" && t !== "object")
-    throw new Error(`Cannot memoize ${t} arguments`);
-
-  let k = keys.get(obj);
-  if (!k) {
-    const rnd = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER);
-    k = `${t}:${rnd.toString(36)}`;
-    keys.set(obj, k);
-  }
-  return k;
-}
-
-export default function memoize<T extends (...args: any[]) => any>(func: T): T {
-  const funcKey = getKey(func);
-  return ((...args) => {
-    const key = JSON.stringify(args.map(getKey)) + funcKey;
-
-    if (cache1.has(key)) return cache1.get(key);
-
-    let r;
-    if (cache2.has(key)) {
-      cache1.set(key, (r = cache2.get(key)));
-    } else {
-      cache1.set(key, (r = func(...args)));
-      // Evict rejected promises
-      if (r instanceof Promise) {
-        r.catch(() => {
-          cache1.delete(key);
-          cache2.delete(key);
-        });
-      }
-    }
-    return r;
-  }) as any;
-}
-
-const interval = setInterval(() => {
-  cache2 = cache1;
-  cache1 = new Map();
-}, 120000);
-
-// Don't hold Node.js process
-if (interval.unref) interval.unref();

lib/common/path-set.ts

@@ -1,130 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./path";
-
-export default class PathSet {
-  private lengthIndex: Set<Path>[];
-  private fragmentIndex: Map<string, Set<Path>>[];
-  private stringIndex: Map<string, Path>;
-
-  public constructor() {
-    this.lengthIndex = [];
-    this.fragmentIndex = [];
-    this.stringIndex = new Map();
-  }
-
-  public get depth(): number {
-    return this.lengthIndex.length;
-  }
-
-  public add(path: Path): Path {
-    if (path.alias) throw new Error("PathSet does not support aliased paths");
-    const p = this.get(path);
-
-    if (p) return p;
-
-    this.stringIndex.set(path.toString(), path);
-
-    while (this.lengthIndex.length <= path.length) {
-      this.lengthIndex.push(new Set());
-      // fragmentIndex is one less than lengthIndex
-      if (this.lengthIndex.length > 1) this.fragmentIndex.push(new Map());
-    }
-
-    const lengthIndex = this.lengthIndex[path.length];
-    lengthIndex.add(path);
-
-    for (let i = 0; i < path.length; ++i) {
-      const fragment = path.segments[i] as string;
-      const fragmentIndex = this.fragmentIndex[i];
-
-      let fragmentIndexSet = fragmentIndex.get(fragment);
-      if (!fragmentIndexSet) {
-        fragmentIndexSet = new Set<Path>();
-        fragmentIndex.set(fragment, fragmentIndexSet);
-      }
-
-      fragmentIndexSet.add(path);
-    }
-
-    return path;
-  }
-
-  public get(path: Path): Path {
-    return this.stringIndex.get(path.toString()) || null;
-  }
-
-  public find(
-    path: Path,
-    superset = false,
-    subset = false,
-    depth: number = path.length
-  ): Path[] {
-    if (path.alias) throw new Error("PathSet does not support aliased paths");
-
-    const len = path.length;
-
-    if (!superset && depth === len && (!subset || !path.wildcard)) {
-      const p = this.get(path);
-      return p ? [p] : [];
-    }
-
-    const lengthIndex = this.lengthIndex.slice(len, depth + 1);
-    if (!lengthIndex.length) return [];
-
-    let res;
-    for (let i = len - 1; i >= 0; --i) {
-      let fragmentIndexSet2;
-      const fragmentIndex = this.fragmentIndex[i];
-
-      if ((path.wildcard >> i) & 1) {
-        if (subset) continue;
-      } else if (superset) {
-        fragmentIndexSet2 = fragmentIndex.get("*");
-      }
-
-      const fragment = path.segments[i] as string;
-      const fragmentIndexSet1 = fragmentIndex.get(fragment);
-
-      if (!fragmentIndexSet1) {
-        if (!fragmentIndexSet2) return [];
-        if (!res) res = [...fragmentIndexSet2];
-        else res = res.filter((r) => fragmentIndexSet2.has(r));
-      } else if (!fragmentIndexSet2) {
-        if (!res) res = [...fragmentIndexSet1];
-        else res = res.filter((r) => fragmentIndexSet1.has(r));
-      } else {
-        if (!res) {
-          res = [...fragmentIndexSet1, ...fragmentIndexSet2];
-        } else {
-          res = res.filter(
-            (r) => fragmentIndexSet1.has(r) || fragmentIndexSet2.has(r)
-          );
-        }
-      }
-      if (!res.length) return res;
-    }
-
-    if (!res) res = [].concat(...lengthIndex.map((a) => [...a]));
-    else res = res.filter((r) => lengthIndex.some((a) => a.has(r)));
-
-    return res;
-  }
-}

lib/common/path.ts

@@ -1,293 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-type Segments = (string | Alias)[];
-type Alias = [Path, string][];
-
-let cache1 = new Map<string, Path>();
-let cache2 = new Map<string, Path>();
-
-const CHAR_SPACE = 32;
-const CHAR_BACKSLASH = 92;
-const CHAR_COLON = 58;
-const CHAR_QUOTE = 34;
-const CHAR_OPEN_BRACKET = 91;
-const CHAR_CLOSE_BRACKET = 93;
-const CHAR_ASTERISK = 42;
-const CHAR_COMMA = 44;
-const CHAR_DOT = 46;
-
-function charCodeAt(str: string, index: number): number {
-  if (index >= str.length) return 0;
-  return str.charCodeAt(index);
-}
-
-function legalChar(c: number): boolean {
-  return (
-    (c >= 97 && c <= 122) ||
-    (c >= 65 && c <= 90) ||
-    (c >= 48 && c <= 57) ||
-    c === 95 ||
-    c === 45
-  );
-}
-
-export default class Path {
-  public readonly segments: Segments;
-  public readonly wildcard: number;
-  public readonly alias: number;
-  protected _string: string;
-  protected _stringIndex: number[];
-
-  protected static parseAliasValue(
-    pattern: string,
-    index: number
-  ): { index: number; value: string } {
-    let i = index;
-    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
-    if (charCodeAt(pattern, i) === CHAR_QUOTE) {
-      for (let j = i + 1; j < pattern.length; ++j) {
-        if (
-          pattern.charCodeAt(j) === CHAR_QUOTE &&
-          pattern.charCodeAt(j - 1) !== CHAR_BACKSLASH
-        ) {
-          try {
-            ++j;
-            const v = JSON.parse(pattern.slice(i, j));
-            return { index: j, value: v };
-          } catch {
-            return { index, value: null };
-          }
-        }
-      }
-      return { index, value: null };
-    }
-
-    for (; i < pattern.length; ++i) {
-      const c = pattern.charCodeAt(i);
-      if (c === CHAR_CLOSE_BRACKET || c === CHAR_COMMA) break;
-    }
-
-    return { index: i, value: pattern.slice(index, i).trim() };
-  }
-
-  protected static parseAliasPath(
-    pattern: string,
-    index: number
-  ): { index: number; path: Path } {
-    let i = index;
-    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
-    const { index: idx, segments } = Path.parsePath(pattern, i);
-    if (idx === i) return { index, path: null };
-    i = idx;
-    while (charCodeAt(pattern, i) === CHAR_SPACE) ++i;
-    if (charCodeAt(pattern, i) !== CHAR_COLON) return { index, path: null };
-    return { index: i + 1, path: new Path(segments) };
-  }
-
-  protected static parseAlias(
-    pattern: string,
-    index: number
-  ): { index: number; alias: Alias } {
-    const alias: Alias = [];
-    let i = index;
-    for (;;) {
-      let path: Path;
-      let value: string;
-      ({ index: i, path } = Path.parseAliasPath(pattern, i));
-      if (!path) break;
-      ({ index: i, value } = Path.parseAliasValue(pattern, i));
-      if (value == null) break;
-      alias.push([path, value]);
-      index = i;
-
-      if (charCodeAt(pattern, index) !== CHAR_COMMA) break;
-      ++i;
-    }
-
-    // Ensure identical expressions have idential string representation
-    alias.sort((a, b) => {
-      if (a[0].toString() > b[0].toString()) return 1;
-      else if (a[0].toString() < b[0].toString()) return -1;
-      else if (a[1] > b[1]) return 1;
-      else if (a[1] < b[1]) return -1;
-      else return 0;
-    });
-
-    Object.freeze(alias);
-    return { index, alias };
-  }
-
-  protected static parsePath(
-    pattern: string,
-    index: number
-  ): { index: number; segments: Segments } {
-    const segments = [];
-
-    let segStart = index;
-    let wildcard = -1;
-    for (let i = index; i <= pattern.length; ++i) {
-      const c = charCodeAt(pattern, i);
-      if (c === CHAR_OPEN_BRACKET && i === segStart) {
-        const { index: idx, alias } = Path.parseAlias(pattern, i + 1);
-        if (charCodeAt(pattern, idx) !== CHAR_CLOSE_BRACKET) break;
-        segments.push(alias);
-        index = i = idx + 1;
-        segStart = i + 1;
-      } else if (c === CHAR_ASTERISK && i === segStart) {
-        wildcard = i + 1;
-      } else if (!legalChar(c) || i === wildcard) {
-        if (i === segStart) break;
-        const s = pattern.slice(segStart, i);
-        segments.push(s);
-        index = i;
-        if (c !== CHAR_DOT) break;
-        segStart = i + 1;
-      }
-    }
-
-    Object.freeze(segments);
-    return { index, segments };
-  }
-
-  protected constructor(segments: Segments) {
-    let alias = 0;
-    let wildcard = 0;
-    const arr = segments.map((s, i) => {
-      if (Array.isArray(s)) {
-        alias |= 1 << i;
-        const parts = s.map(
-          (al) => `${al[0].toString()}:${JSON.stringify(al[1])}`
-        );
-        return `[${parts.join(",")}]`;
-      } else if (s === "*") {
-        wildcard |= 1 << i;
-      }
-      return s;
-    });
-
-    let offset = 0;
-    const stringIndex = arr.map((s, i) => (offset += s.length) + i);
-
-    this.segments = segments;
-    this.wildcard = wildcard;
-    this.alias = alias;
-    this._string = arr.join(".");
-    this._stringIndex = stringIndex;
-  }
-
-  public static parse(str: string): Path {
-    let path = cache1.get(str);
-    if (!path) {
-      path = cache2.get(str);
-      if (!path) {
-        const { index, segments } = Path.parsePath(str, 0);
-        if (index < str.length) throw new Error("Invalid parameter path");
-        path = new Path(segments);
-        if (path.toString() !== str) cache1.set(path.toString(), path);
-      }
-      cache1.set(str, path);
-    }
-    return path;
-  }
-
-  public get length(): number {
-    return this.segments.length;
-  }
-
-  public toString(): string {
-    return this._string;
-  }
-
-  public slice(start = 0, end: number = this.segments.length): Path {
-    if (start < 0) start = Math.max(0, this.segments.length + start);
-    if (end < 0) end = Math.max(0, this.segments.length + end);
-
-    let str;
-    if (start >= end) {
-      str = "";
-    } else {
-      const i1 = start > 0 ? this._stringIndex[start - 1] + 1 : 0;
-      const i2 =
-        end <= this.segments.length
-          ? this._stringIndex[end - 1]
-          : this._string.length;
-      str = this._string.slice(i1, i2);
-    }
-
-    let path = cache1.get(str);
-    if (!path) {
-      path = cache2.get(str);
-      if (!path) {
-        const segments = this.segments.slice(start, end);
-        Object.freeze(segments);
-        path = new Path(segments);
-      }
-      cache1.set(str, path);
-    }
-
-    return path;
-  }
-
-  public concat(path2: Path): Path {
-    if (!path2._string) return this;
-    else if (!this._string) return path2;
-
-    const str = `${this._string}.${path2._string}`;
-
-    let path = cache1.get(str);
-    if (!path) {
-      path = cache2.get(str);
-      if (!path) {
-        const segments = this.segments.concat(path2.segments);
-        Object.freeze(segments);
-        path = new Path(segments);
-      }
-      cache1.set(str, path);
-    }
-
-    return path;
-  }
-
-  public stripAlias(): Path {
-    if (!this.alias) return this;
-    const segments = this.segments.map((s) => (Array.isArray(s) ? "*" : s));
-    const str = segments.join(".");
-
-    let path = cache1.get(str);
-    if (!path) {
-      path = cache2.get(str);
-      if (!path) {
-        Object.freeze(segments);
-        path = new Path(segments);
-      }
-      cache1.set(str, path);
-    }
-
-    return path;
-  }
-}
-
-const interval = setInterval(() => {
-  cache2 = cache1;
-  cache1 = new Map();
-}, 120000);
-
-// Don't hold Node.js process
-if (interval.unref) interval.unref();

lib/common/yaml.ts

@@ -1,218 +0,0 @@
-const LINE_WIDTH = 80;
-const INDENTATION = "  ";
-
-const STRING_RESERVED = new Set([
-  "true",
-  "True",
-  "TRUE",
-  "false",
-  "False",
-  "FALSE",
-  "null",
-  "Null",
-  "NULL",
-]);
-
-function isPrintable(str: string): boolean {
-  return !/[^\t\n\x20-\x7e\x85\u{a0}-\u{d7ff}\u{e000}-\u{fffd}\u{10000}-\u{10ffff}]/u.test(
-    str
-  );
-}
-
-function stringifyKey(str: string): string {
-  if (!str || !isPrintable(str)) return JSON.stringify(str);
-  if (/^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|[\n,[\]{}]|\s$/.test(str))
-    return JSON.stringify(str);
-  return str;
-}
-
-function foldString(str: string): string[] {
-  if (str.length <= LINE_WIDTH) return [str];
-  if (str.startsWith(" ")) return [str];
-  const lines: string[] = [];
-
-  let idx = 0;
-  let cand = 0;
-  for (let i = 1; i < str.length - 1; ++i) {
-    if (str[i] !== " ") continue;
-
-    if (str[i + 1] === " ") {
-      i += 2;
-      while (str[i] === " ") ++i;
-      continue;
-    }
-
-    if (i <= idx + LINE_WIDTH) {
-      cand = i;
-      continue;
-    }
-
-    const c = cand > idx ? cand : i;
-    lines.push(str.slice(idx, c));
-    idx = c + 1;
-    cand = i;
-  }
-
-  if (cand > idx && str.length > idx + LINE_WIDTH) {
-    lines.push(str.slice(idx, cand));
-    idx = cand + 1;
-  }
-
-  lines.push(str.slice(idx));
-
-  return lines;
-}
-
-function stringifyString(str: string, res: string[], prefix1, prefix2): void {
-  if (/^\s*$/.test(str) || STRING_RESERVED.has(str) || !isPrintable(str)) {
-    res.push(prefix1 + JSON.stringify(str));
-    return;
-  }
-
-  if (!prefix2) prefix2 = INDENTATION;
-
-  const lines = str.split("\n");
-  if (lines.length > 1) {
-    let idt = "";
-    let chmp = "-";
-    if ((lines.find((l) => l) || "").startsWith(" "))
-      idt = `${INDENTATION.length}`;
-
-    if (!lines[lines.length - 1]) {
-      lines.pop();
-      if (lines[lines.length - 1]) chmp = "";
-      else chmp = "+";
-    }
-
-    if (/^\s+$/.test(lines[lines.length - 1])) {
-      res.push(prefix1 + JSON.stringify(str));
-      return;
-    }
-
-    let isFolded = false;
-    const folded = lines.map((l) => {
-      const ls = foldString(l);
-      if (ls.length > 1) isFolded = true;
-      return ls;
-    });
-
-    if (!isFolded) {
-      res.push(
-        `${prefix1}|${idt}${chmp}`,
-        ...lines.map((l) => (l ? prefix2 + l : l))
-      );
-      return;
-    }
-
-    res.push(`${prefix1}>${idt}${chmp}`);
-    res.push(...folded[0].map((f) => prefix2 + f));
-    for (let i = 1; i < folded.length; ++i) {
-      const prevLine = folded[i - 1][0];
-      if (prevLine && !folded[i - 1][0].startsWith(" ")) res.push("");
-      res.push(...folded[i].map((f) => prefix2 + f));
-    }
-    return;
-  }
-
-  if (
-    /^[\s-?:,[\]{}#&$!|>'"%@`]|: | #|\s$/.test(str) ||
-    parseFloat(str) === +str
-  ) {
-    res.push(prefix1 + JSON.stringify(str));
-    return;
-  }
-
-  res.push(prefix1 + str);
-}
-
-function stringifyAny(
-  obj: unknown,
-  res: string[],
-  prefix1 = "",
-  prefix2 = ""
-): void {
-  if (obj == null) {
-    res.push(`${prefix1}null`);
-    return;
-  }
-  if (typeof obj === "number" || typeof obj === "boolean") {
-    res.push(`${prefix1}${JSON.stringify(obj)}`);
-    return;
-  }
-  if (obj instanceof Date) {
-    res.push(`${prefix1}${obj.toJSON()}`);
-    return;
-  }
-
-  if (typeof obj === "string") {
-    stringifyString(obj, res, prefix1, prefix2);
-    return;
-  }
-
-  if (Array.isArray(obj)) {
-    if (!obj.length) {
-      res.push(prefix1 + "[]");
-      return;
-    }
-
-    if (!prefix1 || prefix1.endsWith("- ")) {
-      stringifyAny(obj[0], res, prefix1 + "- ", prefix2 + INDENTATION);
-      prefix1 = prefix2 + "- ";
-      prefix2 = prefix2 + INDENTATION;
-      for (let i = 1; i < obj.length; ++i)
-        stringifyAny(obj[i], res, prefix1, prefix2);
-    } else {
-      res.push(prefix1);
-      prefix1 = prefix2 + "- ";
-      prefix2 = prefix2 + INDENTATION;
-      for (let i = 0; i < obj.length; ++i)
-        stringifyAny(obj[i], res, prefix1, prefix2);
-    }
-    return;
-  }
-
-  const entries = Object.entries(obj).filter((e) => e[1] !== undefined);
-
-  if (!entries.length) {
-    res.push(prefix1 + "{}");
-    return;
-  }
-
-  if (!prefix1 || prefix1.endsWith("- ")) {
-    stringifyAny(
-      entries[0][1],
-      res,
-      prefix1 + `${stringifyKey(entries[0][0])}: `,
-      prefix2 + INDENTATION
-    );
-    prefix1 = prefix2;
-    prefix2 = prefix2 + INDENTATION;
-    for (let i = 1; i < entries.length; ++i) {
-      stringifyAny(
-        entries[i][1],
-        res,
-        prefix1 + `${stringifyKey(entries[i][0])}: `,
-        prefix2
-      );
-    }
-  } else {
-    res.push(prefix1);
-    prefix1 = prefix2;
-    prefix2 = prefix2 + INDENTATION;
-    for (let i = 0; i < entries.length; ++i) {
-      stringifyAny(
-        entries[i][1],
-        res,
-        prefix1 + `${stringifyKey(entries[i][0])}: `,
-        prefix2
-      );
-    }
-  }
-}
-
-export function stringify(obj: unknown): string {
-  if (obj === undefined) return undefined;
-  const lines: string[] = [];
-  stringifyAny(obj, lines);
-  return lines.join("\n") + "\n";
-}

lib/config.coffee

@@ -0,0 +1,209 @@
+###
+# Copyright 2013-2017  Zaid Abdulla
+#
+# This file is part of GenieACS.
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
+path = require 'path'
+common = require './common'
+
+options = {
+  CONFIG_DIR : {type : 'path', default : 'config'},
+  MONGODB_CONNECTION_URL : {type : 'string', default : 'mongodb://127.0.0.1/genieacs'},
+  REDIS_PORT : {type : 'int', default : 6379},
+  REDIS_HOST : {type : 'string', default : ''},
+  REDIS_DB : {type : 'int', default : 0},
+
+  CWMP_WORKER_PROCESSES : {type : 'int', default : 0},
+  CWMP_PORT : {type : 'int', default : 7547},
+  CWMP_INTERFACE : {type : 'string', default : '0.0.0.0'},
+  CWMP_SSL : {type : 'bool', default : false},
+  CWMP_LOG_FILE : {type: 'path', default : ''},
+  CWMP_ACCESS_LOG_FILE : {type : 'path', default : ''},
+
+  NBI_WORKER_PROCESSES : {type : 'int', default : 0},
+  NBI_PORT : {type : 'int', default : 7557},
+  NBI_INTERFACE : {type : 'string', default : '0.0.0.0'},
+  NBI_SSL : {type : 'bool', default : false},
+  NBI_LOG_FILE : {type: 'path', default : ''},
+  NBI_ACCESS_LOG_FILE : {type : 'path', default : ''},
+
+  FS_WORKER_PROCESSES : {type : 'int', default : 0},
+  FS_PORT : {type : 'int', default : 7567},
+  FS_INTERFACE : {type : 'string', default : '0.0.0.0'},
+  FS_SSL : {type : 'bool', default : false},
+  FS_HOSTNAME : {type : 'string', default : 'acs.example.com'},
+  FS_LOG_FILE : {type: 'path', default : ''},
+  FS_ACCESS_LOG_FILE : {type : 'path', default : ''},
+
+  UDP_CONNECTION_REQUEST_PORT : {type : 'int', default : 0},
+
+  DOWNLOAD_TIMEOUT: {type : 'int', default : 3600},
+  EXT_TIMEOUT: {type: 'int', default: 3000},
+  MAX_CACHE_TTL : {type : 'int', default : 86400},
+  DEBUG : {type : 'bool', default : false},
+  RETRY_DELAY : {type : 'int', default : 300},
+  SESSION_TIMEOUT : {type : 'int', default : 30},
+  CONNECTION_REQUEST_TIMEOUT : {type : 'int', default: 2000},
+  GPN_NEXT_LEVEL : {type : 'int', default : 0},
+  GPV_BATCH_SIZE : {type : 'int', default : 32},
+  MAX_DEPTH : {type: 'int', default : 16},
+  COOKIES_PATH : {type : 'string'},
+  LOG_FORMAT : {type : 'string', default : 'simple'},
+  ACCESS_LOG_FORMAT : {type : 'string', default : ''},
+  MAX_CONCURRENT_REQUESTS : {type : 'int', default: 20},
+  DATETIME_MILLISECONDS : {type: 'bool', default: true},
+  BOOLEAN_LITERAL : {type: 'bool', default: true},
+  CONNECTION_REQUEST_ALLOW_BASIC_AUTH: {type: 'bool', default: false},
+  MAX_COMMIT_ITERATIONS : {type : 'int', default: 32},
+
+  # XML configuration
+  XML_RECOVER : {type : 'bool', default : false},
+  XML_IGNORE_ENC : {type : 'bool', default : false},
+  XML_FORMAT : {type : 'bool', default : false},
+  XML_NO_DECL : {type : 'bool', default : false},
+  XML_NO_EMPTY : {type : 'bool', default : false},
+  XML_IGNORE_NAMESPACE : {type : 'bool', default : false},
+
+  # Should probably never be changed
+  DEVICE_ONLINE_THRESHOLD : {type : 'int', default : 4000}
+}
+
+allConfig = {}
+
+
+setConfig = (name, value, commandLineArgument) ->
+  return true if allConfig[name]?
+
+  # For compatibility with v1.0
+  if name in ['PRESETS_CACHE_DURATION', 'presets-cache-duration']
+    setConfig('MAX_CACHE_TTL', value)
+
+  if name in ['GET_PARAMETER_NAMES_DEPTH_THRESHOLD', 'get-parameter-names-depth-threshold']
+    setConfig('GPN_NEXT_LEVEL', value)
+
+  if name in ['TASK_PARAMETERS_BATCH_SIZE', 'task-parameters-batch-size']
+    setConfig('GPV_BATCH_SIZE', value)
+
+  if name in ['IGNORE_XML_NAMESPACES', 'ignore-xml-namespaces']
+    setConfig('XML_IGNORE_NAMESPACE', value)
+
+  if name in ['XML_PARSE_IGNORE_ENC', 'xml-parse-ignore-enc']
+    setConfig('XML_IGNORE_ENC', value)
+
+  if name in ['XML_PARSE_RECOVER', 'xml-parse-recover']
+    setConfig('XML_RECOVER', value)
+
+  if name in ['FS_IP', 'fs-ip']
+    setConfig('FS_HOSTNAME', value)
+
+  cast = (val, type) ->
+    switch type
+      when 'int'
+        Number(val)
+      when 'bool'
+        String(val).trim().toLowerCase() in ['true', 'on', 'yes', '1']
+      when 'string'
+        String(val)
+      when 'path'
+        if val then path.resolve(val) else ''
+      else
+        null
+
+  _value = null
+
+  for optionName, optionDetails of options
+    n = optionName
+    if commandLineArgument
+      n = n.toLowerCase().replace(/_/g, '-')
+
+    if name == n
+      _value = cast(value, optionDetails.type)
+      n = optionName
+    else if name.startsWith("#{n}-")
+      _value = cast(value, optionDetails.type)
+      n = "#{optionName}-#{name[optionName.length+1..]}"
+
+    if _value?
+      allConfig[n] = _value
+      # Save as environmnet variable to pass on to any child processes
+      process.env["GENIEACS_#{n}"] = _value
+      return true
+
+  return false
+
+
+# Command line arguments
+argv = process.argv[2..]
+while argv.length
+  arg = argv.shift()
+  if arg[0] == '-'
+    v = argv.shift()
+    setConfig(arg[2..], v, true)
+
+
+# Environment variable
+for k, v of process.env
+  continue if k.lastIndexOf('GENIEACS_', 0) != 0
+  k = k[9..] # remove "GENIEACS_" prefix
+  setConfig(k, v)
+
+
+# Use default config dir if none defined
+setConfig('CONFIG_DIR', options['CONFIG_DIR'].default)
+
+# Configuration file
+for k, v of require(path.resolve(allConfig.CONFIG_DIR, 'config'))
+  if not setConfig(k, v)
+    # Pass as environment variable to be accessable by extensions
+    process.env["GENIEACS_#{k}"] = v
+
+
+# Defaults
+for k, v of options
+  setConfig(k, v.default) if v.default?
+
+
+get = (option, deviceId) ->
+  if deviceId?
+    name = "#{option}-#{deviceId}"
+    v = allConfig[name]
+    return v if v?
+    i = name.lastIndexOf('-')
+    v = allConfig[name[0...i]]
+    return v if v?
+    i = name.lastIndexOf('-', i-1)
+    v = allConfig[name[0...i]]
+    return v if v? or i == -1
+
+  return allConfig[option]
+
+
+getDefault = (optionName) ->
+  option = options[optionName]
+  return null if not option
+  val = option.default
+  if val and option.type == 'path'
+    val = path.resolve(val)
+  return val
+
+
+# load authentication scripts
+try
+  exports.auth = require(path.resolve(allConfig.CONFIG_DIR, 'auth'))
+
+exports.get = get
+exports.getDefault = getDefault

lib/config.ts

@@ -1,295 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { resolve } from "path";
-import { readFileSync, existsSync } from "fs";
-
-// Find project root directory
-export let ROOT_DIR = resolve(__dirname, "..");
-while (!existsSync(`${ROOT_DIR}/package.json`)) {
-  const d = resolve(ROOT_DIR, "..");
-  if (d === ROOT_DIR) {
-    ROOT_DIR = process.cwd();
-    break;
-  }
-  ROOT_DIR = d;
-}
-
-// For compatibility with v1.1
-let configDir, cwmpSsl, nbiSsl, fsSsl, uiSsl, fsHostname;
-
-const options = {
-  EXT_DIR: { type: "path", default: resolve(ROOT_DIR, "config/ext") },
-  MONGODB_CONNECTION_URL: {
-    type: "string",
-    default: "mongodb://127.0.0.1/genieacs",
-  },
-
-  CWMP_WORKER_PROCESSES: { type: "int", default: 0 },
-  CWMP_PORT: { type: "int", default: 7547 },
-  CWMP_INTERFACE: { type: "string", default: "0.0.0.0" },
-  CWMP_SSL_CERT: { type: "string", default: "" },
-  CWMP_SSL_KEY: { type: "string", default: "" },
-  CWMP_LOG_FILE: { type: "path", default: "" },
-  CWMP_ACCESS_LOG_FILE: { type: "path", default: "" },
-
-  NBI_WORKER_PROCESSES: { type: "int", default: 0 },
-  NBI_PORT: { type: "int", default: 7557 },
-  NBI_INTERFACE: { type: "string", default: "0.0.0.0" },
-  NBI_SSL_CERT: { type: "string", default: "" },
-  NBI_SSL_KEY: { type: "string", default: "" },
-  NBI_LOG_FILE: { type: "path", default: "" },
-  NBI_ACCESS_LOG_FILE: { type: "path", default: "" },
-
-  FS_WORKER_PROCESSES: { type: "int", default: 0 },
-  FS_PORT: { type: "int", default: 7567 },
-  FS_INTERFACE: { type: "string", default: "0.0.0.0" },
-  FS_SSL_CERT: { type: "string", default: "" },
-  FS_SSL_KEY: { type: "string", default: "" },
-  FS_URL_PREFIX: { type: "string", default: "" },
-  FS_LOG_FILE: { type: "path", default: "" },
-  FS_ACCESS_LOG_FILE: { type: "path", default: "" },
-
-  UI_WORKER_PROCESSES: { type: "int", default: 0 },
-  UI_PORT: { type: "int", default: 3000 },
-  UI_INTERFACE: { type: "string", default: "0.0.0.0" },
-  UI_SSL_CERT: { type: "string", default: "" },
-  UI_SSL_KEY: { type: "string", default: "" },
-  UI_LOG_FILE: { type: "path", default: "" },
-  UI_ACCESS_LOG_FILE: { type: "path", default: "" },
-  UI_JWT_SECRET: { type: "string", default: "" },
-
-  UDP_CONNECTION_REQUEST_PORT: { type: "int", default: 0 },
-  FORWARDED_HEADER: { type: "string", default: "" },
-
-  DOWNLOAD_TIMEOUT: { type: "int", default: 3600 },
-  EXT_TIMEOUT: { type: "int", default: 3000 },
-  MAX_CACHE_TTL: { type: "int", default: 86400 },
-  DEBUG_FILE: { type: "path", default: "" },
-  DEBUG_FORMAT: { type: "string", default: "yaml" },
-  DEBUG: { type: "bool", default: false },
-  RETRY_DELAY: { type: "int", default: 300 },
-  SESSION_TIMEOUT: { type: "int", default: 30 },
-  CONNECTION_REQUEST_TIMEOUT: { type: "int", default: 2000 },
-  GPN_NEXT_LEVEL: { type: "int", default: 0 },
-  GPV_BATCH_SIZE: { type: "int", default: 32 },
-  MAX_DEPTH: { type: "int", default: 16 },
-  COOKIES_PATH: { type: "string" },
-  LOG_FORMAT: { type: "string", default: "simple" },
-  ACCESS_LOG_FORMAT: { type: "string", default: "" },
-  MAX_CONCURRENT_REQUESTS: { type: "int", default: 20 },
-  DATETIME_MILLISECONDS: { type: "bool", default: true },
-  BOOLEAN_LITERAL: { type: "bool", default: true },
-  CONNECTION_REQUEST_ALLOW_BASIC_AUTH: { type: "bool", default: false },
-  MAX_COMMIT_ITERATIONS: { type: "int", default: 32 },
-
-  // Should probably never be changed
-  DEVICE_ONLINE_THRESHOLD: { type: "int", default: 4000 },
-};
-
-const allConfig: { [name: string]: string | number } = {};
-
-function setConfig(name, value, commandLineArgument = false): boolean {
-  if (allConfig[name] != null) return true;
-
-  // For compatibility with v1.1
-  if (name === "CONFIG_DIR" || name === "config-dir")
-    configDir = configDir || resolve(ROOT_DIR, value);
-
-  if (name === "CWMP_SSL" || name === "cwmp-ssl")
-    cwmpSsl = cwmpSsl || String(value).toLowerCase().trim();
-
-  if (name === "NBI_SSL" || name === "nbi-ssl")
-    nbiSsl = nbiSsl || String(value).toLowerCase().trim();
-
-  if (name === "FS_SSL" || name === "fs-ssl")
-    fsSsl = fsSsl || String(value).toLowerCase().trim();
-
-  if (name === "UI_SSL" || name === "ui-ssl")
-    uiSsl = uiSsl || String(value).toLowerCase().trim();
-
-  if (name === "FS_HOSTNAME" || name === "fs-hostname")
-    fsHostname = fsHostname || String(value).trim();
-
-  // For compatibility with v1.0
-  if (name === "PRESETS_CACHE_DURATION" || name === "presets-cache-duration")
-    setConfig("MAX_CACHE_TTL", value);
-
-  if (
-    name === "GET_PARAMETER_NAMES_DEPTH_THRESHOLD" ||
-    name === "get-parameter-names-depth-threshold"
-  )
-    setConfig("GPN_NEXT_LEVEL", value);
-
-  if (
-    name === "TASK_PARAMETERS_BATCH_SIZE" ||
-    name === "task-parameters-batch-size"
-  )
-    setConfig("GPV_BATCH_SIZE", value);
-
-  if (name === "FS_IP" || name === "fs-ip") setConfig("FS_HOSTNAME", value);
-
-  function cast(val, type): string | number | boolean {
-    switch (type) {
-      case "int":
-        return Number(val);
-      case "bool":
-        return ["true", "1"].includes(String(val).trim().toLowerCase());
-      case "string":
-        return String(val);
-      case "path":
-        if (!val) return "";
-        return resolve(val);
-      default:
-        return null;
-    }
-  }
-
-  let _value = null;
-  for (const [optionName, optionDetails] of Object.entries(options)) {
-    let n = optionName;
-    if (commandLineArgument) n = n.toLowerCase().replace(/_/g, "-");
-
-    if (name === n) {
-      _value = cast(value, optionDetails.type);
-      n = optionName;
-    } else if (name.startsWith(`${n}-`)) {
-      _value = cast(value, optionDetails.type);
-      n = `${optionName}-${name.slice(optionName.length + 1)}`;
-    }
-
-    if (_value != null) {
-      allConfig[n] = _value;
-      // Save as environmnet variable to pass on to any child process
-      process.env[`GENIEACS_${n}`] = _value;
-      return true;
-    }
-  }
-
-  return false;
-}
-
-// Command line arguments
-const argv = process.argv.slice(2);
-while (argv.length) {
-  const arg = argv.shift();
-  if (arg[0] === "-") {
-    const v = argv.shift();
-    setConfig(arg.slice(2), v, true);
-  }
-}
-
-// Environment variable
-for (const [k, v] of Object.entries(process.env))
-  if (k.startsWith("GENIEACS_")) setConfig(k.slice(9), v);
-
-// Configuration file
-const configFilename = configDir
-  ? `${configDir}/config.json`
-  : `${ROOT_DIR}/config/config.json`;
-
-if (existsSync(configFilename)) {
-  const configFile = JSON.parse(readFileSync(configFilename).toString());
-
-  for (const [k, v] of Object.entries(configFile)) {
-    if (!setConfig(k, v))
-      // Pass as environment variable to be accessable by extensions
-      process.env[`GENIEACS_${k}`] = `${v}`;
-  }
-}
-
-if (configDir) setConfig("EXT_DIR", `${configDir}/ext`);
-
-if (["true", "1"].includes(cwmpSsl)) {
-  const d = configDir || `${ROOT_DIR}/config`;
-  setConfig("CWMP_SSL_CERT", `${d}/cwmp.crt`);
-  setConfig("CWMP_SSL_KEY", `${d}/cwmp.key`);
-}
-
-if (["true", "1"].includes(nbiSsl)) {
-  const d = configDir || `${ROOT_DIR}/config`;
-  setConfig("NBI_SSL_CERT", `${d}/cwmp.crt`);
-  setConfig("NBI_SSL_KEY", `${d}/cwmp.key`);
-}
-
-if (["true", "1"].includes(fsSsl)) {
-  const d = configDir || `${ROOT_DIR}/config`;
-  setConfig("FS_SSL_CERT", `${d}/cwmp.crt`);
-  setConfig("FS_SSL_KEY", `${d}/cwmp.key`);
-}
-
-if (["true", "1"].includes(uiSsl)) {
-  const d = configDir || `${ROOT_DIR}/config`;
-  setConfig("UI_SSL_CERT", `${d}/cwmp.crt`);
-  setConfig("UI_SSL_KEY", `${d}/cwmp.key`);
-}
-
-if (fsHostname) {
-  const FS_PORT = allConfig["FS_PORT"] || 7567;
-  const FS_SSL = !!allConfig["FS_SSL_CERT"];
-  setConfig(
-    "FS_URL_PREFIX",
-    (FS_SSL ? "https" : "http") + `://${fsHostname}:${FS_PORT}/`
-  );
-}
-
-// Defaults
-for (const [k, v] of Object.entries(options))
-  if (v["default"] != null) setConfig(k, v["default"]);
-
-export function get(
-  optionName: string,
-  deviceId?: string
-): string | number | boolean {
-  if (!deviceId) return allConfig[optionName];
-
-  optionName = `${optionName}-${deviceId}`;
-  let v = allConfig[optionName];
-  if (v != null) return v;
-
-  let i = optionName.lastIndexOf("-");
-  v = allConfig[optionName.slice(0, i)];
-  if (v != null) return v;
-
-  i = optionName.lastIndexOf("-", i - 1);
-  v = allConfig[optionName.slice(0, i)];
-  if (v != null) return v;
-
-  i = optionName.lastIndexOf("-", i - 1);
-  v = allConfig[optionName.slice(0, i)];
-  if (v != null) return v;
-
-  i = optionName.lastIndexOf("-", i - 1);
-  if (i > 0) {
-    v = allConfig[optionName.slice(0, i)];
-    if (v != null) return v;
-  }
-
-  return null;
-}
-
-export function getDefault(optionName: string): string | number | boolean {
-  const option = options[optionName];
-  if (!option) return null;
-
-  let val = option["default"];
-  if (val && option.type === "path") val = resolve(val);
-
-  return val;
-}

lib/connection-request.ts

@@ -1,239 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as crypto from "crypto";
-import * as dgram from "dgram";
-import { parse } from "url";
-import * as http from "http";
-import { evaluateAsync } from "./common/expression";
-import { Expression } from "./types";
-import * as auth from "./auth";
-import * as extensions from "./extensions";
-import * as debug from "./debug";
-
-async function extractAuth(
-  exp: Expression,
-  dflt: any
-): Promise<[string, string, Expression]> {
-  let username, password;
-  const _exp = await evaluateAsync(
-    exp,
-    {},
-    0,
-    async (e: Expression): Promise<Expression> => {
-      if (!username && Array.isArray(e) && e[0] === "FUNC") {
-        if (e[1] === "EXT") {
-          if (typeof e[2] !== "string" || typeof e[3] !== "string") return null;
-
-          for (let i = 4; i < e.length; i++)
-            if (Array.isArray(e[i])) return null;
-
-          const { fault, value } = await extensions.run(e.slice(2));
-          return fault ? null : value;
-        } else if (e[1] === "AUTH") {
-          if (!Array.isArray(e[2]) && !Array.isArray(e[3])) {
-            username = e[2] || "";
-            password = e[3] || "";
-          }
-          return dflt;
-        }
-      }
-      return e;
-    }
-  );
-  return [username, password, _exp];
-}
-
-function httpGet(
-  options: http.RequestOptions,
-  _debug: boolean,
-  deviceId: string
-): Promise<{ statusCode: number; headers: http.IncomingHttpHeaders }> {
-  return new Promise((resolve, reject) => {
-    const req = http
-      .get(options, (res) => {
-        res.resume();
-        resolve({ statusCode: res.statusCode, headers: res.headers });
-        if (_debug) {
-          debug.outgoingHttpRequest(req, deviceId, options, null);
-          debug.incomingHttpResponse(res, deviceId, null);
-        }
-      })
-      .on("error", (err) => {
-        req.destroy();
-        reject(err);
-        if (_debug) debug.outgoingHttpRequestError(req, deviceId, options, err);
-      })
-      .on("timeout", () => {
-        req.destroy();
-      });
-  });
-}
-
-export async function httpConnectionRequest(
-  address: string,
-  authExp: Expression,
-  allowBasicAuth: boolean,
-  timeout: number,
-  _debug: boolean,
-  deviceId: string
-): Promise<string> {
-  const options: http.RequestOptions = parse(address);
-  if (options.protocol !== "http:")
-    return "Invalid connection request URL or protocol";
-
-  options.agent = new http.Agent({
-    maxSockets: 1,
-    keepAlive: true,
-    timeout: timeout,
-  });
-
-  let authHeader: Record<string, string>;
-  let username: string;
-  let password: string;
-
-  while (!authHeader || (username != null && password != null)) {
-    let opts = options;
-    if (authHeader) {
-      if (authHeader["method"] === "Basic") {
-        if (!allowBasicAuth) return "Basic HTTP authentication not allowed";
-
-        opts = Object.assign(
-          {
-            headers: {
-              Authorization: auth.basic(username || "", password || ""),
-            },
-          },
-          options
-        );
-      } else if (authHeader["method"] === "Digest") {
-        opts = Object.assign(
-          {
-            headers: {
-              Authorization: auth.solveDigest(
-                username,
-                password,
-                options.path,
-                "GET",
-                null,
-                authHeader
-              ),
-            },
-          },
-          options
-        );
-      } else {
-        return "Unrecognized auth method";
-      }
-    }
-
-    let res: { statusCode: number; headers: http.IncomingHttpHeaders };
-    try {
-      res = await httpGet(opts, _debug, deviceId);
-    } catch (err) {
-      // Workaround for some devices unexpectedly closing the connection
-      if (authHeader) {
-        try {
-          res = await httpGet(opts, _debug, deviceId);
-        } catch (err) {
-          return `Connection request error: ${err.message}`;
-        }
-      }
-
-      if (err["code"] === "ECONNRESET" || err["code"] === "ECONNREFUSED")
-        return "Device is offline";
-
-      return `Connection request error: ${err.message}`;
-    }
-
-    if (res.statusCode === 200 || res.statusCode === 204) return "";
-
-    // When a Connection Request is received for the Virtual CWMP Device and the Proxied
-    // Device is offline the CPE Proxier MUST respond with an HTTP 503 failure
-    if (res.statusCode === 503) return "Device is offline";
-
-    if (res.statusCode === 401 && res.headers["www-authenticate"]) {
-      try {
-        authHeader = auth.parseWwwAuthenticateHeader(
-          res.headers["www-authenticate"]
-        );
-      } catch (err) {
-        return "Connection request error: Error parsing www-authenticate header";
-      }
-      [username, password, authExp] = await extractAuth(authExp, false);
-    } else {
-      return `Connection request error: Unexpected status code ${res.statusCode}`;
-    }
-  }
-
-  return "Connection request error: Incorrect connection request credentials";
-}
-
-export async function udpConnectionRequest(
-  host: string,
-  port: number,
-  authExp: Expression,
-  sourcePort = 0,
-  _debug: boolean,
-  deviceId: string
-): Promise<void> {
-  const now = Date.now();
-
-  const client = dgram.createSocket({ type: "udp4", reuseAddr: true });
-  // When a device is NAT'ed, the UDP Connection Request must originate from
-  // the same address and port used by the STUN server, in order to traverse
-  // the firewall. This does require that the Genieacs NBI and STUN server
-  // are allowed to bind to the same address and port. The STUN server needs
-  // to open its UDP port with the SO_REUSEADDR option, allowing the NBI to
-  // also bind to the same port.
-  if (sourcePort) client.bind({ port: sourcePort, exclusive: true });
-
-  let username: string;
-  let password: string;
-
-  [username, password, authExp] = await extractAuth(authExp, null);
-
-  if (username == null) username = "";
-  if (password == null) password = "";
-  while (username != null && password != null) {
-    const ts = Math.trunc(now / 1000);
-    const id = Math.trunc(Math.random() * 4294967295);
-    const cn = crypto.randomBytes(8).toString("hex");
-    const sig = crypto
-      .createHmac("sha1", password)
-      .update(`${ts}${id}${username}${cn}`)
-      .digest("hex");
-    const uri = `http://${host}:${port}?ts=${ts}&id=${id}&un=${username}&cn=${cn}&sig=${sig}`;
-    const msg = `GET ${uri} HTTP/1.1\r\nHost: ${host}:${port}\r\n\r\n`;
-    const message = Buffer.from(msg);
-
-    for (let i = 0; i < 3; ++i) {
-      await new Promise<void>((resolve, reject) => {
-        client.send(message, 0, message.length, port, host, (err: Error) => {
-          if (err) reject(err);
-          else resolve();
-          if (_debug) debug.outgoingUdpMessage(host, deviceId, port, msg);
-        });
-      });
-    }
-
-    [username, password, authExp] = await extractAuth(authExp, null);
-  }
-  client.close();
-}

lib/cwmp.coffee

@@ -0,0 +1,965 @@
+###
+# Copyright 2013-2017  Zaid Abdulla
+#
+# This file is part of GenieACS.
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
+util = require 'util'
+zlib = require 'zlib'
+crypto = require 'crypto'
+
+config = require './config'
+common = require './common'
+soap = require './soap'
+session = require './session'
+query = require './query'
+device = require './device'
+cache = require './cache'
+localCache = require './local-cache'
+db = require './db'
+logger = require './logger'
+scheduling = require './scheduling'
+
+
+MAX_CYCLES = 4
+
+MAX_CONCURRENT_REQUESTS = config.get('MAX_CONCURRENT_REQUESTS')
+stats = {
+  concurrentRequests: 0
+  totalRequests: 0
+  droppedRequests: 0
+  initiatedSessions: 0
+}
+
+throwError = (err, httpResponse) ->
+  if httpResponse
+    currentSessions.delete(httpResponse.connection)
+    httpResponse.writeHead(500, {'Connection' : 'close'})
+    httpResponse.end("#{err.name}: #{err.message}")
+    stats.concurrentRequests -= 1
+
+  throw err
+
+
+writeResponse = (sessionContext, res, close) ->
+  # Close connection after last request in session
+  res.headers['Connection'] = 'close' if close
+
+  if config.get('DEBUG', sessionContext.deviceId)
+    dump = "# RESPONSE #{new Date(Date.now())}\n" + JSON.stringify(res.headers) + "\n#{res.data}\n\n"
+    require('fs').appendFile("./debug/#{sessionContext.deviceId}.dump", dump, (err) ->
+      throwError(err) if err
+    )
+
+  finish = () ->
+    stats.concurrentRequests -= 1
+    if sessionContext.httpRequest.connection.destroyed
+      logger.accessError({
+        sessionContext: sessionContext
+        message: 'Connection dropped'
+      })
+    else if close
+      endSession(sessionContext, (err, isNew) ->
+        return throwError(err) if err
+
+        if isNew
+          logger.accessInfo({
+            sessionContext: sessionContext
+            message: 'New device registered'
+          })
+      )
+    else
+      sessionContext.lastActivity = Date.now()
+      currentSessions.set(sessionContext.httpRequest.connection, sessionContext)
+
+  # respond using the same content-encoding as the request
+  if sessionContext.httpRequest.headers['content-encoding']? and res.data.length > 0
+    switch sessionContext.httpRequest.headers['content-encoding']
+      when 'gzip'
+        res.headers['Content-Encoding'] = 'gzip'
+        compress = zlib.gzip
+      when 'deflate'
+        res.headers['Content-Encoding'] = 'deflate'
+        compress = zlib.deflate
+
+  if compress?
+    compress(res.data, (err, data) ->
+      return throwError(err, sessionContext.httpResponse) if err
+      res.headers['Content-Length'] = data.length
+      sessionContext.httpResponse.writeHead(res.code, res.headers)
+      sessionContext.httpResponse.end(data)
+      finish()
+    )
+  else
+    res.headers['Content-Length'] = res.data.length
+    sessionContext.httpResponse.writeHead(res.code, res.headers)
+    sessionContext.httpResponse.end(res.data)
+    finish()
+
+
+recordFault = (sessionContext, fault, provisions, channels) ->
+  if not provisions
+    provisions = sessionContext.provisions
+    channels = sessionContext.channels
+
+  faults = sessionContext.faults
+
+  for channel of channels
+    provs = sessionContext.faults[channel]?.provisions or []
+    faults[channel] = Object.assign({provisions: provs}, fault)
+    if channel.startsWith('task_')
+      taskId = channel.slice(5)
+      for t in sessionContext.tasks
+        faults[channel].expiry = t.expiry if t._id == taskId and t.expiry
+
+    if sessionContext.retries[channel]?
+      ++ sessionContext.retries[channel]
+    else
+      sessionContext.retries[channel] = 0
+      if Object.keys(channels).length != 1
+        faults[channel].retryNow = true
+
+    if channels[channel] == 0
+      faults[channel].precondition = true
+
+    sessionContext.faultsTouched ?= {}
+    sessionContext.faultsTouched[channel] = true
+
+    logger.accessWarn({
+      sessionContext: sessionContext
+      message: 'Channel has faulted'
+      fault: fault
+      channel: channel
+      retries: sessionContext.retries[channel]
+    })
+
+  for provision, i in provisions
+    for channel of channels
+      if (channels[channel] >> i) & 1
+        faults[channel].provisions.push(provision)
+
+  for channel of channels
+    provs = faults[channel].provisions
+    faults[channel].provisions = []
+    appendProvisions(faults[channel].provisions, provs)
+
+  session.clearProvisions(sessionContext)
+  return
+
+
+inform = (sessionContext, rpc) ->
+  session.inform(sessionContext, rpc.cpeRequest, (err, acsResponse) ->
+    return throwError(err, sessionContext.httpResponse) if err
+    res = soap.response({
+      id: rpc.id,
+      acsResponse: acsResponse,
+      cwmpVersion : sessionContext.cwmpVersion
+    })
+
+    if !!cookiesPath = config.get('COOKIES_PATH', sessionContext.deviceId)
+      res.headers['Set-Cookie'] = "session=#{sessionContext.sessionId}; Path=#{cookiesPath}"
+    else
+      res.headers['Set-Cookie'] = "session=#{sessionContext.sessionId}"
+
+    writeResponse(sessionContext, res)
+  )
+
+
+transferComplete = (sessionContext, rpc) ->
+  session.transferComplete(sessionContext, rpc.cpeRequest, (err, acsResponse, operation, fault) ->
+    return throwError(err, sessionContext.httpResponse) if err
+
+    if not operation?
+      logger.accessWarn({
+        sessionContext: sessionContext
+        message: 'Unrecognized command key'
+        rpc: rpc
+      })
+
+    if fault
+      for k, v of operation.retries
+        sessionContext.retries[k] = v
+
+      recordFault(sessionContext, fault, operation.provisions, operation.channels)
+
+    res = soap.response({
+      id : rpc.id,
+      acsResponse : acsResponse,
+      cwmpVersion : sessionContext.cwmpVersion
+    })
+    writeResponse(sessionContext, res)
+  )
+
+
+# Append providions and remove duplicates
+appendProvisions = (original, toAppend) ->
+  modified = false
+  stringified = new WeakMap()
+  for p, i in original
+    stringified.set(p, JSON.stringify(p))
+
+  for p, i in toAppend by -1
+    s = JSON.stringify(p)
+    for pp, j in original by -1
+      ss = stringified.get(pp)
+      if s == ss
+        if (not p) or j >= original.length - (toAppend.length - i)
+          p = null
+        else
+          original.splice(j, 1)
+          modified = true
+    if p
+      original.splice(original.length - (toAppend.length - i) + 1, 0, p)
+      stringified.set(p, s)
+      modified = true
+
+  return modified
+
+
+applyPresets = (sessionContext) ->
+  localCache.getPresets((err, presetsHash, presets) ->
+    return throwError(err, sessionContext.httpResponse) if err
+
+    # Filter presets based on existing faults
+    blackList = {}
+    whiteList = null
+    whiteListProvisions = null
+
+    RETRY_DELAY = config.get('RETRY_DELAY', sessionContext.deviceId)
+
+    if sessionContext.faults?
+      for channel, fault of sessionContext.faults
+        if fault.retryNow
+          retryTimestamp = 0
+        else
+          retryTimestamp = fault.timestamp + (RETRY_DELAY * Math.pow(2, sessionContext.retries[channel])) * 1000
+
+        if retryTimestamp <= sessionContext.timestamp
+          whiteList = channel
+          whiteListProvisions = fault.provisions
+          break
+
+        blackList[channel] = if fault.precondition then 1 else 2
+
+    sessionContext.deviceData.timestamps.revision = 1
+    sessionContext.deviceData.attributes.revision = 1
+
+    deviceEvents = {}
+    for p in sessionContext.deviceData.paths.find(['Events', '*'], false, true)
+      if sessionContext.timestamp <= sessionContext.deviceData.attributes.get(p)?.value?[1][0]
+        deviceEvents[p[1]] = true
+
+    parameters = {}
+    filteredPresets = []
+
+    for preset in presets
+      if whiteList?
+        continue if preset.channel != whiteList
+      else if blackList[preset.channel] == 1
+        continue
+
+      eventsMatch = true
+      for k, v of preset.events
+        if (!v) != (!deviceEvents[k.replace(' ', '_')])
+          eventsMatch = false
+          break
+
+      continue if not eventsMatch
+
+      if preset.schedule
+        continue if not preset.schedule.schedule
+        r = scheduling.cron(sessionContext.timestamp, preset.schedule.schedule)
+        if not (r[0] + preset.schedule.duration > sessionContext.timestamp)
+          continue
+
+      filteredPresets.push(preset)
+      for k of preset.precondition
+        sessionContext.channels[preset.channel] = 0
+        p = k.split(/([^a-zA-Z0-9\-\_\.].*)/, 1)[0]
+        parameters[p] = common.parsePath(p)
+
+    declarations = []
+    for k, v of parameters
+      declarations.push([v, 1, {value: 1}])
+
+    session.rpcRequest(sessionContext, declarations, (err, fault, id, acsRequest) ->
+      return throwError(err, sessionContext.httpResponse) if err
+
+      if fault
+        recordFault(sessionContext, fault)
+        session.clearProvisions(sessionContext)
+        return applyPresets(sessionContext)
+
+      if acsRequest
+        return sendAcsRequest(sessionContext, id, acsRequest)
+
+      session.clearProvisions(sessionContext)
+
+      for k, v of parameters
+        unpacked = device.unpack(sessionContext.deviceData, v)
+        if unpacked[0] and (vv = sessionContext.deviceData.attributes.get(unpacked[0]).value?[1])?
+          parameters[k] = vv[0]
+        else
+          delete parameters[k]
+
+      if whiteList?
+        session.addProvisions(sessionContext, whiteList, whiteListProvisions)
+
+      appendProvisionsToFaults = {}
+      for p in filteredPresets
+        if query.testFilter(parameters, p.precondition)
+          if blackList[p.channel] == 2
+            appendProvisionsToFaults[p.channel] =
+              (appendProvisionsToFaults[p.channel] or []).concat(p.provisions)
+          else
+            session.addProvisions(sessionContext, p.channel, p.provisions)
+
+      for channel, provisions of appendProvisionsToFaults
+        if appendProvisions(sessionContext.faults[channel].provisions, provisions)
+          sessionContext.faultsTouched ?= {}
+          sessionContext.faultsTouched[channel] = true
+
+      # Don't increment when processing individual channels (e.g. after fault)
+      sessionContext.presetCycles = (sessionContext.presetCycles or 0) + 1 if not whiteList?
+
+      if sessionContext.presetCycles > MAX_CYCLES
+        fault = {
+          code: 'preset_loop'
+          message: 'The presets are stuck in an endless configuration loop'
+          timestamp: sessionContext.timestamp
+        }
+        recordFault(sessionContext, fault)
+        # No need to save retryNow
+        for c of sessionContext.faults
+          delete sessionContext.faults[c].retryNow
+        session.clearProvisions(sessionContext)
+        return sendAcsRequest(sessionContext)
+
+      sessionContext.deviceData.timestamps.dirty = 0;
+      sessionContext.deviceData.attributes.dirty = 0;
+
+      session.rpcRequest(sessionContext, null, (err, fault, id, acsRequest) ->
+        return throwError(err, sessionContext.httpResponse) if err
+
+        if fault
+          recordFault(sessionContext, fault)
+          session.clearProvisions(sessionContext)
+          return applyPresets(sessionContext)
+
+        if not acsRequest
+          for channel, flags of sessionContext.channels
+            if channel of sessionContext.faults
+              delete sessionContext.faults[channel]
+              sessionContext.faultsTouched ?= {}
+              sessionContext.faultsTouched[channel] = true
+
+          if whiteList?
+            return applyPresets(sessionContext)
+
+          if sessionContext.deviceData.timestamps.dirty > 1 or
+              sessionContext.deviceData.attributes.dirty > 1
+            return applyPresets(sessionContext)
+
+        sendAcsRequest(sessionContext, id, acsRequest)
+      )
+    )
+  )
+
+
+nextRpc = (sessionContext) ->
+  session.rpcRequest(sessionContext, null, (err, fault, id, acsRequest) ->
+    return throwError(err, sessionContext.httpResponse) if err
+
+    if fault
+      recordFault(sessionContext, fault)
+      session.clearProvisions(sessionContext)
+      return nextRpc(sessionContext)
+
+    if acsRequest
+      return sendAcsRequest(sessionContext, id, acsRequest)
+
+    for channel, flags of sessionContext.channels
+      if flags and channel of sessionContext.faults
+        delete sessionContext.faults[channel]
+        sessionContext.faultsTouched ?= {}
+        sessionContext.faultsTouched[channel] = true
+
+      if channel.startsWith('task_')
+        taskId = channel.slice(5)
+        sessionContext.doneTasks ?= []
+        sessionContext.doneTasks.push(taskId)
+        for t, j in sessionContext.tasks
+          if t._id == taskId
+            sessionContext.tasks.splice(j, 1)
+            break
+
+    session.clearProvisions(sessionContext)
+
+    # Clear expired tasks
+    sessionContext.tasks = sessionContext.tasks.filter((task) ->
+      return true if not (task.expiry <= sessionContext.timestamp)
+
+      logger.accessInfo({
+        sessionContext: sessionContext
+        message: 'Task expired'
+        task: task
+      })
+      sessionContext.doneTasks ?= []
+      sessionContext.doneTasks.push(task._id)
+      channel = "task_#{task._id}"
+      if channel of sessionContext.faults
+        delete sessionContext.faults[channel]
+        sessionContext.faultsTouched ?= {}
+        sessionContext.faultsTouched[channel] = true
+
+      return false
+    )
+
+    task = sessionContext.tasks.find((task) -> !sessionContext.faults["task_#{task._id}"])
+
+    return applyPresets(sessionContext) if not task
+
+    switch task.name
+      when 'getParameterValues'
+        # Set channel in case params array is empty
+        sessionContext.channels["task_#{task._id}"] = 0
+        for p in task.parameterNames
+          session.addProvisions(sessionContext, "task_#{task._id}",
+            [['refresh', p]])
+      when 'setParameterValues'
+        # Set channel in case params array is empty
+        sessionContext.channels["task_#{task._id}"] = 0
+        for p in task.parameterValues
+          session.addProvisions(sessionContext, "task_#{task._id}",
+            [['value', p[0], p[1]]])
+      when 'refreshObject'
+        session.addProvisions(sessionContext, "task_#{task._id}",
+          [['refresh', task.objectName]])
+      when 'reboot'
+        session.addProvisions(sessionContext, "task_#{task._id}",
+          [['reboot']])
+      when 'factoryReset'
+        session.addProvisions(sessionContext, "task_#{task._id}",
+          [['reset']])
+      when 'download'
+        session.addProvisions(sessionContext, "task_#{task._id}",
+          [['download', task.fileType, task.fileName, task.targetFileName]])
+      when 'addObject'
+        alias = ("#{p[0]}:#{JSON.stringify(p[1])}" for p in task.parameterValues or []).join(',')
+        session.addProvisions(sessionContext, "task_#{task._id}",
+          [['instances', "#{task.objectName}.[#{alias}]", "+1"]]);
+      when 'deleteObject'
+        session.addProvisions(sessionContext, "task_#{task._id}",
+          [['instances', task.objectName, 0]]);
+      else
+        return throwError(new Error('Task name not recognized'), sessionContext.httpResponse)
+
+    return nextRpc(sessionContext)
+  )
+
+
+endSession = (sessionContext, callback) ->
+  saveCache = sessionContext.cacheUntil?
+
+  counter = 3
+
+  counter += 2
+  db.saveDevice(sessionContext.deviceId, sessionContext.deviceData, sessionContext.new, sessionContext.timestamp, (err) ->
+    if err
+      callback(err) if counter & 1
+      return counter = 0
+
+    return callback(null, sessionContext.new) if (counter -= 2) == 1
+  )
+
+  for k of sessionContext.operationsTouched
+    counter += 2
+    saveCache = true
+    if sessionContext.operations[k]?
+      db.saveOperation(sessionContext.deviceId, k, sessionContext.operations[k], (err) ->
+        if err
+          callback(err) if counter & 1
+          return counter = 0
+
+        return callback(null, sessionContext.new) if (counter -= 2) == 1
+      )
+    else
+      db.deleteOperation(sessionContext.deviceId, k, (err) ->
+        if err
+          callback(err) if counter & 1
+          return counter = 0
+
+        return callback(null, sessionContext.new) if (counter -= 2) == 1
+      )
+
+  if sessionContext.doneTasks?.length
+    counter += 2
+    saveCache = true
+    db.clearTasks(sessionContext.deviceId, sessionContext.doneTasks, (err) ->
+      if err
+        callback(err) if counter & 1
+        return counter = 0
+
+      return callback(null, sessionContext.new) if (counter -= 2) == 1
+    )
+
+  for k of sessionContext.faultsTouched
+    counter += 2
+    saveCache = true
+    if sessionContext.faults[k]
+      sessionContext.faults[k].retries = sessionContext.retries[k]
+      db.saveFault(sessionContext.deviceId, k, sessionContext.faults[k], (err) ->
+        if err
+          callback(err) if counter & 1
+          return counter = 0
+
+        return callback(null, sessionContext.new) if (counter -= 2) == 1
+      )
+    else
+      db.deleteFault(sessionContext.deviceId, k, (err) ->
+        if err
+          callback(err) if counter & 1
+          return counter = 0
+
+        return callback(null, sessionContext.new) if (counter -= 2) == 1
+      )
+
+  if saveCache
+    counter += 2
+    cacheDueTasksAndFaultsAndOperations(sessionContext.deviceId, sessionContext.tasks, sessionContext.faults, sessionContext.operations, sessionContext.cacheUntil, (err) ->
+      if err
+        callback(err) if counter & 1
+        return counter = 0
+
+      return callback(null, sessionContext.new) if (counter -= 2) == 1
+    )
+
+  callback(null, sessionContext.new) if (counter -= 2) == 1
+
+
+sendAcsRequest = (sessionContext, id, acsRequest) ->
+  if not acsRequest?
+    return writeResponse(sessionContext, soap.response(null), true)
+
+  if acsRequest.name is 'Download'
+    if not acsRequest.url?
+      FS_PORT = config.get('FS_PORT')
+      FS_HOSTNAME = config.get('FS_HOSTNAME')
+      FS_SSL = config.get('FS_SSL')
+      acsRequest.url = if FS_SSL then 'https://' else 'http://'
+      acsRequest.url += FS_HOSTNAME
+      acsRequest.url += ":#{FS_PORT}" if FS_PORT != 80
+      acsRequest.url += "/#{encodeURI(acsRequest.fileName)}"
+
+    if not acsRequest.fileSize?
+      return localCache.getFiles((err, hash, files) ->
+        return throwError(err, sessionContext.httpResponse) if err
+
+        if acsRequest.fileName of files
+          acsRequest.fileSize = files[acsRequest.fileName].length
+        else
+          acsRequest.fileSize = 0
+
+        return sendAcsRequest(sessionContext, id, acsRequest)
+      )
+
+  rpc = {
+    id : id,
+    acsRequest : acsRequest,
+    cwmpVersion : sessionContext.cwmpVersion
+  }
+
+  logger.accessInfo({
+    sessionContext: sessionContext
+    message: 'ACS request'
+    rpc: rpc
+  })
+
+  res = soap.response(rpc)
+
+  writeResponse(sessionContext, res)
+
+
+getSession = (connection, sessionId, callback) ->
+  sessionContext = currentSessions.get(connection)
+
+  if sessionContext
+    currentSessions.delete(connection)
+    return callback(null, sessionContext)
+
+  return callback() if not sessionId
+
+  setTimeout(() ->
+    cache.pop("session_#{sessionId}", (err, sessionContextString) ->
+      return callback(err) if err or not sessionContextString
+      session.deserialize(sessionContextString, callback)
+    )
+  , 100)
+
+
+currentSessions = new WeakMap()
+
+# When socket closes, store active sessions in cache
+onConnection = (socket) ->
+  # The property remoteAddress may be undefined after the connection is
+  # closed, unless we read it at least once (caching?)
+  socket.remoteAddress
+
+  socket.on('close', () ->
+    sessionContext = currentSessions.get(socket)
+    return if not sessionContext
+    currentSessions.delete(socket)
+    now = Date.now()
+
+    lastActivity = sessionContext.lastActivity
+    timeoutMsg = logger.flatten({
+      sessionContext: sessionContext
+      message: 'Session timeout'
+      sessionTimestamp: sessionContext.timestamp
+    })
+
+    timeout = (sessionContext.lastActivity + sessionContext.timeout * 1000) - now
+    if timeout <= 0
+      return logger.accessError(timeoutMsg)
+
+    setTimeout(() ->
+      cache.get("session_#{sessionContext.sessionId}", (err, sessionContextString) ->
+        return throwError(err) if err
+        return if not sessionContextString
+        session.deserialize(sessionContextString, (err, sessionContext) ->
+          return throwError(err) if err
+          if sessionContext.lastActivity == lastActivity
+            logger.accessError(timeoutMsg)
+        )
+      )
+    , timeout + 1000).unref()
+
+    session.serialize(sessionContext, (err, sessionContextString) ->
+      return throwError(err) if err
+      cache.set("session_#{sessionContext.sessionId}", sessionContextString, Math.ceil(timeout / 1000) + 3, (err) ->
+        return throwError(err) if err
+      )
+    )
+  )
+
+setInterval(() ->
+  if stats.droppedRequests
+    logger.warn({
+      message: 'Worker overloaded'
+      droppedRequests: stats.droppedRequests
+      totalRequests: stats.totalRequests
+      initiatedSessions: stats.initiatedSessions
+      pid: process.pid
+    })
+
+  stats.totalRequests = 0
+  stats.droppedRequests = 0
+  stats.initiatedSessions = 0
+, 10000).unref()
+
+
+getDueTasksAndFaultsAndOperations = (deviceId, timestamp, callback) ->
+  cache.get("#{deviceId}_tasks_faults_operations", (err, res) ->
+    return callback(err) if err
+
+    if res
+      res = JSON.parse(res)
+      return callback(null, res.tasks or [], res.faults or {}, res.operations or {})
+
+    faults = tasks = operations = cacheUntil = null
+
+    db.getFaults(deviceId, (err, _faults) ->
+      if err
+        callback?(err)
+        return callback = null
+
+      faults = _faults
+      if tasks? and operations?
+        return callback(null, tasks, faults, operations, cacheUntil)
+    )
+
+    db.getDueTasks(deviceId, timestamp, (err, dueTasks, nextTimestamp) ->
+      if err
+        callback?(err)
+        return callback = null
+
+      tasks = dueTasks
+      cacheUntil = nextTimestamp or 0
+
+      if faults? and operations?
+        return callback(null, tasks, faults, operations, cacheUntil)
+    )
+
+    db.getOperations(deviceId, (err, _operations) ->
+      if err
+        callback?(err)
+        return callback = null
+
+      operations = _operations
+
+      if faults? and tasks?
+        return callback(null, tasks, faults, operations, cacheUntil)
+    )
+
+    if faults? and tasks? and operations?
+      return callback(null, tasks, faults, operations, cacheUntil)
+  )
+
+
+cacheDueTasksAndFaultsAndOperations = (deviceId, tasks, faults, operations, cacheUntil, callback) ->
+  v = {}
+  v.tasks = tasks if tasks.length
+  v.faults = faults if Object.keys(faults).length
+  v.operations = operations if Object.keys(operations).length
+
+  if cacheUntil
+    ttl = Math.trunc((Date.now() - cacheUntil) / 1000)
+  else
+    ttl = config.get('MAX_CACHE_TTL', deviceId)
+
+  cache.set("#{deviceId}_tasks_faults_operations", JSON.stringify(v), ttl, callback)
+
+
+processRequest = (sessionContext, rpc) ->
+  if rpc.cpeRequest?
+    if rpc.cpeRequest.name is 'Inform'
+      logger.accessInfo({
+        sessionContext: sessionContext
+        message: 'Inform'
+        rpc: rpc
+      })
+      inform(sessionContext, rpc)
+    else if rpc.cpeRequest.name is 'TransferComplete'
+      logger.accessInfo({
+        sessionContext: sessionContext
+        message: 'CPE request'
+        rpc: rpc
+      })
+      transferComplete(sessionContext, rpc)
+    else if rpc.cpeRequest.name is 'GetRPCMethods'
+      logger.accessInfo({
+        sessionContext: sessionContext
+        message: 'CPE request'
+        rpc: rpc
+      })
+      res = soap.response({
+        id : rpc.id
+        acsResponse : {name: 'GetRPCMethodsResponse', methodList: ['Inform', 'GetRPCMethods', 'TransferComplete']}
+        cwmpVersion : sessionContext.cwmpVersion
+      })
+      writeResponse(sessionContext, res)
+    else
+      return throwError(new Error('ACS method not supported'), sessionContext.httpResponse)
+  else if rpc.cpeResponse
+    session.rpcResponse(sessionContext, rpc.id, rpc.cpeResponse, (err) ->
+      return throwError(err, sessionContext.httpResponse) if err
+      nextRpc(sessionContext)
+    )
+  else if rpc.cpeFault
+    logger.accessWarn({
+      sessionContext: sessionContext
+      message: 'CPE fault'
+      rpc:rpc
+    })
+    session.rpcFault(sessionContext, rpc.id, rpc.cpeFault, (err, fault) ->
+      return throwError(err, sessionContext.httpResponse) if err
+
+      if fault
+        recordFault(sessionContext, fault)
+        session.clearProvisions(sessionContext)
+      nextRpc(sessionContext)
+    )
+  else # CPE sent empty response
+    session.timeoutOperations(sessionContext, (err, faults, operations) ->
+      return throwError(err, sessionContext.httpResponse) if err
+
+      for fault, i in faults
+        for k, v of operations[i].retries
+          sessionContext.retries[k] = v
+
+        recordFault(sessionContext, fault, operations[i].provisions, operations[i].channels)
+
+      nextRpc(sessionContext)
+    )
+
+
+listener = (httpRequest, httpResponse) ->
+  stats.totalRequests += 1
+  if httpRequest.method != 'POST'
+    httpResponse.writeHead 405, {'Allow': 'POST', 'Connection': 'close'}
+    httpResponse.end('405 Method Not Allowed')
+    return
+
+  sessionId = null
+  # Separation by comma is important as some devices don't comform to standard
+  COOKIE_REGEX = /\s*([a-zA-Z0-9\-_]+?)\s*=\s*"?([a-zA-Z0-9\-_]*?)"?\s*(,|;|$)/g
+  while match = COOKIE_REGEX.exec(httpRequest.headers.cookie)
+    sessionId = match[2] if match[1] == 'session'
+
+  # If overloaded, ask CPE to retry in 60 seconds
+  if not sessionId and stats.concurrentRequests > MAX_CONCURRENT_REQUESTS
+    httpResponse.writeHead(503, {'Retry-after': 60, 'Connection': 'close'})
+    httpResponse.end('503 Service Unavailable')
+    stats.droppedRequests += 1
+    return
+
+  if httpRequest.headers['content-encoding']?
+    switch httpRequest.headers['content-encoding']
+      when 'gzip'
+        stream = httpRequest.pipe(zlib.createGunzip())
+      when 'deflate'
+        stream = httpRequest.pipe(zlib.createInflate())
+      else
+        httpResponse.writeHead(415, {'Connection': 'close'})
+        httpResponse.end('415 Unsupported Media Type')
+        return
+  else
+    stream = httpRequest
+
+  stats.concurrentRequests += 1
+  httpRequest.on('aborted', () ->
+    stats.concurrentRequests -= 1
+    # In some cases event end can be emitted after aborted event
+    httpRequest.removeAllListeners('end')
+  )
+
+  chunks = []
+  bytes = 0
+
+  stream.on('data', (chunk) ->
+    chunks.push(chunk)
+    bytes += chunk.length
+  )
+
+  stream.on('end', () ->
+    body = new Buffer(bytes)
+    offset = 0
+    chunks.forEach((chunk) ->
+      chunk.copy(body, offset, 0, chunk.length)
+      offset += chunk.length
+    )
+
+    parsedRpc = (sessionContext, rpc, parseWarnings) ->
+      for w in parseWarnings
+        w.sessionContext = sessionContext
+        w.rpc = rpc;
+        logger.accessWarn(w)
+
+      if config.get('DEBUG', sessionContext.deviceId)
+        dump = "# REQUEST #{new Date(Date.now())}\n" + JSON.stringify(httpRequest.headers) + "\n#{body}\n\n"
+        require('fs').appendFile("./debug/#{sessionContext.deviceId}.dump", dump, (err) ->
+          return throwError(err) if err
+        )
+
+      processRequest(sessionContext, rpc)
+
+    getSession(httpRequest.connection, sessionId, (err, sessionContext) ->
+      return throwError(err, httpResponse) if err
+
+      if sessionContext
+        sessionContext.httpRequest = httpRequest
+        sessionContext.httpResponse = httpResponse
+
+        if sessionContext.sessionId != sessionId or
+            sessionContext.lastActivity + sessionContext.timeout * 1000 < Date.now()
+          logger.accessError({message: 'Invalid session', sessionContext: sessionContext})
+          httpResponse.writeHead(400, {'Connection': 'close'})
+          httpResponse.end('Invalid session')
+          stats.concurrentRequests -= 1
+          return
+      else if stats.concurrentRequests > MAX_CONCURRENT_REQUESTS
+        # Check again just in case device included old session ID from the previous session
+        httpResponse.writeHead(503, {'Retry-after': 60, 'Connection': 'close'})
+        httpResponse.end('503 Service Unavailable')
+        stats.droppedRequests += 1
+        stats.concurrentRequests -= 1
+        return
+
+      parseWarnings = []
+      try
+        rpc = soap.request(body, sessionContext?.cwmpVersion, parseWarnings)
+      catch err
+        logger.accessError({
+          message: 'XML parse error'
+          parseError: err.message.trim()
+          sessionContext: sessionContext or {httpRequest: httpRequest, httpResponse: httpResponse}
+        })
+        httpResponse.writeHead(400, {'Connection': 'close'})
+        httpResponse.end(err.message)
+        stats.concurrentRequests -= 1
+        return
+
+      if sessionContext
+        if rpc.cpeRequest?.name is 'Inform' or
+            (!sessionContext.rpcRequest ^ !(rpc.cpeResponse or rpc.cpeFault))
+          logger.accessError({
+            message: 'Bad session state'
+            sessionContext: sessionContext
+          })
+          httpResponse.writeHead(400, {'Connection': 'close'})
+          httpResponse.end('Bad session state')
+          stats.concurrentRequests -= 1
+          return
+
+        return parsedRpc(sessionContext, rpc, parseWarnings)
+
+      if rpc.cpeRequest?.name isnt 'Inform'
+        logger.accessError({
+          message: 'Invalid session'
+          sessionContext: sessionContext or {httpRequest: httpRequest, httpResponse: httpResponse}
+        })
+        httpResponse.writeHead(400, {'Connection': 'close'})
+        httpResponse.end('Invalid session')
+        stats.concurrentRequests -= 1
+        return
+
+      stats.initiatedSessions += 1
+      deviceId = common.generateDeviceId(rpc.cpeRequest.deviceId)
+      return session.init(deviceId, rpc.cwmpVersion, rpc.sessionTimeout ? config.get('SESSION_TIMEOUT', deviceId), (err, sessionContext) ->
+        return throwError(err, httpResponse) if err
+
+        sessionContext.httpRequest = httpRequest
+        sessionContext.httpResponse = httpResponse
+        sessionContext.sessionId = crypto.randomBytes(8).toString('hex')
+        httpRequest.connection.setTimeout(sessionContext.timeout * 1000)
+
+        getDueTasksAndFaultsAndOperations(deviceId, sessionContext.timestamp, (err, dueTasks, faults, operations, cacheUntil) ->
+          return throwError(err, httpResponse) if err
+
+          sessionContext.tasks = dueTasks
+          sessionContext.faults = faults
+          sessionContext.retries = {}
+          for k, v of faults
+            sessionContext.retries[k] = v.retries
+          sessionContext.operations = operations
+          sessionContext.cacheUntil = cacheUntil
+
+          # Delete expired faults
+          for k, v of sessionContext.faults
+            if v.expiry >= sessionContext.timestamp
+              delete sessionContext.faults[k]
+              sessionContext.faultsTouched ?= {}
+              sessionContext.faultsTouched[k] = true
+
+          parsedRpc(sessionContext, rpc, parseWarnings)
+        )
+      )
+    )
+  )
+
+
+exports.listener = listener
+exports.onConnection = onConnection

lib/db.coffee

@@ -0,0 +1,652 @@
+###
+# Copyright 2013-2017  Zaid Abdulla
+#
+# This file is part of GenieACS.
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
+config = require './config'
+mongodb = require 'mongodb'
+common = require './common'
+
+tasksCollection = null
+devicesCollection = null
+presetsCollection = null
+objectsCollection = null
+provisionsCollection = null
+virtualParametersCollection = null
+faultsCollection = null
+filesCollection = null
+operationsCollection = null
+
+
+connect = (callback) ->
+  callbackCounter = 9
+  mongodb.MongoClient.connect(config.get('MONGODB_CONNECTION_URL'), (err, db) ->
+    return callback(err) if err
+    exports.mongoDb = db
+    db.collection('tasks', (err, collection) ->
+      exports.tasksCollection = tasksCollection = collection
+      collection?.ensureIndex({device: 1, timestamp: 1}, (err) ->
+      )
+
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
+    )
+
+    db.collection('devices', (err, collection) ->
+      exports.devicesCollection  = devicesCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
+    )
+
+    db.collection('presets', (err, collection) ->
+      exports.presetsCollection = presetsCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
+    )
+
+    db.collection('objects', (err, collection) ->
+      exports.objectsCollection = objectsCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
+    )
+
+    db.collection('fs.files', (err, collection) ->
+      exports.filesCollection = filesCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
+    )
+
+    db.collection('provisions', (err, collection) ->
+      exports.provisionsCollection = provisionsCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
+    )
+
+    db.collection('virtualParameters', (err, collection) ->
+      exports.virtualParametersCollection = virtualParametersCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
+    )
+
+    db.collection('faults', (err, collection) ->
+      exports.faultsCollection = faultsCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
+    )
+
+    db.collection('operations', (err, collection) ->
+      exports.operationsCollection = operationsCollection = collection
+      if --callbackCounter == 0 or err
+        callbackCounter = 0
+        return callback(err)
+    )
+
+  )
+
+
+disconnect = () ->
+  exports.mongoDb?.close()
+
+
+# Optimize projection by removing overlaps
+# This can modify the object
+optimizeProjection = (obj) ->
+  if obj['']
+    return {'': obj['']}
+
+  keys = Object.keys(obj).sort()
+  return obj if keys.length <= 1
+  i = 1
+  while i < keys.length
+    a = keys[i-1]
+    b = keys[i]
+    if b.startsWith(a)
+      if b.charAt(a.length) == '.' or b.charAt(a.length - 1) == '.'
+        delete obj[b]
+        keys.splice(i--, 1)
+    ++ i
+  return obj
+
+
+fetchDevice = (id, timestamp, patterns, callback) ->
+  MAX_DEPTH = config.get('MAX_DEPTH', id)
+
+  if not patterns?.length
+    patterns = [[[], (1 << MAX_DEPTH) - 1]]
+
+  projection = {}
+  projectionTree = {}
+  func = (path, pats, projTree) ->
+    children = {}
+    for pat in pats
+      fragment = pat[0][path.length] or '*'
+
+      if fragment == '*'
+        projection[path.join('.')] = 1 if pat[1] << path.length
+        return
+
+      projTree[fragment] ?= {}
+
+      if pat[1] & (1 << path.length)
+        projection[path.concat('_timestamp').join('.')] = 1
+        projection[path.concat('_object').join('.')] = 1
+        projection[path.concat('_instance').join('.')] = 1
+        projection[path.concat([fragment, '_timestamp']).join('.')] = 1
+        projection[path.concat([fragment, '_value']).join('.')] = 1
+        projection[path.concat([fragment, '_type']).join('.')] = 1
+        projection[path.concat([fragment, '_object']).join('.')] = 1
+        projection[path.concat([fragment, '_instance']).join('.')] = 1
+        projection[path.concat([fragment, '_writable']).join('.')] = 1
+        projection[path.concat([fragment, '_orig']).join('.')] = 1
+
+      if pat[1] >> (path.length + 1)
+        children[fragment] ?= []
+        children[fragment].push(pat)
+
+    for k, v of children
+      func(path.concat(k), v, projTree[k])
+
+  func([], patterns, projectionTree)
+
+  delete projectionTree['DeviceID']
+  delete projectionTree['Events']
+  delete projectionTree['Tags']
+
+  res = []
+  loaded = []
+
+  projection = optimizeProjection(projection)
+
+  for k, v of projection
+    if k == '' or k == 'Events' or k.startsWith('Events.')
+      if k == '' or k == 'Events' or k == 'Events._writable'
+        res.push([['Events'], timestamp,
+          {object: [timestamp, 1], writable: [timestamp, 0]}])
+        loaded.push([['Events'], 1]) if k == 'Events._writable'
+      if k == 'Events'
+        projection['_lastInform'] = 1
+        projection['_lastBoot'] = 1
+        projection['_lastBootstrap'] = 1
+        projection['_registered'] = 1
+        loaded.push([['Events'], ((1 << MAX_DEPTH) - 1)])
+      else if k == 'Events.Inform._writable' or k == 'Events.Inform'
+        projection['_lastInform'] = 1
+        loaded.push([['Events', 'Inform'], 1 ^ ((1 << MAX_DEPTH) - 1)])
+      else if k == 'Events.1_BOOT._writable' or k == 'Events.1_BOOT'
+        projection['_lastBoot'] = 1
+        loaded.push([['Events', '1_BOOT'], 1 ^ ((1 << MAX_DEPTH) - 1)])
+      else if k == 'Events.0_BOOTSTRAP._writable' or k == 'Events.0_BOOTSTRAP'
+        projection['_lastBootstrap'] = 1
+        loaded.push([['Events', '0_BOOTSTRAP'], 1 ^ ((1 << MAX_DEPTH) - 1)])
+      else if k == 'Events.Registered._writable' or k == 'Events.Registered'
+        projection['_registered'] = 1
+        loaded.push([['Events', 'Registered'], 1 ^ ((1 << MAX_DEPTH) - 1)])
+      else if k.endsWith('._writable') and k != 'Events._writable'
+        loaded.push([k.split('.').slice(0, 2), 1 ^ ((1 << MAX_DEPTH) - 1)])
+      delete projection[k] if k != ''
+
+    if k == '' or k == 'DeviceID' or k.startsWith('DeviceID.')
+      if k == '' or k == 'DeviceID' or k == 'DeviceID._writable'
+        res.push([['DeviceID'], timestamp,
+          {object: [timestamp, 1], writable: [timestamp, 0]}])
+        loaded.push([['DeviceID'], 1]) if k == 'DeviceID._writable'
+      if k == 'DeviceID'
+        projection['_id'] = 1
+        projection['_deviceId'] = 1
+        loaded.push([['DeviceID'], ((1 << MAX_DEPTH) - 1)])
+      else if k == 'DeviceID.ID._writable' or k == 'DeviceID.ID'
+        projection['_id'] = 1
+        loaded.push([['DeviceID', 'ID'], 1 ^ ((1 << MAX_DEPTH) - 1)])
+      else if k == 'DeviceID.Manufacturer._writable' or k == 'DeviceID.DeManufacturer'
+        projection['_deviceId._Manufacturer'] = 1
+        loaded.push([['DeviceID', 'Manufacturer'], 1 ^ ((1 << MAX_DEPTH) - 1)])
+      else if k == 'DeviceID.ProductClass._writable' or k == 'DeviceID.ProductClass'
+        projection['_deviceId._ProductClass'] = 1
+        loaded.push([['DeviceID', 'ProductClass'], 1 ^ ((1 << MAX_DEPTH) - 1)])
+      else if k == 'DeviceID.OUI._writable' or k == 'DeviceID.OUI'
+        projection['_deviceId._OUI'] = 1
+        loaded.push([['DeviceID', 'ProductClass'], 1 ^ ((1 << MAX_DEPTH) - 1)])
+      else if k == 'DeviceID.SerialNumber._writable' or k == 'DeviceID.SerialNumber'
+        projection['_deviceId._SerialNumber'] = 1
+        loaded.push([['DeviceID', 'SerialNumber'], 1 ^ ((1 << MAX_DEPTH) - 1)])
+      else if k.endsWith('._writable') and k != 'DeviceID._writable'
+        loaded.push([k.split('.').slice(0, 2), 1 ^ ((1 << MAX_DEPTH) - 1)])
+      delete projection[k] if k != ''
+
+    if k == 'Tags' or k.startsWith('Tags.')
+      if not projection['_tags']
+        projection['_tags'] = 1
+        loaded.push([['Tags'], ((1 << MAX_DEPTH) - 1)])
+      delete projection[k]
+
+  if projection['']
+    proj = {}
+  else if Object.keys(projection).length == 0
+    proj = {'_id': 1}
+  else
+    proj = projection
+
+  devicesCollection.findOne({'_id' : id}, proj, (err, device) ->
+    return callback(err) if err or not device?
+
+    storeParams = (obj, path, timestamp, descendantsFetched, projTree) ->
+      if descendantsFetched
+        thisFetched = true
+      else
+        if projection[path.join('.')]
+          descendantsFetched = true
+          if path.length and projection[path.slice(0, -1).concat("_timestamp").join('.')]
+            thisFetched = true
+            loaded.push([path, ((1 << (path.length - 1)) - 1) ^ ((1 << MAX_DEPTH) - 1)])
+          else
+            loaded.push([path, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
+        else if projection[path.concat('_writable').join('.')]
+          loaded.push([path, 1 << (path.length - 1)])
+          thisFetched = true
+
+      if obj['_timestamp']?
+        obj['_timestamp'] = +obj['_timestamp']
+
+      # For compatibility with v1.0 database
+      if obj['_instance'] and not obj['_object']?
+        obj['_object'] = true
+
+      if thisFetched
+        attrs = {}
+
+        t = obj['_timestamp'] or 1
+        t = timestamp if timestamp > t
+
+        if obj['_value']?
+          attrs.value = [obj['_timestamp'] or 1, [obj['_value'], obj['_type']]]
+          attrs.value[1][0] = +attrs.value[1][0] if obj['_type'] == 'xsd:dateTime'
+          obj['_object'] = false
+
+        if obj['_writable']?
+          attrs.writable = [timestamp or 1, if obj['_writable'] then 1 else 0]
+
+        if obj['_object']?
+          attrs.object = [t, if obj['_object'] then 1 else 0]
+
+        res.push([path, t, attrs])
+
+      for k, v of obj
+        if not k.startsWith('_')
+          kk = k
+          obj['_object'] = true
+          storeParams(v, path.concat(k), obj['_timestamp'], descendantsFetched, projTree?[k])
+          delete projTree[kk] if projTree
+
+      if not descendantsFetched
+        for k, v of projTree
+          p = path.concat(k)
+          loaded.push([p, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
+          if (obj['_object'] or path.length == 0) and obj['_timestamp']
+            res.push([p, obj['_timestamp']])
+      else if (obj['_object'] or path.length == 0) and obj['_timestamp']
+          res.push([path.concat('*'), obj['_timestamp']])
+
+    for k, v of device
+      switch k
+        when '_lastInform'
+          res.push([['Events', 'Inform'], +v,
+            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
+          delete device[k]
+        when '_lastBoot'
+          res.push([['Events', '1_BOOT'], +v,
+            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
+          delete device[k]
+        when '_lastBootstrap'
+          res.push([['Events', '0_BOOTSTRAP'], +v,
+            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
+          delete device[k]
+        when '_registered'
+          # Use current timestamp for registered event attribute timestamps
+          res.push([['Events', 'Registered'], timestamp,
+            {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [+v, 'xsd:dateTime']]}])
+          delete device[k]
+        when '_id'
+          if projection[''] or projection['_id']
+            res.push([['DeviceID', 'ID'], timestamp,
+              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v, 'xsd:string']]}])
+          delete device[k]
+        when '_tags'
+          if v.length
+            res.push([['Tags'], timestamp,
+              {object: [timestamp, 1], writable: [timestamp, 0]}])
+          for t in v
+            t = t.replace(/[^a-zA-Z0-9\-]+/g, '_')
+            res.push([['Tags', t], timestamp,
+              {object: [timestamp, 0], writable: [timestamp, 1], value: [timestamp, [true, 'xsd:boolean']]}])
+          delete device[k]
+        when '_deviceId'
+          if v['_Manufacturer']?
+            res.push([['DeviceID', 'Manufacturer'], timestamp,
+              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_Manufacturer'], 'xsd:string']]}])
+          if v['_OUI']?
+            res.push([['DeviceID', 'OUI'], timestamp,
+              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_OUI'], 'xsd:string']]}])
+          if v['_ProductClass']?
+            res.push([['DeviceID', 'ProductClass'], timestamp,
+              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_ProductClass'], 'xsd:string']]}])
+          if v['_SerialNumber']?
+            res.push([['DeviceID', 'SerialNumber'], timestamp,
+              {object: [timestamp, 0], writable: [timestamp, 0], value: [timestamp, [v['_SerialNumber'], 'xsd:string']]}])
+          delete device[k]
+
+    storeParams(device, [], 0, false, projectionTree)
+
+    return callback(null, res, loaded)
+  )
+
+
+saveDevice = (deviceId, deviceData, isNew, sessionTimestamp, callback) ->
+  update = {'$set' : {}, '$unset' : {}, '$addToSet' : {}, '$pull' : {}}
+
+  iter = deviceData.timestamps.diff()
+  while diff = iter.next().value
+    continue if diff[0].wildcard != (1 << (diff[0].length - 1))
+    continue if diff[0][0] in ['Events', 'DeviceID', 'Tags']
+
+    # Param timestamps may be greater than session timestamp to track revisions
+    if diff[2] > sessionTimestamp
+      diff[2] = sessionTimestamp
+
+    if not diff[2]? and diff[1]?
+      update['$unset'][diff[0].slice(0, -1).concat('_timestamp').join('.')] = 1
+    else if diff[2] != diff[1]
+      parent = deviceData.paths.get(diff[0].slice(0, -1))
+      if parent and (parent.length == 0 or deviceData.attributes.has(parent))
+        update['$set'][diff[0].slice(0, -1).concat('_timestamp').join('.')] = new Date(diff[2])
+
+  iter = deviceData.attributes.diff()
+  while diff = iter.next().value
+    continue if diff[1] == diff[2]
+    path = diff[0]
+    switch path[0]
+      when 'Events'
+        if diff[0].length == 2 and diff[2]?.value?[1][0] != diff[1]?.value?[1][0]
+          if not diff[2]
+            switch path[1]
+              when 'Inform'
+                update['$unset']['_lastInform'] = 1
+              when '1_BOOT'
+                update['$unset']['_lastBoot'] = 1
+              when '0_BOOTSTRAP'
+                update['$unset']['_lastBootstrap'] = 1
+              when 'Registered'
+                update['$unset']['_registered'] = 1
+          else
+            t = new Date(diff[2].value[1][0])
+            switch path[1]
+              when 'Inform'
+                update['$set']['_lastInform'] = t
+              when '1_BOOT'
+                update['$set']['_lastBoot'] = t
+              when '0_BOOTSTRAP'
+                update['$set']['_lastBootstrap'] = t
+              when 'Registered'
+                update['$set']['_registered'] = t
+      when 'DeviceID'
+        if diff[2].value?[1]?[0] != diff[1]?.value?[1]?[0]
+          v = diff[2].value[1][0]
+          switch path[1]
+            when 'ID'
+              update['$set']['_id'] = v
+            when 'Manufacturer'
+              update['$set']['_deviceId._Manufacturer'] = v
+            when 'OUI'
+              update['$set']['_deviceId._OUI'] = v
+            when 'ProductClass'
+              update['$set']['_deviceId._ProductClass'] = v
+            when 'SerialNumber'
+              update['$set']['_deviceId._SerialNumber'] = v
+      when 'Tags'
+        if diff[2]?.value?[1]?[0] != diff[1]?.value?[1]?[0]
+          v = diff[2]?.value?[1][0]
+          if v?
+            update['$addToSet']['_tags'] ?= {'$each' : []}
+            update['$addToSet']['_tags']['$each'].push(diff[0][1])
+          else
+            update['$pull']['_tags'] ?= {'$in' : []}
+            update['$pull']['_tags']['$in'].push(diff[0][1])
+      else
+        if not diff[2]
+          update['$unset'][diff[0].join('.')] = 1
+          continue
+
+        for attrName of diff[2]
+          # Param timestamps may be greater than session timestamp to track revisions
+          if diff[2][attrName][0] > sessionTimestamp
+            diff[2][attrName][0] = sessionTimestamp
+
+          if diff[2][attrName][1]?
+            switch attrName
+              when 'value'
+                if diff[2].value[1][0] != diff[1]?.value?[1]?[0]
+                  if diff[2].value[1][1] == 'xsd:dateTime' and Number.isInteger(diff[2].value[1][0])
+                    update['$set'][path.concat('_value').join('.')] = new Date(diff[2].value[1][0])
+                  else
+                    update['$set'][path.concat('_value').join('.')] = diff[2].value[1][0]
+                if diff[2].value[1][1] != diff[1]?.value?[1]?[1]
+                  update['$set'][path.concat('_type').join('.')] = diff[2].value[1][1]
+                if diff[2].value[0] != diff[1]?.value?[0]
+                  update['$set'][path.concat('_timestamp').join('.')] = new Date(diff[2].value[0])
+              when 'object'
+                if not diff[1]?.object or diff[2].object[1] != diff[1].object?[1]
+                  update['$set'][path.concat('_object').join('.')] = !!diff[2].object[1]
+              when 'writable'
+                if not diff[1]?.writable or diff[2].writable[1] != diff[1].writable?[1]
+                  update['$set'][path.concat('_writable').join('.')] = !!diff[2].writable[1]
+
+        if diff[1]
+          for attrName of diff[1]
+            if diff[1][attrName][1]? and not diff[2]?[attrName]?[1]?
+              update['$unset'][path.concat("_#{attrName}").join('.')] = 1
+              if attrName is 'value'
+                update['$unset'][path.concat('_type').join('.')] = 1
+                update['$unset'][path.concat('_timestamp').join('.')] = 1
+
+  update['$unset'] = optimizeProjection(update['$unset'])
+
+  # Remove overlap possibly caused by parameters changing from objects
+  # to regular parameters or vice versa. Reason being that _timestamp
+  # represents two different things depending on whether the parameter
+  # is an object or not.
+  for k of update['$unset']
+    delete update['$unset'][k] if update['$set'][k]?
+
+  # Remove empty keys
+  for k of update
+    if k == '$addToSet'
+      for kk of update[k]
+        delete update[k][kk] if update[k][kk]['$each'].length == 0
+    else if k == '$pull'
+      for kk of update[k]
+        delete update[k][kk] if update[k][kk]['$in'].length == 0
+    delete update[k] if Object.keys(update[k]).length == 0
+
+  return callback() if Object.keys(update).length == 0
+
+  if update['$addToSet'] and update['$pull']
+    # Mongo doesn't allow $addToSet and $pull at the same time
+    update2 = {'$pull': update['$pull']}
+    delete update['$pull']
+
+  devicesCollection.update({'_id' : deviceId}, update, {upsert: isNew}, (err, result) ->
+    if not err and result.result.n != 1
+      return callback(new Error("Device #{deviceId} not found in database"))
+
+    if update2
+      return devicesCollection.update({'_id': deviceId}, update2, callback)
+
+    return callback(err)
+  )
+
+
+getFaults = (deviceId, callback) ->
+  faultsCollection.find({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}).toArray((err, res) ->
+    return callback(err) if err
+
+    faults = {}
+    for r in res
+      channel = r._id.slice(deviceId.length + 1)
+      delete r._id
+      delete r.channel
+      delete r.device
+      r.timestamp = +r.timestamp
+      r.provisions = JSON.parse(r.provisions)
+      faults[channel] = r
+
+    callback(err, faults)
+  )
+
+
+saveFault = (deviceId, channel, fault, callback) ->
+  fault = Object.assign({}, fault)
+  fault._id = "#{deviceId}:#{channel}"
+  fault.device = deviceId
+  fault.channel = channel
+  fault.timestamp = new Date(fault.timestamp)
+  fault.provisions = JSON.stringify(fault.provisions)
+  faultsCollection.save(fault, callback)
+
+
+deleteFault = (deviceId, channel, callback) ->
+  faultsCollection.remove({_id: "#{deviceId}:#{channel}"}, callback)
+
+
+getDueTasks = (deviceId, timestamp, callback) ->
+  cur = tasksCollection.find({'device' : deviceId}).sort(['timestamp'])
+
+  tasks = []
+
+  cur.nextObject(f = (err, task) ->
+    return callback(err) if err
+
+    if not task?
+      return callback(null, tasks, null)
+
+    task.timestamp = +task.timestamp if task.timestamp?
+    task.expiry = +task.expiry if task.expiry?
+
+    if task.timestamp >= timestamp
+      return callback(null, tasks, +task.timestamp)
+
+    task._id = String(task._id)
+    tasks.push(task)
+
+    # For API compatibility
+    if task.name is 'download' and task.file?
+      if mongodb.ObjectID.isValid(task.file)
+        q = {_id: {'$in' : [task.file, new mongodb.ObjectID(task.file)]}}
+      else
+        q = {_id: task.file}
+
+      filesCollection.find(q).toArray((err, res) ->
+        return callback(err) if err
+        if res[0]?
+          task.fileType ?= res[0].metadata.fileType
+          task.fileName ?= res[0].filename or res[0]._id.toString()
+        cur.nextObject(f)
+      )
+    else
+      cur.nextObject(f)
+  )
+
+
+clearTasks = (deviceId, taskIds, callback) ->
+  tasksCollection.remove({'_id' : {'$in' : (new mongodb.ObjectID(id) for id in taskIds)}}, callback)
+
+
+getOperations = (deviceId, callback) ->
+  operationsCollection.find({'_id' : {'$regex' : "^#{common.escapeRegExp(deviceId)}\\:"}}).toArray((err, res) ->
+    return callback(err) if err
+
+    operations = {}
+    for r in res
+      commandKey = r._id.slice(deviceId.length + 1)
+      delete r._id
+      r.timestamp = +r.timestamp
+      r.args = JSON.parse(r.args) if r.args
+      r.provisions = JSON.parse(r.provisions)
+      r.retries = JSON.parse(r.retries)
+      operations[commandKey] = r
+
+    callback(err, operations)
+  )
+
+
+saveOperation = (deviceId, commandKey, operation, callback) ->
+  operation = Object.assign({}, operation)
+  operation._id = "#{deviceId}:#{commandKey}"
+  operation.timestamp = new Date(operation.timestamp)
+  operation.provisions = JSON.stringify(operation.provisions)
+  operation.retries = JSON.stringify(operation.retries)
+  operation.args = JSON.stringify(operation.args)
+  operationsCollection.save(operation, callback)
+
+
+deleteOperation = (deviceId, commandKey, callback) ->
+  operationsCollection.remove({_id: "#{deviceId}:#{commandKey}"}, callback)
+
+
+getPresets = (callback) ->
+  presetsCollection.find().toArray(callback)
+
+
+getObjects = (callback) ->
+  objectsCollection.find().toArray(callback)
+
+
+getProvisions = (callback) ->
+  provisionsCollection.find().toArray(callback)
+
+
+getVirtualParameters = (callback) ->
+  virtualParametersCollection.find().toArray(callback)
+
+
+getFiles = (callback) ->
+  filesCollection.find().toArray(callback)
+
+
+exports.connect = connect
+exports.disconnect = disconnect
+exports.fetchDevice = fetchDevice
+exports.saveDevice = saveDevice
+exports.getFaults = getFaults
+exports.saveFault = saveFault
+exports.deleteFault = deleteFault
+exports.clearTasks = clearTasks
+exports.saveOperation = saveOperation
+exports.deleteOperation = deleteOperation
+exports.getPresets = getPresets
+exports.getObjects = getObjects
+exports.getProvisions = getProvisions
+exports.getVirtualParameters = getVirtualParameters
+exports.getFiles = getFiles
+exports.getDueTasks = getDueTasks
+exports.getOperations = getOperations

lib/db.ts

@@ -1,822 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { MongoClient, ObjectID, Collection, Db } from "mongodb";
-import { get } from "./config";
-import { decodeTag, encodeTag, escapeRegExp } from "./common";
-import { parse } from "./common/expression-parser";
-import {
-  DeviceData,
-  Attributes,
-  SessionFault,
-  Task,
-  Operation,
-  Expression,
-} from "./types";
-import Path from "./common/path";
-
-export let tasksCollection: Collection,
-  devicesCollection: Collection,
-  presetsCollection: Collection,
-  objectsCollection: Collection,
-  provisionsCollection: Collection,
-  virtualParametersCollection: Collection,
-  faultsCollection: Collection,
-  filesCollection: Collection,
-  operationsCollection: Collection,
-  permissionsCollection: Collection,
-  usersCollection: Collection,
-  configCollection: Collection;
-
-let clientPromise: Promise<MongoClient>;
-
-const INVALID_PATH_SUFFIX = "__invalid";
-
-function compareAccessLists(list1: string[], list2: string[]): boolean {
-  if (list1.length !== list2.length) return false;
-  for (const [i, v] of list1.entries()) if (v !== list2[i]) return false;
-  return true;
-}
-
-const onConnectCallbacks: ((db: Db) => Promise<void>)[] = [];
-
-export function onConnect(callback: (db: Db) => Promise<void>): void {
-  onConnectCallbacks.push(callback);
-}
-
-export async function connect(): Promise<void> {
-  clientPromise = MongoClient.connect("" + get("MONGODB_CONNECTION_URL"), {
-    useNewUrlParser: true,
-    useUnifiedTopology: true,
-  });
-
-  const client = await clientPromise;
-  const db = client.db();
-  await Promise.all(onConnectCallbacks.map((c) => c(db)));
-}
-
-onConnect(async (db) => {
-  tasksCollection = db.collection("tasks");
-  await tasksCollection.createIndex({ device: 1, timestamp: 1 });
-
-  devicesCollection = db.collection("devices");
-  presetsCollection = db.collection("presets");
-  objectsCollection = db.collection("objects");
-  filesCollection = db.collection("fs.files");
-  provisionsCollection = db.collection("provisions");
-  virtualParametersCollection = db.collection("virtualParameters");
-  faultsCollection = db.collection("faults");
-  operationsCollection = db.collection("operations");
-  permissionsCollection = db.collection("permissions");
-  usersCollection = db.collection("users");
-  configCollection = db.collection("config");
-});
-
-export async function disconnect(): Promise<void> {
-  if (clientPromise) await (await clientPromise).close();
-}
-
-// Optimize projection by removing overlaps
-// This can modify the object
-function optimizeProjection(obj: { [path: string]: 1 }): { [path: string]: 1 } {
-  if (obj[""]) return { "": obj[""] };
-
-  const keys = Object.keys(obj).sort();
-  if (keys.length <= 1) return obj;
-
-  for (let i = 1; i < keys.length; ++i) {
-    const a = keys[i - 1];
-    const b = keys[i];
-    if (b.startsWith(a)) {
-      if (b.charAt(a.length) === "." || b.charAt(a.length - 1) === ".") {
-        delete obj[b];
-        keys.splice(i--, 1);
-      }
-    }
-  }
-  return obj;
-}
-
-export async function fetchDevice(
-  id: string,
-  timestamp: number
-): Promise<[Path, number, Attributes?][]> {
-  const res: [Path, number, Attributes?][] = [
-    [
-      Path.parse("Events"),
-      timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 0] },
-    ],
-    [
-      Path.parse("DeviceID"),
-      timestamp,
-      { object: [timestamp, 1], writable: [timestamp, 0] },
-    ],
-  ];
-
-  const device = await devicesCollection.findOne({ _id: id });
-  if (!device) return null;
-
-  function storeParams(
-    obj,
-    path: string,
-    pathLength: number,
-    ts: number
-  ): void {
-    if (obj["_timestamp"]) obj["_timestamp"] = +obj["_timestamp"];
-    if (obj["_attributesTimestamp"])
-      obj["_attributesTimestamp"] = +obj["_attributesTimestamp"];
-
-    const attrs: Attributes = {};
-    let t = obj["_timestamp"] || 1;
-    if (ts > t) t = ts;
-
-    if (obj["_value"] != null) {
-      attrs.value = [obj["_timestamp"] || 1, [obj["_value"], obj["_type"]]];
-      if (obj["_type"] === "xsd:dateTime")
-        attrs.value[1][0] = +attrs.value[1][0];
-
-      obj["_object"] = false;
-    }
-    if (obj["_writable"] != null)
-      attrs.writable = [ts || 1, obj["_writable"] ? 1 : 0];
-
-    if (obj["_object"] != null) attrs.object = [t, obj["_object"] ? 1 : 0];
-
-    if (obj["_notification"] != null) {
-      attrs.notification = [
-        obj["_attributesTimestamp"] || 1,
-        obj["_notification"],
-      ];
-    }
-
-    if (obj["_accessList"] != null)
-      attrs.accessList = [obj["_attributesTimestamp"] || 1, obj["_accessList"]];
-
-    try {
-      res.push([Path.parse(path), t, attrs]);
-    } catch (err) {
-      // The path parser is now more strict so we might be in a situation where
-      // the database contains invalid paths from before this change So here we
-      // encode the invalid characters.
-      const splits = path.split(".");
-      splits[splits.length - 1] =
-        encodeTag(splits[splits.length - 1]) + INVALID_PATH_SUFFIX;
-      path = splits.join(".");
-      res.push([Path.parse(path), t, attrs]);
-      return;
-    }
-
-    for (const [k, v] of Object.entries(obj)) {
-      if (!k.startsWith("_")) {
-        obj["_object"] = true;
-        storeParams(v, `${path}.${k}`, pathLength + 1, obj["_timestamp"]);
-      }
-    }
-
-    if (obj["_object"] && obj["_timestamp"])
-      res.push([Path.parse(path + ".*"), obj["_timestamp"]]);
-  }
-
-  const ts: number = device["_timestamp"] || 0;
-  if (ts) res.push([Path.parse("*"), ts]);
-
-  for (const [k, v] of Object.entries(device)) {
-    switch (k) {
-      case "_lastInform":
-        res.push([
-          Path.parse("Events.Inform"),
-          +v,
-          {
-            object: [+v, 0],
-            writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]],
-          },
-        ]);
-        break;
-      case "_lastBoot":
-        res.push([
-          Path.parse("Events.1_BOOT"),
-          +v,
-          {
-            object: [+v, 0],
-            writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]],
-          },
-        ]);
-        break;
-      case "_lastBootstrap":
-        res.push([
-          Path.parse("Events.0_BOOTSTRAP"),
-          +v,
-          {
-            object: [+v, 0],
-            writable: [+v, 0],
-            value: [+v, [+v, "xsd:dateTime"]],
-          },
-        ]);
-        break;
-      case "_registered":
-        // Use current timestamp for registered event attribute timestamps
-        res.push([
-          Path.parse("Events.Registered"),
-          timestamp,
-          {
-            object: [timestamp, 0],
-            writable: [timestamp, 0],
-            value: [timestamp, [+v, "xsd:dateTime"]],
-          },
-        ]);
-        break;
-      case "_id":
-        res.push([
-          Path.parse("DeviceID.ID"),
-          timestamp,
-          {
-            object: [timestamp, 0],
-            writable: [timestamp, 0],
-            value: [timestamp, [v as string, "xsd:string"]],
-          },
-        ]);
-        break;
-      case "_tags":
-        if ((v as string[]).length) {
-          res.push([
-            Path.parse("Tags"),
-            timestamp,
-            { object: [timestamp, 1], writable: [timestamp, 0] },
-          ]);
-        }
-
-        for (const t of v as string[]) {
-          res.push([
-            Path.parse("Tags." + encodeTag(t)),
-            timestamp,
-            {
-              object: [timestamp, 0],
-              writable: [timestamp, 1],
-              value: [timestamp, [true, "xsd:boolean"]],
-            },
-          ]);
-        }
-        break;
-      case "_deviceId":
-        if (v["_Manufacturer"] != null) {
-          res.push([
-            Path.parse("DeviceID.Manufacturer"),
-            timestamp,
-            {
-              object: [timestamp, 0],
-              writable: [timestamp, 0],
-              value: [timestamp, [v["_Manufacturer"], "xsd:string"]],
-            },
-          ]);
-        }
-
-        if (v["_OUI"] != null) {
-          res.push([
-            Path.parse("DeviceID.OUI"),
-            timestamp,
-            {
-              object: [timestamp, 0],
-              writable: [timestamp, 0],
-              value: [timestamp, [v["_OUI"], "xsd:string"]],
-            },
-          ]);
-        }
-
-        if (v["_ProductClass"] != null) {
-          res.push([
-            Path.parse("DeviceID.ProductClass"),
-            timestamp,
-            {
-              object: [timestamp, 0],
-              writable: [timestamp, 0],
-              value: [timestamp, [v["_ProductClass"], "xsd:string"]],
-            },
-          ]);
-        }
-
-        if (v["_SerialNumber"] != null) {
-          res.push([
-            Path.parse("DeviceID.SerialNumber"),
-            timestamp,
-            {
-              object: [timestamp, 0],
-              writable: [timestamp, 0],
-              value: [timestamp, [v["_SerialNumber"], "xsd:string"]],
-            },
-          ]);
-        }
-        break;
-      default:
-        if (!k.startsWith("_")) storeParams(v, k, 1, ts);
-    }
-  }
-  return res;
-}
-
-export async function saveDevice(
-  deviceId: string,
-  deviceData: DeviceData,
-  isNew: boolean,
-  sessionTimestamp: number
-): Promise<void> {
-  const update = { $set: {}, $unset: {}, $addToSet: {}, $pull: {} };
-
-  for (const diff of deviceData.timestamps.diff()) {
-    if (diff[0].wildcard !== 1 << (diff[0].length - 1)) continue;
-
-    if (
-      diff[0].segments[0] === "Events" ||
-      diff[0].segments[0] === "DeviceID" ||
-      diff[0].segments[0] === "Tags"
-    )
-      continue;
-
-    const parent = deviceData.paths.get(diff[0].slice(0, -1));
-
-    // Param timestamps may be greater than session timestamp to track revisions
-    if (diff[2] > sessionTimestamp) diff[2] = sessionTimestamp;
-
-    if (diff[2] == null && diff[1] != null) {
-      update["$unset"][
-        parent.length ? parent.toString() + "._timestamp" : "_timestamp"
-      ] = 1;
-    } else {
-      if (parent && (!parent.length || deviceData.attributes.has(parent))) {
-        update["$set"][
-          parent.length ? parent.toString() + "._timestamp" : "_timestamp"
-        ] = new Date(diff[2]);
-      }
-    }
-  }
-
-  for (const diff of deviceData.attributes.diff()) {
-    const path = diff[0];
-    const value1 = (((diff[1] || {}).value || [])[1] || [])[0];
-    const value2 = (((diff[2] || {}).value || [])[1] || [])[0];
-    const valueType1 = (((diff[1] || {}).value || [])[1] || [])[1];
-    const valueType2 = (((diff[2] || {}).value || [])[1] || [])[1];
-    const valueTimestamp1 = ((diff[1] || {}).value || [])[0];
-    const valueTimestamp2 = ((diff[2] || {}).value || [])[0];
-    const object1 = ((diff[1] || {}).object || [])[1];
-    const object2 = ((diff[2] || {}).object || [])[1];
-    const writable2 = ((diff[2] || {}).writable || [])[1];
-    const writable1 = ((diff[1] || {}).writable || [])[1];
-    const attributesTimestamp1 = ((diff[1] || {}).notification || [])[0];
-    const attributesTimestamp2 = ((diff[2] || {}).notification || [])[0];
-    const notification1 = ((diff[1] || {}).notification || [])[1];
-    const notification2 = ((diff[2] || {}).notification || [])[1];
-    const accessList1 = ((diff[1] || {}).accessList || [])[1];
-    const accessList2 = ((diff[2] || {}).accessList || [])[1];
-
-    switch (path.segments[0]) {
-      case "Events":
-        if (path.length === 2 && value2 !== value1) {
-          if (!diff[2]) {
-            switch (path.segments[1]) {
-              case "Inform":
-                update["$unset"]["_lastInform"] = 1;
-                break;
-              case "1_BOOT":
-                update["$unset"]["_lastBoot"] = 1;
-                break;
-              case "0_BOOTSTRAP":
-                update["$unset"]["_lastBootstrap"] = 1;
-                break;
-              case "Registered":
-                update["$unset"]["_registered"] = 1;
-            }
-          } else {
-            const t = new Date(diff[2].value[1][0] as number);
-            switch (path.segments[1]) {
-              case "Inform":
-                update["$set"]["_lastInform"] = t;
-                break;
-              case "1_BOOT":
-                update["$set"]["_lastBoot"] = t;
-                break;
-              case "0_BOOTSTRAP":
-                update["$set"]["_lastBootstrap"] = t;
-                break;
-              case "Registered":
-                update["$set"]["_registered"] = t;
-            }
-          }
-        }
-
-        break;
-      case "DeviceID":
-        if (value2 !== value1) {
-          const v = diff[2].value[1][0];
-          switch (path.segments[1]) {
-            case "ID":
-              update["$set"]["_id"] = v;
-              break;
-            case "Manufacturer":
-              update["$set"]["_deviceId._Manufacturer"] = v;
-              break;
-            case "OUI":
-              update["$set"]["_deviceId._OUI"] = v;
-              break;
-            case "ProductClass":
-              update["$set"]["_deviceId._ProductClass"] = v;
-              break;
-            case "SerialNumber":
-              update["$set"]["_deviceId._SerialNumber"] = v;
-          }
-        }
-        break;
-      case "Tags":
-        if (value2 !== value1) {
-          if (value2 != null) {
-            if (!update["$addToSet"]["_tags"])
-              update["$addToSet"]["_tags"] = { $each: [] };
-            update["$addToSet"]["_tags"]["$each"].push(path.segments[1]);
-          } else {
-            if (!update["$pull"]["_tags"]) {
-              update["$pull"]["_tags"] = {
-                $in: [],
-              };
-            }
-            update["$pull"]["_tags"]["$in"].push(path.segments[1]);
-          }
-        }
-
-        break;
-      default:
-        if (!diff[2]) {
-          let pathStr = path.toString();
-          // Paths with that suffix are encoded and need to be decoded
-          if (pathStr.endsWith(INVALID_PATH_SUFFIX)) {
-            const splits = pathStr.split(".");
-            splits[splits.length - 1] = decodeTag(
-              splits[splits.length - 1].slice(0, 0 - INVALID_PATH_SUFFIX.length)
-            );
-            pathStr = splits.join(".");
-          }
-          update["$unset"][pathStr] = 1;
-          continue;
-        }
-
-        for (const attrName of Object.keys(diff[2])) {
-          // Param timestamps may be greater than session timestamp to track revisions
-          if (diff[2][attrName][0] > sessionTimestamp)
-            diff[2][attrName][0] = sessionTimestamp;
-
-          if (diff[2][attrName][1] != null) {
-            switch (attrName) {
-              case "value":
-                if (value2 !== value1) {
-                  if (
-                    valueType2 === "xsd:dateTime" &&
-                    Number.isInteger(value2 as number)
-                  ) {
-                    update["$set"][path.toString() + "._value"] = new Date(
-                      value2 as number
-                    );
-                  } else {
-                    update["$set"][path.toString() + "._value"] = value2;
-                  }
-                }
-
-                if (valueType2 !== valueType1)
-                  update["$set"][path.toString() + "._type"] = valueType2;
-
-                if (valueTimestamp2 !== valueTimestamp1) {
-                  update["$set"][path.toString() + "._timestamp"] = new Date(
-                    valueTimestamp2
-                  );
-                }
-
-                break;
-              case "object":
-                if (!diff[1] || !diff[1].object || object2 !== object1) {
-                  update["$set"][
-                    path.length ? path.toString() + "._object" : "_object"
-                  ] = !!object2;
-                }
-
-                break;
-              case "writable":
-                if (!diff[1] || !diff[1].writable || writable2 !== writable1) {
-                  update["$set"][
-                    path.length ? path.toString() + "._writable" : "_writable"
-                  ] = !!writable2;
-                }
-
-                break;
-              case "notification":
-                if (
-                  !diff[1] ||
-                  !diff[1].notification ||
-                  notification2 !== notification1
-                ) {
-                  update["$set"][
-                    path.length
-                      ? path.toString() + "._notification"
-                      : "_notification"
-                  ] = notification2;
-                }
-
-                if (attributesTimestamp2 !== attributesTimestamp1) {
-                  update["$set"][
-                    path.toString() + "._attributesTimestamp"
-                  ] = new Date(attributesTimestamp2);
-                }
-
-                break;
-              case "accessList":
-                if (
-                  !diff[1] ||
-                  !diff[1].accessList ||
-                  !compareAccessLists(accessList2, accessList1)
-                ) {
-                  update["$set"][
-                    path.length
-                      ? path.toString() + "._accessList"
-                      : "_accessList"
-                  ] = accessList2;
-                }
-
-                if (attributesTimestamp2 !== attributesTimestamp1) {
-                  update["$set"][
-                    path.toString() + "._attributesTimestamp"
-                  ] = new Date(attributesTimestamp2);
-                }
-            }
-          }
-        }
-
-        if (diff[1]) {
-          for (const attrName of Object.keys(diff[1])) {
-            if (
-              diff[1][attrName][1] != null &&
-              (!diff[2] || !diff[2][attrName] || diff[2][attrName][1] == null)
-            ) {
-              const p = path.length ? path.toString() + "." : "";
-              update["$unset"][`${p}_${attrName}`] = 1;
-              if (attrName === "value") {
-                update["$unset"][p + "_type"] = 1;
-                update["$unset"][p + "_timestamp"] = 1;
-              } else if (attrName === "notification") {
-                if (accessList2 == null)
-                  update["$unset"][`${p}_attributesTimestamp`] = 1;
-              } else if (attrName === "accessList") {
-                if (notification2 == null)
-                  update["$unset"][`${p}_attributesTimestamp`] = 1;
-              }
-            }
-          }
-        }
-    }
-  }
-
-  update["$unset"] = optimizeProjection(update["$unset"]);
-
-  // Remove overlap possibly caused by parameters changing from objects
-  // to regular parameters or vice versa. Reason being that _timestamp
-  // represents two different things depending on whether the parameter
-  // is an object or not.
-  for (const k of Object.keys(update["$unset"]))
-    if (update["$set"][k] != null) delete update["$unset"][k];
-
-  // Remove empty keys
-  for (const [k, v] of Object.entries(update)) {
-    if (k === "$addToSet") {
-      for (const [kk, vv] of Object.entries(v))
-        if (!vv["$each"].length) delete v[kk];
-    } else if (k === "$pull") {
-      for (const [kk, vv] of Object.entries(v))
-        if (!vv["$in"].length) delete v[kk];
-    }
-    if (!Object.keys(v).length) delete update[k];
-  }
-
-  if (!Object.keys(update).length) return;
-
-  // Mongo doesn't allow $addToSet and $pull at the same time
-  let update2;
-  if (update["$addToSet"] && update["$pull"]) {
-    update2 = { $pull: update["$pull"] };
-    delete update["$pull"];
-  }
-
-  const result = await devicesCollection.updateOne({ _id: deviceId }, update, {
-    upsert: isNew,
-  });
-
-  if (result.result.n !== 1)
-    throw new Error(`Device ${deviceId} not found in database`);
-
-  if (update2) {
-    await devicesCollection.updateOne({ _id: deviceId }, update2);
-    return;
-  }
-}
-
-export async function getFaults(
-  deviceId: string
-): Promise<{ [channel: string]: SessionFault }> {
-  const res = await faultsCollection
-    .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
-    .toArray();
-
-  const faults: { [channel: string]: SessionFault } = {};
-  for (const r of res) {
-    const channel = r._id.slice(deviceId.length + 1);
-    delete r._id;
-    delete r.channel;
-    delete r.device;
-    r.timestamp = +r.timestamp;
-    r.provisions = JSON.parse(r.provisions);
-    faults[channel] = r;
-  }
-
-  return faults;
-}
-
-export async function saveFault(
-  deviceId: string,
-  channel: string,
-  fault: SessionFault
-): Promise<void> {
-  const id = `${deviceId}:${channel}`;
-  const f = Object.assign({}, fault) as Record<string, any>;
-  f["_id"] = id;
-  f["device"] = deviceId;
-  f["channel"] = channel;
-  f["timestamp"] = new Date(fault.timestamp);
-  f["provisions"] = JSON.stringify(fault.provisions);
-  await faultsCollection.replaceOne({ _id: id }, f, { upsert: true });
-}
-
-export async function deleteFault(
-  deviceId: string,
-  channel: string
-): Promise<void> {
-  await faultsCollection.deleteOne({ _id: `${deviceId}:${channel}` });
-}
-
-export async function getDueTasks(
-  deviceId: string,
-  timestamp: number
-): Promise<[Task[], number]> {
-  const cur = tasksCollection.find({ device: deviceId }).sort({ timestamp: 1 });
-  const tasks = [] as Task[];
-
-  for await (const task of cur) {
-    if (task.timestamp) task.timestamp = +task.timestamp;
-    if (task.expiry) task.expiry = +task.expiry;
-    if (task.timestamp >= timestamp) return [tasks, +task.timestamp];
-    task._id = String(task._id);
-
-    tasks.push(task);
-
-    // For API compatibility
-    if (task.name === "download" && task.file) {
-      let q;
-      if (ObjectID.isValid(task.file))
-        q = { _id: { $in: [task.file, new ObjectID(task.file)] } };
-      else q = { _id: task.file };
-
-      const res = await filesCollection.find(q).toArray();
-
-      if (res[0]) {
-        if (!task.fileType) task.fileType = res[0].metadata.fileType;
-
-        if (!task.fileName)
-          task.fileName = res[0].filename || res[0]._id.toString();
-      }
-    }
-  }
-  return [tasks, null];
-}
-
-export async function clearTasks(
-  deviceId: string,
-  taskIds: string[]
-): Promise<void> {
-  await tasksCollection.deleteMany({
-    _id: { $in: taskIds.map((id) => new ObjectID(id)) },
-  });
-}
-
-export async function getOperations(
-  deviceId: string
-): Promise<{ [commandKey: string]: Operation }> {
-  const res = await operationsCollection
-    .find({ _id: { $regex: `^${escapeRegExp(deviceId)}\\:` } })
-    .toArray();
-
-  const operations: { [commandKey: string]: Operation } = {};
-  for (const r of res) {
-    const commandKey = r._id.slice(deviceId.length + 1);
-    delete r._id;
-    // Workaround for a bug in v1.2.1 where operation object is saved without deserialization
-    if (typeof r.provisions !== "string") {
-      operations[commandKey] = r;
-      continue;
-    }
-    r.timestamp = +r.timestamp;
-    if (r.args) r.args = JSON.parse(r.args);
-    r.provisions = JSON.parse(r.provisions);
-    r.retries = JSON.parse(r.retries);
-    operations[commandKey] = r;
-  }
-  return operations;
-}
-
-export async function saveOperation(
-  deviceId: string,
-  commandKey: string,
-  operation: Operation
-): Promise<void> {
-  const id = `${deviceId}:${commandKey}`;
-  const o = Object.assign({}, operation) as Record<string, any>;
-  o["_id"] = id;
-  o["timestamp"] = new Date(operation.timestamp);
-  o["provisions"] = JSON.stringify(operation.provisions);
-  o["retries"] = JSON.stringify(operation.retries);
-  o["args"] = JSON.stringify(operation.args);
-  await operationsCollection.replaceOne({ _id: id }, o, {
-    upsert: true,
-  });
-}
-
-export async function deleteOperation(
-  deviceId: string,
-  commandKey: string
-): Promise<void> {
-  await operationsCollection.deleteOne({ _id: `${deviceId}:${commandKey}` });
-}
-
-export async function getPresets(): Promise<Record<string, any>[]> {
-  return presetsCollection.find().toArray();
-}
-
-export async function getObjects(): Promise<Record<string, any>[]> {
-  return objectsCollection.find().toArray();
-}
-
-export async function getProvisions(): Promise<Record<string, any>[]> {
-  return provisionsCollection.find().toArray();
-}
-
-export async function getVirtualParameters(): Promise<Record<string, any>[]> {
-  return virtualParametersCollection.find().toArray();
-}
-
-export function getFiles(): Promise<Record<string, any>[]> {
-  return filesCollection.find().toArray();
-}
-
-export async function getConfig(): Promise<
-  { id: string; value: Expression }[]
-> {
-  const res = await configCollection.find().toArray();
-  return res.map((c) => ({
-    id: c["_id"],
-    value: parse(c["value"]),
-  }));
-}
-
-interface Permission {
-  role: string;
-  resource: string;
-  access: number;
-  filter: string;
-  validate: string;
-}
-
-export async function getPermissions(): Promise<Permission[]> {
-  return permissionsCollection.find().toArray();
-}
-
-interface User {
-  _id: string;
-  password: string;
-  salt: string;
-  roles: string;
-}
-
-export async function getUsers(): Promise<User[]> {
-  return usersCollection.find().toArray();
-}

lib/debug.ts

@@ -1,204 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import {
-  IncomingMessage,
-  ServerResponse,
-  ClientRequest,
-  RequestOptions,
-} from "http";
-import { Socket } from "net";
-import { appendFileSync } from "fs";
-import { stringify } from "./common/yaml";
-import * as config from "./config";
-
-const DEBUG_FILE = "" + config.get("DEBUG_FILE");
-const DEBUG_FORMAT = "" + config.get("DEBUG_FORMAT");
-
-const connectionTimestamps = new WeakMap<Socket, Date>();
-
-function getConnectionTimestamp(connection: Socket): Date {
-  let t = connectionTimestamps.get(connection);
-  if (!t) {
-    t = new Date();
-    connectionTimestamps.set(connection, t);
-  }
-  return t;
-}
-
-export function incomingHttpRequest(
-  httpRequest: IncomingMessage,
-  deviceId: string,
-  body: string
-): void {
-  if (!DEBUG_FILE) return;
-  const now = new Date();
-  const con = httpRequest.socket;
-  const msg = {
-    event: "incoming HTTP request",
-    timestamp: now,
-    remoteAddress: con.remoteAddress,
-    deviceId: deviceId,
-    connection: getConnectionTimestamp(con),
-    localPort: con.localPort,
-    method: httpRequest.method,
-    url: httpRequest.url,
-    headers: httpRequest.headers,
-    body: body,
-  };
-
-  if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
-  else if (DEBUG_FORMAT === "json")
-    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
-  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
-}
-
-export function outgoingHttpResponse(
-  httpResponse: ServerResponse,
-  deviceId: string,
-  body: string
-): void {
-  if (!DEBUG_FILE) return;
-  const now = new Date();
-  const con = httpResponse.socket;
-  const msg = {
-    event: "outgoing HTTP response",
-    timestamp: now,
-    remoteAddress: con.remoteAddress,
-    deviceId: deviceId,
-    connection: getConnectionTimestamp(con),
-    statusCode: httpResponse.statusCode,
-    headers: httpResponse.getHeaders(),
-    body: body,
-  };
-
-  if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
-  else if (DEBUG_FORMAT === "json")
-    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
-  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
-}
-
-export function outgoingHttpRequest(
-  httpRequest: ClientRequest,
-  deviceId: string,
-  options: RequestOptions,
-  body: string
-): void {
-  if (!DEBUG_FILE) return;
-  const now = new Date();
-  const con = httpRequest.socket;
-  const msg = {
-    event: "outgoing HTTP request",
-    timestamp: now,
-    remoteAddress: con.remoteAddress,
-    deviceId: deviceId,
-    connection: getConnectionTimestamp(con),
-    remotePort: options.port,
-    method: options.method || "GET",
-    url: options.path,
-    headers: httpRequest.getHeaders(),
-    body: body,
-  };
-
-  if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
-  else if (DEBUG_FORMAT === "json")
-    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
-  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
-}
-
-export function outgoingHttpRequestError(
-  httpRequest: ClientRequest,
-  deviceId: string,
-  options: RequestOptions,
-  err: Error
-): void {
-  if (!DEBUG_FILE) return;
-  const now = new Date();
-  const msg = {
-    event: "outgoing HTTP request",
-    timestamp: now,
-    remoteAddress: options.hostname,
-    deviceId: deviceId,
-    connection: null,
-    remotePort: options.port,
-    method: options.method,
-    url: options.path,
-    headers: httpRequest.getHeaders(),
-    error: err.message,
-  };
-
-  if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
-  else if (DEBUG_FORMAT === "json")
-    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
-  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
-}
-
-export function incomingHttpResponse(
-  httpResponse: IncomingMessage,
-  deviceId: string,
-  body: string
-): void {
-  if (!DEBUG_FILE) return;
-  const now = new Date();
-  const con = httpResponse.socket;
-  const msg = {
-    event: "incoming HTTP response",
-    timestamp: now,
-    remoteAddress: con.remoteAddress,
-    deviceId: deviceId,
-    connection: getConnectionTimestamp(httpResponse.socket),
-    statusCode: httpResponse.statusCode,
-    headers: httpResponse.headers,
-    body: body,
-  };
-
-  if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
-  else if (DEBUG_FORMAT === "json")
-    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
-  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
-}
-
-export function outgoingUdpMessage(
-  remoteAddress: string,
-  deviceId: string,
-  remotePort: number,
-  body: string
-): void {
-  if (!DEBUG_FILE) return;
-  const now = new Date();
-  const msg = {
-    event: "outgoing UDP message",
-    timestamp: now,
-    remoteAddress: remoteAddress,
-    deviceId: deviceId,
-    remotePort: remotePort,
-    body: body,
-  };
-
-  if (DEBUG_FORMAT === "yaml")
-    appendFileSync(DEBUG_FILE, "---\n" + stringify(msg));
-  else if (DEBUG_FORMAT === "json")
-    appendFileSync(DEBUG_FILE, JSON.stringify(msg) + "\n");
-  else throw new Error(`Unrecognized DEBUG_FORMAT option`);
-}

lib/default-provisions.js

@@ -0,0 +1,117 @@
+/**
+ * Copyright 2013-2017  Zaid Abdulla
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+"use strict";
+
+const config = require("./config");
+const common = require("./common");
+const device = require("./device");
+const scheduling = require("./scheduling");
+
+const MAX_DEPTH = config.get("MAX_DEPTH");
+
+
+function refresh(sessionContext, provision, declarations, startRevision, endRevision) {
+  let path = common.parsePath(provision[1]).slice();
+  let l = path.length;
+  path.length = MAX_DEPTH;
+  path.fill("*", l);
+  let every = 1000 * (provision[2] || 1);
+  let offset = scheduling.variance(sessionContext.deviceId, every);
+  let t = scheduling.interval(sessionContext.timestamp, every, offset);
+
+  for (let i = l; i < path.length; ++ i)
+    declarations.push([path.slice(0, i), t, {object: 1, writable: 1, value: t}]);
+
+  return true;
+}
+
+
+function value(sessionContext, provision, declarations, startRevision, endRevision) {
+  let path = common.parsePath(provision[1]);
+  declarations.push([path, 1, {value: 1}, null, {value: [provision[2]]}]);
+
+  return true;
+}
+
+
+function tag(sessionContext, provision, declarations, startRevision, endRevision) {
+  let path = ["Tags", provision[1]];
+  declarations.push([path, 1, {value: 1}, null, {value: [provision[2]]}]);
+
+  return true;
+}
+
+
+function reboot(sessionContext, provision, declarations, startRevision, endRevision) {
+  declarations.push([["Reboot"], 1, {value: 1}, null, {value: [sessionContext.timestamp]}]);
+
+  return true;
+}
+
+
+function reset(sessionContext, provision, declarations, startRevision, endRevision) {
+  declarations.push([["FactoryReset"], 1, {value: 1}, null, {value: [sessionContext.timestamp]}]);
+
+  return true;
+}
+
+
+function download(sessionContext, provision, declarations, startRevision, endRevision) {
+  let alias = [["FileType"], provision[1] || "", ["FileName"], provision[2] || "",
+    ["TargetFileName"], provision[3] || ""];
+
+  declarations.push([["Downloads", alias], 1, {}, 1]);
+  declarations.push([["Downloads", alias, "Download"], 1, {value: 1}, null,
+    {value: [sessionContext.timestamp]}]);
+
+  return true;
+}
+
+
+function instances(sessionContext, provision, declarations, startRevision, endRevision) {
+  let count = Number(provision[2]);
+
+  if (Number.isNaN(count))
+    return true;
+
+  let path = common.parsePath(provision[1]);
+
+  if (provision[2][0] === "+" || provision[2][0] === "-") {
+    declarations.push([path, 1]);
+
+    if (endRevision === startRevision)
+      return false;
+
+    let unpacked = device.unpack(sessionContext.deviceData, path, startRevision + 1);
+    count = Math.max(0, unpacked.length + count);
+  }
+
+  declarations.push([path, 1, null, count]);
+
+  return true;
+}
+
+
+exports.refresh = refresh;
+exports.value = value;
+exports.tag = tag;
+exports.reboot = reboot;
+exports.reset = reset;
+exports.download = download;
+exports.instances = instances;

lib/default-provisions.ts

@@ -1,278 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
-import * as config from "./config";
-import * as device from "./device";
-import * as scheduling from "./scheduling";
-import { SessionContext, Declaration } from "./types";
-
-const MAX_DEPTH = +config.get("MAX_DEPTH");
-
-export function refresh(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
-  if (
-    (provision.length !== 2 || typeof provision[1] !== "string") &&
-    (provision.length !== 3 ||
-      typeof provision[1] !== "string" ||
-      typeof provision[2] !== "number") &&
-    (provision.length < 4 ||
-      typeof provision[1] !== "string" ||
-      typeof provision[2] !== "number" ||
-      typeof provision[3] !== "boolean")
-  )
-    throw new Error("Invalid arguments");
-
-  const every = 1000 * ((provision[2] as number) || 1);
-  const offset = scheduling.variance(sessionContext.deviceId, every);
-  const t = scheduling.interval(sessionContext.timestamp, every, offset);
-
-  let attrGet;
-  let refreshChildren;
-  if (provision[3] == null) {
-    refreshChildren = true;
-    attrGet = { object: 1, writable: 1, value: t };
-  } else {
-    attrGet = {};
-    refreshChildren = !!provision[3];
-    for (const a of provision.slice(4)) attrGet[a as string] = t;
-  }
-
-  let path = Path.parse(provision[1]);
-  let l = path.length;
-  if (refreshChildren) {
-    const segments = path.segments.slice();
-    l = segments.length;
-    segments.length = MAX_DEPTH;
-    segments.fill("*", l);
-    path = Path.parse(segments.join("."));
-  }
-
-  for (let i = l; i <= path.length; ++i) {
-    declarations.push({
-      path: path.slice(0, i),
-      pathGet: t,
-      pathSet: null,
-      attrGet: attrGet,
-      attrSet: null,
-      defer: true,
-    });
-  }
-
-  return true;
-}
-
-export function value(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
-  if (
-    provision.length < 3 ||
-    provision.length > 4 ||
-    typeof provision[1] !== "string"
-  )
-    throw new Error("Invalid arguments");
-
-  let attr: string, val: any;
-
-  if (provision.length === 3) {
-    attr = "value";
-    val = provision[2];
-  } else {
-    attr = (provision[2] as string) || "";
-    val = provision[3];
-  }
-
-  if (attr === "accessList") {
-    val = (val || "")
-      .split(",")
-      .map((s) => s.trim())
-      .filter((s) => !!s);
-  } else if (attr === "value") {
-    val = [val];
-  }
-
-  declarations.push({
-    path: Path.parse(provision[1]),
-    pathGet: 1,
-    pathSet: null,
-    attrGet: { [attr]: 1 },
-    attrSet: { [attr]: val },
-    defer: true,
-  });
-
-  return true;
-}
-
-export function tag(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
-  if (
-    provision.length !== 3 ||
-    typeof provision[1] !== "string" ||
-    typeof provision[2] !== "boolean"
-  )
-    throw new Error("Invalid arguments");
-
-  declarations.push({
-    path: Path.parse(`Tags.${provision[1]}`),
-    pathGet: 1,
-    pathSet: null,
-    attrGet: { value: 1 },
-    attrSet: { value: [provision[2]] },
-    defer: true,
-  });
-
-  return true;
-}
-
-export function reboot(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
-  if (provision.length !== 1) throw new Error("Invalid arguments");
-
-  declarations.push({
-    path: Path.parse("Reboot"),
-    pathGet: 1,
-    pathSet: null,
-    attrGet: { value: 1 },
-    attrSet: { value: [sessionContext.timestamp] },
-    defer: true,
-  });
-
-  return true;
-}
-
-export function reset(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
-  if (provision.length !== 1) throw new Error("Invalid arguments");
-
-  declarations.push({
-    path: Path.parse("FactoryReset"),
-    pathGet: 1,
-    pathSet: null,
-    attrGet: { value: 1 },
-    attrSet: { value: [sessionContext.timestamp] },
-    defer: true,
-  });
-
-  return true;
-}
-
-export function download(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[]
-): boolean {
-  if (
-    (provision.length !== 3 ||
-      typeof provision[1] !== "string" ||
-      typeof provision[2] !== "string") &&
-    (provision.length !== 4 ||
-      typeof provision[1] !== "string" ||
-      typeof provision[2] !== "string" ||
-      typeof provision[3] !== "string")
-  )
-    throw new Error("Invalid arguments");
-
-  const alias = [
-    `FileType:${JSON.stringify(provision[1] || "")}`,
-    `FileName:${JSON.stringify(provision[2] || "")}`,
-    `TargetFileName:${JSON.stringify(provision[3] || "")}`,
-  ].join(",");
-
-  declarations.push({
-    path: Path.parse(`Downloads.[${alias}]`),
-    pathGet: 1,
-    pathSet: 1,
-    attrGet: null,
-    attrSet: null,
-    defer: true,
-  });
-
-  declarations.push({
-    path: Path.parse(`Downloads.[${alias}].Download`),
-    pathGet: 1,
-    pathSet: null,
-    attrGet: { value: 1 },
-    attrSet: { value: [sessionContext.timestamp] },
-    defer: true,
-  });
-
-  return true;
-}
-
-export function instances(
-  sessionContext: SessionContext,
-  provision: (string | number | boolean)[],
-  declarations: Declaration[],
-  startRevision: number,
-  endRevision: number
-): boolean {
-  if (provision.length !== 3 || typeof provision[1] !== "string")
-    throw new Error("Invalid arguments");
-
-  let count = Number(provision[2]);
-
-  if (Number.isNaN(count)) throw new Error("Invalid arguments");
-
-  const path = Path.parse(provision[1]);
-
-  if (provision[2][0] === "+" || provision[2][0] === "-") {
-    declarations.push({
-      path: path,
-      pathGet: 1,
-      pathSet: null,
-      attrGet: null,
-      attrSet: null,
-      defer: true,
-    });
-
-    if (endRevision === startRevision) return false;
-
-    const unpacked = device.unpack(
-      sessionContext.deviceData,
-      path,
-      startRevision + 1
-    );
-    count = Math.max(0, unpacked.length + count);
-  }
-
-  declarations.push({
-    path: path,
-    pathGet: 1,
-    pathSet: count,
-    attrGet: null,
-    attrSet: null,
-    defer: true,
-  });
-
-  return true;
-}

lib/device.js

@@ -0,0 +1,395 @@
+/**
+ * Copyright 2013-2017  Zaid Abdulla
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+"use strict";
+
+const common = require('./common');
+
+
+const CHANGE_FLAGS = {
+  object: 2,
+  writable: 4,
+  value: 8
+};
+
+
+function parseBool(v) {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1")
+    return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else
+    return null;
+}
+
+
+function sanitizeParameterValue(parameterValue) {
+  if (parameterValue[0] != null) {
+    switch (parameterValue[1]) {
+      case 'xsd:boolean':
+        if (typeof parameterValue[0] !== 'boolean') {
+          let v = parseBool(parameterValue[0]);
+          if (v == null)
+            v = '' + parameterValue[0];
+          parameterValue[0] = v;
+        }
+        break;
+      case 'xsd:string':
+      case 'xsd:base64':
+      case 'xsd:hexBinary':
+        if (typeof parameterValue[0] !== 'string') {
+          parameterValue[0] = '' + parameterValue[0];
+        }
+        break;
+      case 'xsd:int':
+      case 'xsd:unsignedInt':
+        if (typeof parameterValue[0] !== 'number') {
+          let v = parseInt(parameterValue[0]);
+          if (isNaN(v))
+            v = '' + parameterValue[0];
+          parameterValue[0] = v;
+        }
+        break;
+      case 'xsd:dateTime':
+        if (typeof parameterValue[0] !== 'number') {
+          // Don't use parseInt because it reads date string as a number
+          let v = +parameterValue[0];
+          if (isNaN(v)) {
+            v = Date.parse(parameterValue[0]);
+            if (isNaN(v))
+              v = '' + parameterValue[0];
+          }
+          parameterValue[0] = v;
+        }
+        break;
+      default:
+        if (parameterValue[1] != null)
+          throw new Error(`Parameter value type "${parameterValue[1]}" not recognized.`);
+
+        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
+        break;
+    }
+  }
+  return parameterValue;
+}
+
+
+function getAliasDeclarations(path, timestamp) {
+  var decs = [];
+
+  function recursive(pattern, prefix) {
+    var pattern = prefix.concat(pattern);
+
+    for (let i = prefix.length; i < pattern.length; ++ i) {
+      if (Array.isArray(pattern[i])) {
+        let pat = pattern[i];
+        pattern[i] = '*';
+        for (let j = 0; j < pat.length; j += 2)
+          recursive(pat[j], pattern.slice(0, i + 1));
+      }
+    }
+
+    if (prefix.length)
+      decs.push([pattern, timestamp, {value: timestamp}]);
+    else
+      decs.push([pattern, timestamp]);
+  }
+
+  recursive(path, []);
+
+  return decs;
+}
+
+
+function unpack(deviceData, path, revision) {
+  let allMatches = [];
+  if (!path.alias) {
+    for (let p of deviceData.paths.find(path, false, true))
+      if (deviceData.attributes.has(p, revision))
+        allMatches.push(p);
+  }
+  else {
+    let wildcardPath = path.slice();
+    wildcardPath.wildcard = path.wildcard;
+    for (let i = 0; i < wildcardPath.length; ++ i)
+      if (Array.isArray(wildcardPath[i])) {
+        wildcardPath[i] = '*';
+        wildcardPath.wildcard |= 1 << i;
+      }
+
+    for (let p of deviceData.paths.find(wildcardPath, false, true))
+      if (deviceData.attributes.has(p, revision))
+        allMatches.push(p);
+
+    for (let i = path.length - 1; i >= 0; -- i) {
+      if (Array.isArray(path[i])) {
+        for (let j = 0; j < path[i].length; j += 2) {
+          let p = wildcardPath.slice(0, i + 1).concat(path[i][j]);
+          p.alias = (wildcardPath.alias & ((1 << (i + 1)) - 1)) + path[i][j].alias << (i + 1);
+          p.wildcard = (wildcardPath.wildcard & ((1 << (i + 1)) - 1)) + path[i][j].wildcard << (i + 1);
+          let unpacked = unpack(deviceData, p, revision);
+          let filtered = [];
+          for (let up of unpacked) {
+            let attributes = deviceData.attributes.get(up, revision);
+            if (attributes && attributes.value && attributes.value[1] &&
+                sanitizeParameterValue([path[i][j + 1], attributes.value[1][1]])[0] === attributes.value[1][0]) {
+              for (let m = 0; m < allMatches.length; ++ m) {
+                let k;
+                for (k = i; k >= 0; -- k)
+                  if (allMatches[m][k] != up[k])
+                    break
+
+                if (k < 0) {
+                  filtered.push(allMatches[m]);
+                  allMatches[m] = [];
+                }
+              }
+            }
+          }
+          allMatches = filtered;
+        }
+      }
+    }
+  }
+
+  allMatches.sort(function(a, b) {
+    for (let i = 0; i < a.length; ++ i) {
+      if (a === b)
+        continue;
+
+      // Use numeric sorting for numbers
+      let ia = parseInt(a), ib = parseInt(b);
+      if (ia.toString() === a && ib.toString() === b) {
+        a = ia;
+        b = ib;
+      }
+
+      if (a < b)
+        return -1;
+      else
+        return 1;
+    }
+    return 0;
+  });
+
+  return allMatches;
+}
+
+
+function clear(deviceData, path, timestamp, attributes, changeFlags = 0) {
+  let changeTrackers = {};
+
+  timestamp = timestamp || 0;
+
+  let descendantsTimestamp = timestamp;
+  if (attributes && attributes.object) {
+    if (attributes.object > descendantsTimestamp)
+      descendantsTimestamp = attributes.object;
+    if (!(attributes.object <= attributes.value))
+      attributes.value = attributes.object;
+  }
+
+  for (let p of deviceData.paths.find(path, true, true, descendantsTimestamp ? 99 : path.length)) {
+    let tracker = deviceData.trackers.get(p);
+    for (let k in tracker)
+      changeTrackers[k] |= tracker[k];
+
+    let currentTimestamp = deviceData.timestamps.get(p);
+    if (currentTimestamp === undefined)
+      continue;
+
+    if (timestamp > currentTimestamp ||
+      (descendantsTimestamp > currentTimestamp && p.length > path.length)) {
+      deviceData.timestamps.delete(p);
+      deviceData.attributes.delete(p);
+      changeFlags |= 1;
+    }
+    else if (attributes && p.length == path.length) {
+      let currentAttributes = deviceData.attributes.get(p);
+      if (currentAttributes) {
+        let newAttrs;
+        for (let attrName in attributes) {
+          if (attrName in currentAttributes &&
+              attributes[attrName] > currentAttributes[attrName][0]) {
+            changeFlags |= CHANGE_FLAGS[attrName];
+            if (!newAttrs) {
+              newAttrs = Object.assign({}, currentAttributes);
+              deviceData.attributes.set(p, newAttrs);
+            }
+            delete newAttrs[attrName];
+          }
+        }
+      }
+    }
+  }
+
+  // Note: For performance, we're merging all changes together rather than
+  // mark changes based the exact parameters affected.
+  for (let k in changeTrackers) {
+    if (changeTrackers[k] & changeFlags)
+      deviceData.changes.add(k);
+  }
+}
+
+
+function compareEquality(a, b) {
+  var t = typeof a;
+  if (a === null ||
+      a === undefined ||
+      t === 'number' ||
+      t === 'boolean' ||
+      t === 'string' ||
+      t === 'symbol')
+    return a === b;
+
+  return JSON.stringify(a) === JSON.stringify(b);
+}
+
+
+function set(deviceData, path, timestamp, attributes, toClear) {
+  path = deviceData.paths.add(path);
+
+  let currentTimestamp = deviceData.timestamps.get(path);
+
+  let currentAttributes;
+
+  if (path.wildcard)
+    attributes = undefined;
+  else if (currentTimestamp)
+    currentAttributes = deviceData.attributes.get(path);
+
+  let changeFlags = 0;
+
+  if (attributes) {
+    if (attributes.value && attributes.value[1]
+        && attributes.value[0] >= (attributes.object ? attributes.object[0] : 0))
+      attributes.object = [attributes.value[0], 0];
+
+    if (attributes.object && attributes.object[1]
+      && attributes.object[0] >= (attributes.value ? attributes.value[0] : 0))
+      attributes.value = [attributes.object[0]];
+
+    let newAttributes = Object.assign({}, currentAttributes, attributes);
+
+    if (currentAttributes) {
+      for (let attrName in attributes) {
+        timestamp = Math.max(timestamp, attributes[attrName][0]);
+        if (!(attrName in currentAttributes))
+          changeFlags |= CHANGE_FLAGS[attrName];
+        else if (attributes[attrName][0] <= currentAttributes[attrName][0])
+          newAttributes[attrName] = currentAttributes[attrName];
+        else if (!compareEquality(attributes[attrName][1], currentAttributes[attrName][1]))
+          changeFlags |= CHANGE_FLAGS[attrName];
+      }
+    }
+    else
+      changeFlags |= 1;
+
+    deviceData.attributes.set(path, newAttributes);
+
+    if (!(timestamp <= currentTimestamp)) {
+      deviceData.timestamps.set(path, timestamp);
+      if (path.length > 1)
+        toClear = set(deviceData, path.slice(0, path.length - 1), timestamp,
+          {object: [timestamp, 1]}, toClear);
+    }
+  }
+  else if (!(timestamp <= currentTimestamp)) {
+    deviceData.timestamps.set(path, timestamp);
+
+    if (currentAttributes) {
+      deviceData.attributes.delete(path);
+      changeFlags |= 1;
+    }
+    else if (path.wildcard) {
+      for (let p of deviceData.paths.find(path, false, true, path.length))
+        if (timestamp > deviceData.timestamps.get(p)) {
+          toClear = toClear || [];
+          toClear.push([p, timestamp]);
+        }
+    }
+  }
+
+  if (changeFlags) {
+    if (changeFlags & 1) {
+      toClear = toClear || [];
+      toClear.push([path, timestamp, null, changeFlags]);
+    }
+    else if (changeFlags & CHANGE_FLAGS.object) {
+      toClear = toClear || [];
+      toClear.push([path, 0, {object: attributes.object[0]}, changeFlags]);
+    }
+    else {
+      for (let p of deviceData.paths.find(path, true, false, path.length)) {
+        let tracker = deviceData.trackers.get(p);
+        for (let k in tracker) {
+          if (tracker[k] & changeFlags)
+            deviceData.changes.add(k);
+        }
+      }
+    }
+  }
+
+  return toClear;
+}
+
+
+function track(deviceData, path, marker, attributes) {
+  path = deviceData.paths.add(path);
+  let f = 1;
+
+  if (attributes)
+    for (let attrName of attributes)
+      f |= CHANGE_FLAGS[attrName];
+
+  let cur = deviceData.trackers.get(path);
+  if (!cur) {
+    cur = {};
+    deviceData.trackers.set(path, cur);
+  }
+
+  cur[marker] |= f;
+}
+
+
+function clearTrackers(deviceData, tracker) {
+  if (Array.isArray(tracker)) {
+    deviceData.trackers.forEach(function(v, k) {
+      for (let t of tracker)
+        delete v[t];
+    });
+    for (let t of tracker)
+      deviceData.changes.delete(t);
+  }
+  else {
+    deviceData.trackers.forEach(function(v, k) {
+      delete v[tracker];
+    });
+    deviceData.changes.delete(tracker);
+  }
+}
+
+
+exports.sanitizeParameterValue = sanitizeParameterValue;
+exports.getAliasDeclarations = getAliasDeclarations;
+exports.unpack = unpack;
+exports.set = set;
+exports.clear = clear;
+exports.track = track;
+exports.clearTrackers = clearTrackers;

lib/device.ts

@@ -1,416 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
-import {
-  DeviceData,
-  Declaration,
-  Clear,
-  AttributeTimestamps,
-  Attributes,
-} from "./types";
-
-const CHANGE_FLAGS = {
-  object: 2,
-  writable: 4,
-  value: 8,
-  notification: 16,
-  accessList: 32,
-};
-
-function parseBool(v): boolean {
-  v = "" + v;
-  if (v === "true" || v === "TRUE" || v === "True" || v === "1") return true;
-  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
-    return false;
-  else return null;
-}
-
-export function sanitizeParameterValue(
-  parameterValue: [string | number | boolean, string]
-): [string | number | boolean, string] {
-  if (parameterValue[0] != null) {
-    switch (parameterValue[1]) {
-      case "xsd:boolean":
-        if (typeof parameterValue[0] !== "boolean") {
-          const b = parseBool(parameterValue[0]);
-          if (b == null) parameterValue[0] = "" + parameterValue[0];
-          else parameterValue[0] = b;
-        }
-        break;
-      case "xsd:string":
-      case "xsd:base64":
-      case "xsd:hexBinary":
-        if (typeof parameterValue[0] !== "string")
-          parameterValue[0] = "" + parameterValue[0];
-
-        break;
-      case "xsd:int":
-      case "xsd:unsignedInt":
-        if (typeof parameterValue[0] !== "number") {
-          const i = parseInt(parameterValue[0] as string);
-          if (isNaN(i)) parameterValue[0] = "" + parameterValue[0];
-          else parameterValue[0] = i;
-        }
-        break;
-      case "xsd:dateTime":
-        if (typeof parameterValue[0] !== "number") {
-          // Don't use parseInt because it reads date string as a number
-          let i = +parameterValue[0];
-          if (isNaN(i)) {
-            i = Date.parse(parameterValue[0] as string);
-            if (isNaN(i)) parameterValue[0] = "" + parameterValue[0];
-            else parameterValue[0] = i;
-          } else {
-            parameterValue[0] = i;
-          }
-        }
-        break;
-      default:
-        parameterValue[0] = JSON.parse(JSON.stringify(parameterValue[0]));
-        break;
-    }
-  }
-
-  return parameterValue;
-}
-
-export function getAliasDeclarations(
-  path: Path,
-  timestamp: number,
-  attrGet = null
-): Declaration[] {
-  const stripped = path.stripAlias();
-  let decs: Declaration[] = [
-    {
-      path: stripped,
-      pathGet: timestamp,
-      pathSet: null,
-      attrGet: attrGet,
-      attrSet: null,
-      defer: true,
-    },
-  ];
-
-  if (path.alias) {
-    for (const [i, alias] of path.segments.entries()) {
-      if (Array.isArray(alias)) {
-        const parent = stripped.slice(0, i + 1);
-        for (const [p] of alias as [Path, string][]) {
-          decs = decs.concat(
-            getAliasDeclarations(parent.concat(p), timestamp, {
-              value: timestamp,
-            })
-          );
-        }
-      }
-    }
-  }
-
-  return decs;
-}
-
-export function unpack(
-  deviceData: DeviceData,
-  path: Path,
-  revision?: number
-): Path[] {
-  let allMatches = [] as Path[];
-  if (!path.alias) {
-    for (const p of deviceData.paths.find(path, false, true))
-      if (deviceData.attributes.has(p, revision)) allMatches.push(p);
-  } else {
-    const wildcardPath = path.stripAlias();
-
-    for (const p of deviceData.paths.find(wildcardPath, false, true))
-      if (deviceData.attributes.has(p, revision)) allMatches.push(p);
-
-    for (let i = path.length - 1; i >= 0; --i) {
-      if (path.alias & (1 << i)) {
-        for (const [param, val] of path.segments[i] as [Path, string][]) {
-          const p = wildcardPath.slice(0, i + 1).concat(param);
-          const unpacked = unpack(deviceData, p, revision);
-          const filtered: Path[] = [];
-          for (const up of unpacked) {
-            const attributes = deviceData.attributes.get(up, revision);
-            if (
-              attributes &&
-              attributes.value &&
-              attributes.value[1] &&
-              sanitizeParameterValue([val, attributes.value[1][1]])[0] ===
-                attributes.value[1][0]
-            ) {
-              for (let m = 0; m < allMatches.length; ++m) {
-                let k;
-                const match = allMatches[m];
-                if (!match) continue;
-                for (k = i; k >= 0; --k)
-                  if (match.segments[k] !== up.segments[k]) break;
-
-                if (k < 0) {
-                  filtered.push(match);
-                  allMatches[m] = null;
-                }
-              }
-            }
-          }
-          allMatches = filtered;
-        }
-      }
-    }
-  }
-
-  allMatches.sort((p1, p2) => {
-    for (let i = 0; i < p1.length; ++i) {
-      const a = p1.segments[i] as string;
-      const b = p2.segments[i] as string;
-      if (a !== b) {
-        // Use numeric sorting for numbers
-        const ia = parseInt(a);
-        const ib = parseInt(b);
-
-        if (ia === +a && ib === +b) return ia - ib;
-        else if (a < b) return -1;
-        else return 1;
-      }
-    }
-    return 0;
-  });
-
-  return allMatches;
-}
-
-export function clear(
-  deviceData: DeviceData,
-  path: Path,
-  timestamp: number,
-  attributes: AttributeTimestamps,
-  changeFlags = 0
-): void {
-  const changeTrackers = {};
-
-  timestamp = timestamp || 0;
-
-  let descendantsTimestamp = timestamp;
-  if (attributes && attributes.object) {
-    if (attributes.object > descendantsTimestamp)
-      descendantsTimestamp = attributes.object;
-    if (!(attributes.object <= attributes.value))
-      attributes.value = attributes.object;
-  }
-
-  for (const p of deviceData.paths.find(
-    path,
-    true,
-    true,
-    descendantsTimestamp ? 99 : path.length
-  )) {
-    const tracker = deviceData.trackers.get(p);
-    for (const k in tracker) changeTrackers[k] |= tracker[k];
-
-    const currentTimestamp = deviceData.timestamps.get(p);
-    if (currentTimestamp === undefined) continue;
-
-    if (
-      timestamp > currentTimestamp ||
-      (descendantsTimestamp > currentTimestamp && p.length > path.length)
-    ) {
-      deviceData.timestamps.delete(p);
-      deviceData.attributes.delete(p);
-      changeFlags |= 1;
-    } else if (attributes && p.length === path.length) {
-      const currentAttributes = deviceData.attributes.get(p);
-      if (currentAttributes) {
-        let newAttrs;
-        for (const attrName in attributes) {
-          if (
-            attrName in currentAttributes &&
-            attributes[attrName] > currentAttributes[attrName][0]
-          ) {
-            changeFlags |= CHANGE_FLAGS[attrName];
-            if (!newAttrs) {
-              newAttrs = Object.assign({}, currentAttributes);
-              deviceData.attributes.set(p, newAttrs);
-            }
-            delete newAttrs[attrName];
-          }
-        }
-      }
-    }
-  }
-
-  // Note: For performance, we're merging all changes together rather than
-  // mark changes based the exact parameters affected.
-  for (const k in changeTrackers)
-    if (changeTrackers[k] & changeFlags) deviceData.changes.add(k);
-}
-
-function compareEquality(a, b): boolean {
-  const t = typeof a;
-  if (
-    a === null ||
-    a === undefined ||
-    t === "number" ||
-    t === "boolean" ||
-    t === "string" ||
-    t === "symbol"
-  )
-    return a === b;
-
-  return JSON.stringify(a) === JSON.stringify(b);
-}
-
-export function set(
-  deviceData: DeviceData,
-  path: Path,
-  timestamp: number,
-  attributes: Attributes,
-  toClear?: Clear[]
-): Clear[] {
-  path = deviceData.paths.add(path);
-
-  const currentTimestamp = deviceData.timestamps.get(path);
-
-  let currentAttributes;
-
-  if (path.wildcard) attributes = undefined;
-  else if (currentTimestamp)
-    currentAttributes = deviceData.attributes.get(path);
-
-  let changeFlags = 0;
-
-  if (attributes) {
-    if (
-      attributes.value &&
-      attributes.value[1] &&
-      attributes.value[0] >= (attributes.object ? attributes.object[0] : 0)
-    )
-      attributes.object = [attributes.value[0], 0];
-
-    if (
-      attributes.object &&
-      attributes.object[1] &&
-      attributes.object[0] >= (attributes.value ? attributes.value[0] : 0)
-    )
-      attributes.value = [attributes.object[0], null];
-
-    const newAttributes = Object.assign({}, currentAttributes, attributes);
-
-    if (currentAttributes) {
-      for (const attrName in attributes) {
-        timestamp = Math.max(timestamp, attributes[attrName][0]);
-        if (!(attrName in currentAttributes))
-          changeFlags |= CHANGE_FLAGS[attrName];
-        else if (attributes[attrName][0] <= currentAttributes[attrName][0])
-          newAttributes[attrName] = currentAttributes[attrName];
-        else if (
-          !compareEquality(
-            attributes[attrName][1],
-            currentAttributes[attrName][1]
-          )
-        )
-          changeFlags |= CHANGE_FLAGS[attrName];
-      }
-    } else {
-      changeFlags |= 1;
-    }
-
-    deviceData.attributes.set(path, newAttributes);
-
-    if (!(timestamp <= currentTimestamp)) {
-      deviceData.timestamps.set(path, timestamp);
-      if (path.length > 1) {
-        toClear = set(
-          deviceData,
-          path.slice(0, path.length - 1),
-          timestamp,
-          { object: [timestamp, 1] },
-          toClear
-        );
-      }
-    }
-  } else if (!(timestamp <= currentTimestamp)) {
-    deviceData.timestamps.set(path, timestamp);
-
-    if (currentAttributes) {
-      deviceData.attributes.delete(path);
-      changeFlags |= 1;
-    } else if (path.wildcard) {
-      for (const p of deviceData.paths.find(path, false, true, path.length)) {
-        if (timestamp > deviceData.timestamps.get(p)) {
-          toClear = toClear || [];
-          toClear.push([p, timestamp]);
-        }
-      }
-    }
-  }
-
-  if (changeFlags) {
-    if (changeFlags & 1) {
-      toClear = toClear || [];
-      toClear.push([path, timestamp, null, changeFlags]);
-    } else if (changeFlags & CHANGE_FLAGS.object) {
-      toClear = toClear || [];
-      toClear.push([path, 0, { object: attributes.object[0] }, changeFlags]);
-    } else {
-      for (const p of deviceData.paths.find(path, true, false, path.length)) {
-        const tracker = deviceData.trackers.get(p);
-        for (const k in tracker)
-          if (tracker[k] & changeFlags) deviceData.changes.add(k);
-      }
-    }
-  }
-
-  return toClear;
-}
-
-export function track(
-  deviceData: DeviceData,
-  path: Path,
-  marker: string,
-  attributes?: string[]
-): void {
-  path = deviceData.paths.add(path);
-  let f = 1;
-
-  if (attributes)
-    for (const attrName of attributes) f |= CHANGE_FLAGS[attrName];
-
-  let cur = deviceData.trackers.get(path);
-  if (!cur) {
-    cur = {};
-    deviceData.trackers.set(path, cur);
-  }
-
-  cur[marker] |= f;
-}
-
-export function clearTrackers(
-  deviceData: DeviceData,
-  tracker: string | string[]
-): void {
-  if (Array.isArray(tracker)) {
-    for (const v of deviceData.trackers.values())
-      for (const t of tracker) delete v[t];
-    for (const t of tracker) deviceData.changes.delete(t);
-  } else {
-    for (const v of deviceData.trackers.values()) delete v[tracker];
-    deviceData.changes.delete(tracker);
-  }
-}

lib/extension-wrapper.js

@@ -1,5 +1,5 @@
 /**
- * Copyright 2013-2019  GenieACS Inc.
+ * Copyright 2013-2017  Zaid Abdulla
  *
  * This file is part of GenieACS.
  *
@@ -16,58 +16,53 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
-
-import { Fault } from "../lib/types";
+"use strict";
 
 const jobs = new Set();
 const fileName = process.argv[2];
 let script;
 
-function errorToFault(err: Error): Fault {
-  if (!err) return null;
+function errorToFault(err) {
+  if (!err)
+    return null;
 
-  if (!err.name) return { code: "ext", message: `${err}` };
+  if (!err.name)
+    return {code: "ext", message: `${err}`};
 
-  const fault: Fault = {
+  let fault = {
     code: `ext.${err.name}`,
     message: err.message,
     detail: {
       name: err.name,
-      message: err.message,
-    },
+      message: err.message
+    }
   };
 
   if (err.stack) {
-    fault.detail["stack"] = err.stack;
+    fault.detail.stack = err.stack;
     // Trim the stack trace
-    const stackTrimIndex = fault.detail["stack"].match(
-      /\s+at\s[^\s]+\s\(.*genieacs-ext:.+\)/
-    );
-    if (stackTrimIndex) {
-      fault.detail["stack"] = fault.detail["stack"].slice(
-        0,
-        stackTrimIndex.index
-      );
-    }
+    let stackTrimIndex = fault.detail.stack.match(/\s+at\s[^\s]+\s\(.*extension-wrapper\.js:.+\)/);
+    if (stackTrimIndex)
+      fault.detail.stack = fault.detail.stack.slice(0, stackTrimIndex.index);
   }
 
   return fault;
 }
 
-process.on("uncaughtException", (err) => {
-  const fault = errorToFault(err);
-  jobs.forEach((jobId) => {
-    process.send([jobId, fault, null]);
+process.on("uncaughtException", function(err) {
+  let fault = errorToFault(err);
+  jobs.forEach(function(jobId) {
+    process.send([jobId, fault]);
   });
   jobs.clear();
   process.disconnect();
 });
 
-process.on("message", (message) => {
+process.on("message", function(message) {
   jobs.add(message[0]);
 
   if (!script) {
-    const cwd = process.env["GENIEACS_EXT_DIR"];
+    let cwd = process.env["GENIEACS_CONFIG_DIR"] + "/ext";
     process.chdir(cwd);
     script = require(`${cwd}/${fileName}`);
   }
@@ -75,22 +70,21 @@ process.on("message", (message) => {
   const funcName = message[1][0];
 
   if (!script[funcName]) {
-    const fault = {
+    let fault = {
       code: "ext",
-      message: `No such function '${funcName}' in extension '${fileName}'`,
+      message: `No such function '${funcName}' in extension '${fileName}'`
     };
-    process.send([message[0], fault, null]);
+    process.send([message[0], fault]);
     return;
   }
 
-  script[funcName](message[1].slice(1), (err, res) => {
-    if (!jobs.delete(message[0])) return;
+  script[funcName](message[1].slice(1), function(err, res) {
+    if (!jobs.delete(message[0]))
+      return;
 
     process.send([message[0], errorToFault(err), res]);
   });
 });
 
 // Ignore SIGINT
-process.on("SIGINT", () => {
-  // Ignore
-});
+process.on('SIGINT', function() {});

lib/extensions.js

@@ -0,0 +1,117 @@
+/**
+ * Copyright 2013-2017  Zaid Abdulla
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+"use strict";
+
+const childProcess = require("child_process");
+const crypto = require ("crypto");
+
+const config = require("./config");
+
+const TIMEOUT = config.get('EXT_TIMEOUT');
+
+const processes = {};
+const jobs = {};
+
+function messageHandler(message) {
+  let func = jobs[message[0]];
+  if (func) {
+    delete jobs[message[0]];
+    func(null, message[1], message[2]);
+  }
+}
+
+
+function run(args, callback) {
+  let scriptName = args[0];
+
+  if (!processes[scriptName]) {
+    let p = childProcess.fork('lib/extension-wrapper', [scriptName]);
+    processes[scriptName] = p;
+
+    p.on('error', function() {
+      if (processes[scriptName] === p) {
+        kill(processes[scriptName]);
+        delete processes[scriptName];
+      }
+    });
+
+    p.on('disconnect', function() {
+      if (processes[scriptName] === p) {
+        kill(processes[scriptName]);
+        delete processes[scriptName];
+      }
+    });
+
+    p.on('message', messageHandler);
+  }
+
+  let id = crypto.randomBytes(8).toString('hex');
+  jobs[id] = callback;
+  setTimeout(function() {
+    if (id in jobs) {
+      delete jobs[id];
+      return callback(null, {code: "timeout", message: "Extension timed out"});
+    }
+  }, TIMEOUT);
+
+  return processes[scriptName].send([id, args.slice(1)]);
+};
+
+
+function kill(process, callback) {
+  if (process.signalCode !== null || process.exitCode !== null)
+    return;
+
+  let timeToKill = Date.now() + 5000;
+
+  process.kill();
+
+  let t = setInterval(function() {
+    if (process.signalCode !== null || process.exitCode !== null) {
+      clearInterval(t);
+      if (callback)
+        callback();
+    }
+    else if (Date.now() > timeToKill) {
+      process.kill('SIGKILL');
+      clearInterval(t);
+      if (callback)
+        callback();
+    }
+  }, 100);
+}
+
+
+function killAll(callback) {
+  let counter = 1;
+  for (let n in processes) {
+    ++counter;
+    kill(processes[n], function() {
+      if (--counter == 0 && callback)
+        callback();
+    });
+    delete processes[n];
+  }
+  if (--counter == 0 && callback)
+    callback();
+}
+
+
+exports.run = run;
+exports.killAll = killAll;

lib/extensions.ts

@@ -1,128 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { spawn, ChildProcess } from "child_process";
-import * as crypto from "crypto";
-import * as config from "./config";
-import { Fault } from "./types";
-import { ROOT_DIR } from "./config";
-import * as logger from "./logger";
-import readline from "readline";
-
-const TIMEOUT = +config.get("EXT_TIMEOUT");
-
-const processes: { [script: string]: ChildProcess } = {};
-const jobs = new Map();
-
-export function run(args: string[]): Promise<{ fault: Fault; value: any }> {
-  return new Promise((resolve) => {
-    const scriptName = args[0];
-
-    const id = crypto.randomBytes(8).toString("hex");
-    jobs.set(id, resolve);
-
-    if (!processes[scriptName]) {
-      const p = spawn(ROOT_DIR + "/bin/genieacs-ext", [scriptName], {
-        stdio: ["ignore", "pipe", "pipe", "ipc"],
-      });
-      processes[scriptName] = p;
-
-      p.on("error", (err) => {
-        if (processes[scriptName] === p) {
-          if (jobs.delete(id)) {
-            resolve({
-              fault: { code: err.name, message: err.message },
-              value: null,
-            });
-          }
-
-          // eslint-disable-next-line @typescript-eslint/no-floating-promises
-          kill(processes[scriptName]);
-          delete processes[scriptName];
-        }
-      });
-
-      p.on("disconnect", () => {
-        if (processes[scriptName] === p) delete processes[scriptName];
-      });
-
-      p.on("message", (message) => {
-        const func = jobs.get(message[0]);
-        if (func) {
-          jobs.delete(message[0]);
-          // Wait for any disconnect even to fire
-          setTimeout(() => {
-            func({ fault: message[1], value: message[2] });
-          });
-        }
-      });
-
-      const rlstdout = readline.createInterface(p.stdout);
-      rlstdout.on("line", (line) => {
-        logger.info({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
-      });
-
-      const rlstderr = readline.createInterface(p.stderr);
-      rlstderr.on("line", (line) => {
-        logger.warn({ message: `Ext ${scriptName}(${p.pid}): ${line}` });
-      });
-    }
-
-    setTimeout(() => {
-      if (jobs.delete(id)) {
-        resolve({
-          fault: { code: "timeout", message: "Extension timed out" },
-          value: null,
-        });
-      }
-    }, TIMEOUT);
-
-    if (!processes[scriptName].connected) return false;
-
-    return processes[scriptName].send([id, args.slice(1)]);
-  });
-}
-
-function kill(process: ChildProcess): Promise<void> {
-  return new Promise((resolve) => {
-    const timeToKill = Date.now() + 5000;
-
-    process.kill();
-
-    const t = setInterval(() => {
-      if (!process.connected) {
-        clearInterval(t);
-        resolve();
-      } else if (Date.now() > timeToKill) {
-        process.kill("SIGKILL");
-        clearInterval(t);
-        resolve();
-      }
-    }, 100);
-  });
-}
-
-export async function killAll(): Promise<void> {
-  await Promise.all(
-    Object.entries(processes).map(([k, p]) => {
-      delete processes[k];
-      return kill(p);
-    })
-  );
-}

lib/forwarded.ts

@@ -1,175 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { IncomingMessage } from "http";
-import { TLSSocket } from "tls";
-import { parseCIDR, parse, IPv6, IPv4 } from "ipaddr.js";
-import * as config from "./config";
-
-interface RequestOrigin {
-  localAddress: string;
-  localPort: number;
-  remoteAddress: string;
-  remotePort: number;
-  host: string;
-  encrypted: boolean;
-}
-
-const FORWARDED_HEADER = "" + config.get("FORWARDED_HEADER");
-const cache = new WeakMap<IncomingMessage, RequestOrigin>();
-const cidrs: [IPv4 | IPv6, number][] = [];
-
-for (const str of FORWARDED_HEADER.split(",").map((s) => s.trim())) {
-  try {
-    cidrs.push(parseCIDR(str));
-  } catch (err) {
-    // Not a valid CIDR format, try parsing as IP
-    try {
-      const ip = parse(str);
-      cidrs.push([ip, ip.toByteArray().length * 8]);
-    } catch (err) {
-      // Not a valid IP either, ignore
-    }
-  }
-}
-
-function parseForwardedHeader(str: string): { [name: string]: string } {
-  str = str.toLowerCase();
-  const res: { [name: string]: string } = {};
-  let keyIdx = 0;
-  let valueIdx = -1;
-  let key: string;
-  for (let i = 0; i < str.length; ++i) {
-    const char = str.charCodeAt(i);
-    if (char === 61 /* = */) {
-      if (keyIdx >= 0) {
-        key = str.slice(keyIdx, i).trim();
-        keyIdx = -1;
-        valueIdx = i + 1;
-      }
-    } else if (char === 59 /* ; */) {
-      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
-      valueIdx = -1;
-      keyIdx = i + 1;
-    } else if (char === 44 /* , */) {
-      if (valueIdx >= 0) res[key] = str.slice(valueIdx, i).trim();
-      return res;
-    } else if (char === 34 /* " */) {
-      if (valueIdx >= 0) {
-        const quoteIdx = i;
-        if (!str.slice(valueIdx, quoteIdx).trim()) {
-          for (i = i + 1; i < str.length; ++i) {
-            const c = str.charCodeAt(i);
-            if (c === 92 /* \ */) ++i;
-            if (c === 34 /* " */) {
-              res[key] = JSON.parse(str.slice(quoteIdx, i + 1).trim());
-              valueIdx = -1;
-              keyIdx = i + 1;
-              break;
-            }
-          }
-        }
-      }
-    }
-  }
-
-  if (valueIdx >= 0) res[key] = str.slice(valueIdx).trim();
-
-  return res;
-}
-
-export function getRequestOrigin(request: IncomingMessage): RequestOrigin {
-  let origin = cache.get(request);
-  if (!origin) {
-    const soc = request.socket;
-    origin = {
-      localAddress: soc.localAddress,
-      localPort: soc.localPort,
-      remoteAddress: soc.remoteAddress,
-      remotePort: soc.remotePort,
-      host: request.headers["host"],
-      encrypted: !!(request.socket as TLSSocket).encrypted,
-    };
-
-    const header = request.headers["forwarded"];
-    if (header) {
-      const ip = parse(soc.remoteAddress) as IPv4;
-      if (cidrs.some((cidr) => ip.match(cidr as [IPv4, number]))) {
-        const parsed = parseForwardedHeader(header);
-
-        if (parsed["proto"] === "https") {
-          origin.encrypted = true;
-          origin.localPort = 443;
-        } else if (parsed["proto"] === "http") {
-          origin.encrypted = false;
-          origin.localPort = 80;
-        }
-
-        if (parsed["host"]) {
-          origin.host = parsed["host"];
-          const [, port] = parsed["host"].split(":", 2);
-          origin.localPort = +port || origin.localPort;
-        }
-
-        if (parsed["for"]) {
-          if (parsed["for"].startsWith("[")) {
-            const i = parsed["for"].lastIndexOf("]");
-            if (i >= 0) {
-              origin.remoteAddress = parsed["for"].slice(1, i);
-              origin.remotePort =
-                parseInt(parsed["for"].slice(i + 2)) || origin.remotePort;
-            }
-          } else {
-            const i = parsed["for"].lastIndexOf(":");
-            if (i >= 0) {
-              origin.remoteAddress = parsed["for"].slice(0, i);
-              origin.remotePort =
-                parseInt(parsed["for"].slice(i + 1)) || origin.remotePort;
-            } else {
-              origin.remoteAddress = parsed["for"];
-            }
-          }
-        }
-
-        if (parsed["by"]) {
-          if (parsed["by"].startsWith("[")) {
-            const i = parsed["by"].lastIndexOf("]");
-            if (i >= 0) {
-              origin.localAddress = parsed["by"].slice(1, i);
-              origin.localPort =
-                parseInt(parsed["by"].slice(i + 2)) || origin.localPort;
-            }
-          } else {
-            const i = parsed["by"].lastIndexOf(":");
-            if (i >= 0) {
-              origin.localAddress = parsed["by"].slice(0, i);
-              origin.localPort =
-                parseInt(parsed["by"].slice(i + 1)) || origin.localPort;
-            } else {
-              origin.localAddress = parsed["by"];
-            }
-          }
-        }
-      }
-    }
-
-    cache.set(request, origin);
-  }
-  return origin;
-}

lib/fs.coffee

@@ -0,0 +1,49 @@
+###
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
+config = require './config'
+url = require 'url'
+mongodb = require 'mongodb'
+querystring = require 'querystring'
+db = require './db'
+
+
+listener = (request, response) ->
+  urlParts = url.parse(request.url, true)
+  if request.method == 'GET'
+    filename = querystring.unescape(urlParts.pathname.substring(1))
+    gs = new mongodb.GridStore(db.mongoDb, filename, 'r', {})
+    gs.open((err, gs) ->
+      if err
+        response.writeHead(404)
+        response.end()
+        return
+      stream = gs.stream(true)
+      response.writeHead(200, {'Content-Type' : 'application/octet-stream', 'Content-Length' : gs.length})
+      stream.pipe(response)
+    )
+  else
+    response.writeHead(405, {'Allow': 'GET'})
+    response.end('405 Method Not Allowed')
+
+
+exports.listener = listener

lib/fs.ts

@@ -1,94 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as url from "url";
-import * as querystring from "querystring";
-import { IncomingMessage, ServerResponse } from "http";
-import { Collection, GridFSBucket } from "mongodb";
-import { onConnect } from "./db";
-import * as logger from "./logger";
-import { getRequestOrigin } from "./forwarded";
-import memoize from "./common/memoize";
-
-let filesCollection: Collection;
-let filesBucket: GridFSBucket;
-
-onConnect(async (db) => {
-  filesCollection = db.collection("fs.files");
-  filesBucket = new GridFSBucket(db);
-});
-
-const getFile = memoize(
-  (md5: string, size: number, filename: string): Promise<Buffer> => {
-    return new Promise((resolve, reject) => {
-      const buffer = Buffer.allocUnsafe(size);
-      let i = 0;
-      const downloadStream = filesBucket.openDownloadStreamByName(filename);
-      downloadStream.on("error", reject);
-      downloadStream.on("data", (data: Buffer) => {
-        data.copy(buffer, i);
-        i += data.length;
-      });
-      downloadStream.on("end", () => {
-        if (i !== size) reject(new Error("File size mismatch"));
-        else resolve(buffer);
-      });
-    });
-  }
-);
-
-export async function listener(
-  request: IncomingMessage,
-  response: ServerResponse
-): Promise<void> {
-  if (request.method !== "GET") {
-    response.writeHead(405, { Allow: "GET" });
-    response.end("405 Method Not Allowed");
-    return;
-  }
-
-  const urlParts = url.parse(request.url, true);
-  const filename = querystring.unescape(urlParts.pathname.substring(1));
-
-  const log = {
-    message: "Fetch file",
-    filename: filename,
-    remoteAddress: getRequestOrigin(request).remoteAddress,
-  };
-
-  const file = await filesCollection.findOne({ _id: filename });
-
-  if (!file) {
-    response.writeHead(404);
-    response.end();
-    log.message += " not found";
-    logger.accessError(log);
-    return;
-  }
-
-  const buffer = await getFile(file.md5, file.length, filename);
-
-  response.writeHead(200, {
-    "Content-Type": file.contentType || "application/octet-stream",
-    "Content-Length": file.length,
-  });
-
-  response.end(buffer);
-  logger.accessInfo(log);
-}

lib/gpn-heuristic.js

@@ -1,7 +1,5 @@
-import Path from "./common/path";
-
 /**
- * Copyright 2013-2019  GenieACS Inc.
+ * Copyright 2013-2017  Zaid Abdulla
  *
  * This file is part of GenieACS.
  *
@@ -18,6 +16,7 @@ import Path from "./common/path";
  * You should have received a copy of the GNU Affero General Public License
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
+"use strict";
 
 const WILDCARD_MULTIPLIER = 2;
 const UNDISCOVERED_DEPTH = 7;
@@ -29,50 +28,48 @@ const UNDISCOVERED_DEPTH = 7;
 // flags is an int where its bits mark the segments in the pattern that
 // need refreshing. Leading 0s indicate that the pattern up to that
 // point has been discovered.
-export function estimateGpnCount(
-  gpnPatterns: [Path, number][],
-  depth = 0
-): number {
-  const children: { [segment: string]: [Path, number][] } = {};
-  const wildcardChildren: [Path, number][] = [];
+function estimateGpnCount(gpnPatterns, depth = 0) {
+  const children = {};
+  const wildcardChildren = [];
   let wildcardDiscovered = false;
   let gpnCount = 0;
 
-  for (const pattern of gpnPatterns) {
+  for (let pattern of gpnPatterns) {
     const path = pattern[0];
     const flags = pattern[1] >> depth;
 
-    const k = path.segments[depth] as string;
+    let k = path[depth];
 
     if (!k) {
-      if (flags & 1) gpnCount = 1;
+      if (flags & 1)
+        gpnCount = 1;
       continue;
     }
 
     if (flags & 1) {
       gpnCount = 1;
 
-      if (depth > UNDISCOVERED_DEPTH) continue;
-    } else if (k === "*") {
-      wildcardDiscovered = true;
+      if (depth > UNDISCOVERED_DEPTH)
+        continue;
     }
+    else if (k === "*")
+      wildcardDiscovered = true;
 
     if (k === "*") {
       wildcardChildren.push(pattern);
-    } else {
+    }
+    else {
       children[k] = children[k] || [];
       children[k].push(pattern);
     }
   }
 
   let wildcardGpnCount = 0;
-  if (!wildcardDiscovered && wildcardChildren.length) {
-    wildcardGpnCount +=
-      estimateGpnCount(wildcardChildren, depth + 1) * WILDCARD_MULTIPLIER;
-  }
+  if (!wildcardDiscovered && wildcardChildren.length)
+      wildcardGpnCount += estimateGpnCount(wildcardChildren, depth + 1) * WILDCARD_MULTIPLIER;
 
-  for (const k of Object.keys(children)) {
-    const c = estimateGpnCount(children[k].concat(wildcardChildren), depth + 1);
+  for (let k of Object.keys(children)) {
+    let c = estimateGpnCount(children[k].concat(wildcardChildren), depth + 1);
     wildcardGpnCount -= c;
     gpnCount += c;
   }
@@ -81,3 +78,5 @@ export function estimateGpnCount(
 
   return gpnCount;
 }
+
+exports.estimateGpnCount = estimateGpnCount;

lib/init.ts

@@ -1,426 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as localCache from "./local-cache";
-import { generateSalt, hashPassword } from "./auth";
-import * as db from "./ui/db";
-import { del } from "./cache";
-
-interface Status {
-  users: boolean;
-  presets: boolean;
-  filters: boolean;
-  device: boolean;
-  index: boolean;
-  overview: boolean;
-}
-
-const BOOTSTRAP_SCRIPT = `
-const now = Date.now();
-
-// Clear cached data model to force a refresh
-clear("Device", now);
-clear("InternetGatewayDevice", now);
-`.trim();
-
-const DEFAULT_SCRIPT = `
-const hourly = Date.now(3600000);
-
-// Refresh basic parameters hourly
-declare("InternetGatewayDevice.DeviceInfo.HardwareVersion", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.DeviceInfo.SoftwareVersion", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID", {path: hourly, value: hourly});
-// Don't refresh password field periodically because CPEs always report blank passowrds for security reasons
-declare("InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase", {path: hourly, value: 1});
-declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress", {path: hourly, value: hourly});
-declare("InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress", {path: hourly, value: hourly});
-`.trim();
-
-const INFORM_SCRIPT = `
-// Device ID as user name
-const username = declare("DeviceID.ID", {value: 1}).value[0]
-
-// Password will be fixed a given device because Math.random() is seeded with devcie ID by default.
-const password = Math.trunc(Math.random() * Number.MAX_SAFE_INTEGER).toString(36);
-
-const informInterval = 300;
-
-// Refresh values daily
-const daily = Date.now(86400000);
-
-// Unique inform offset per device for better load distribution
-const informTime = daily % 86400000;
-
-declare("InternetGatewayDevice.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
-declare("InternetGatewayDevice.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
-declare("InternetGatewayDevice.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
-declare("InternetGatewayDevice.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
-declare("InternetGatewayDevice.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
-
-declare("Device.ManagementServer.ConnectionRequestUsername", {value: daily}, {value: username});
-declare("Device.ManagementServer.ConnectionRequestPassword", {value: daily}, {value: password});
-declare("Device.ManagementServer.PeriodicInformEnable", {value: daily}, {value: true});
-declare("Device.ManagementServer.PeriodicInformInterval", {value: daily}, {value: informInterval});
-declare("Device.ManagementServer.PeriodicInformTime", {value: daily}, {value: informTime});
-`.trim();
-
-export async function getStatus(): Promise<Status> {
-  const configSnapshot = await localCache.getCurrentSnapshot();
-  const users = localCache.getUsers(configSnapshot);
-  const presets = localCache.getPresets(configSnapshot);
-  const ui = localCache.getUiConfig(configSnapshot);
-
-  return {
-    users: !Object.keys(users).length,
-    presets: !presets.length,
-    filters: !Object.keys(ui["filters"]).length,
-    device: !Object.keys(ui["device"]).length,
-    index: !Object.keys(ui["index"]).length,
-    overview: !Object.keys(ui["overview"]).length,
-  };
-}
-
-export async function seed(options: Record<string, boolean>): Promise<void> {
-  const resources = {};
-  const proms = [];
-
-  if (options.users) {
-    resources["permissions"] = [
-      { role: "admin", resource: "devices", access: 3, validate: "true" },
-      { role: "admin", resource: "faults", access: 3, validate: "true" },
-      { role: "admin", resource: "files", access: 3, validate: "true" },
-      { role: "admin", resource: "presets", access: 3, validate: "true" },
-      { role: "admin", resource: "provisions", access: 3, validate: "true" },
-      { role: "admin", resource: "config", access: 3, validate: "true" },
-      { role: "admin", resource: "permissions", access: 3, validate: "true" },
-      { role: "admin", resource: "users", access: 3, validate: "true" },
-      {
-        role: "admin",
-        resource: "virtualParameters",
-        access: 3,
-        validate: "true",
-      },
-    ];
-
-    resources["users"] = [
-      { username: "admin", password: "admin", roles: ["admin"] },
-    ];
-  }
-
-  if (options.filters) {
-    resources["config"] = (resources["config"] || []).concat([
-      { _id: "ui.filters.0.label", value: "'Serial number'" },
-      { _id: "ui.filters.0.parameter", value: "DeviceID.SerialNumber" },
-      { _id: "ui.filters.0.type", value: "'string'" },
-      { _id: "ui.filters.1.label", value: "'Product class'" },
-      { _id: "ui.filters.1.parameter", value: "DeviceID.ProductClass" },
-      { _id: "ui.filters.1.type", value: "'string'" },
-      { _id: "ui.filters.2.label", value: "'Tag'" },
-      { _id: "ui.filters.2.type", value: "'tag'" },
-    ]);
-  }
-
-  if (options.device) {
-    resources["config"] = (resources["config"] || []).concat([
-      { _id: "ui.device.0.type", value: "'tags'" },
-      { _id: "ui.device.1.type", value: "'ping'" },
-      { _id: "ui.device.2.type", value: "'parameter-list'" },
-      { _id: "ui.device.2.parameters.0.type", value: "'container'" },
-      { _id: "ui.device.2.parameters.0.element", value: "'span.inform'" },
-      { _id: "ui.device.2.parameters.0.label", value: "'Last inform'" },
-      {
-        _id: "ui.device.2.parameters.0.components.0.type",
-        value: "'parameter'",
-      },
-      { _id: "ui.device.2.parameters.0.components.1.chart", value: "'online'" },
-      {
-        _id: "ui.device.2.parameters.0.components.1.type",
-        value: "'overview-dot'",
-      },
-      {
-        _id: "ui.device.2.parameters.0.components.2.type",
-        value: "'summon-button'",
-      },
-      {
-        _id: "ui.device.2.parameters.0.components.2.parameters.0",
-        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
-      },
-      {
-        _id: "ui.device.2.parameters.0.components.2.parameters.1",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
-      },
-      {
-        _id: "ui.device.2.parameters.0.components.2.parameters.2",
-        value:
-          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.MACAddress",
-      },
-      {
-        _id: "ui.device.2.parameters.0.components.2.parameters.3",
-        value:
-          "InternetGatewayDevice.WANDevice.*.WANConnectionDevice.*.WANIPConnection.*.ExternalIPAddress",
-      },
-      {
-        _id: "ui.device.2.parameters.0.components.2.parameters.4",
-        value: "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.SSID",
-      },
-      {
-        _id: "ui.device.2.parameters.0.components.2.parameters.5",
-        value:
-          "InternetGatewayDevice.LANDevice.*.WLANConfiguration.*.KeyPassphrase",
-      },
-      {
-        _id: "ui.device.2.parameters.0.components.2.parameters.6",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.HostName",
-      },
-      {
-        _id: "ui.device.2.parameters.0.components.2.parameters.7",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.IPAddress",
-      },
-      {
-        _id: "ui.device.2.parameters.0.components.2.parameters.8",
-        value: "InternetGatewayDevice.LANDevice.*.Hosts.Host.*.MACAddress",
-      },
-      {
-        _id: "ui.device.2.parameters.0.parameter",
-        value: "DATE_STRING(Events.Inform)",
-      },
-      { _id: "ui.device.2.parameters.1.label", value: "'Serial number'" },
-      {
-        _id: "ui.device.2.parameters.1.parameter",
-        value: "DeviceID.SerialNumber",
-      },
-      { _id: "ui.device.2.parameters.2.label", value: "'Product class'" },
-      {
-        _id: "ui.device.2.parameters.2.parameter",
-        value: "DeviceID.ProductClass",
-      },
-      { _id: "ui.device.2.parameters.3.label", value: "'OUI'" },
-      {
-        _id: "ui.device.2.parameters.3.parameter",
-        value: "DeviceID.OUI",
-      },
-      { _id: "ui.device.2.parameters.4.label", value: "'Manufacturer'" },
-      {
-        _id: "ui.device.2.parameters.4.parameter",
-        value: "DeviceID.Manufacturer",
-      },
-      { _id: "ui.device.2.parameters.5.label", value: "'Hardware version'" },
-      {
-        _id: "ui.device.2.parameters.5.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.HardwareVersion",
-      },
-      { _id: "ui.device.2.parameters.6.label", value: "'Software version'" },
-      {
-        _id: "ui.device.2.parameters.6.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
-      },
-      { _id: "ui.device.2.parameters.7.label", value: "'MAC'" },
-      {
-        _id: "ui.device.2.parameters.7.parameter",
-        value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.MACAddress",
-      },
-      { _id: "ui.device.2.parameters.8.label", value: "'IP'" },
-      {
-        _id: "ui.device.2.parameters.8.parameter",
-        value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
-      },
-      { _id: "ui.device.2.parameters.9.label", value: "'WLAN SSID'" },
-      {
-        _id: "ui.device.2.parameters.9.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
-      },
-      { _id: "ui.device.2.parameters.10.label", value: "'WLAN passphrase'" },
-      {
-        _id: "ui.device.2.parameters.10.parameter",
-        value:
-          "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.KeyPassphrase",
-      },
-      { _id: "ui.device.3.type", value: "'parameter-table'" },
-      { _id: "ui.device.3.label", value: "'WLAN hosts'" },
-      {
-        _id: "ui.device.3.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.Hosts.Host",
-      },
-      { _id: "ui.device.3.childParameters.0.label", value: "'Host name'" },
-      { _id: "ui.device.3.childParameters.0.parameter", value: "HostName" },
-      { _id: "ui.device.3.childParameters.1.label", value: "'IP address'" },
-      { _id: "ui.device.3.childParameters.1.parameter", value: "IPAddress" },
-      { _id: "ui.device.3.childParameters.2.label", value: "'MAC address'" },
-      { _id: "ui.device.3.childParameters.2.parameter", value: "MACAddress" },
-      { _id: "ui.device.4.type", value: "'container'" },
-      { _id: "ui.device.4.element", value: "'div'" },
-      { _id: "ui.device.4.components.0.components.0", value: "'Faults'" },
-      { _id: "ui.device.4.components.0.element", value: "'h3'" },
-      { _id: "ui.device.4.components.0.type", value: "'container'" },
-      { _id: "ui.device.4.components.1.type", value: "'device-faults'" },
-      { _id: "ui.device.5.type", value: "'container'" },
-      { _id: "ui.device.5.element", value: "'div.container-full-width'" },
-      {
-        _id: "ui.device.5.components.0.components.0",
-        value: "'All parameters'",
-      },
-      { _id: "ui.device.5.components.0.element", value: "'h3'" },
-      { _id: "ui.device.5.components.0.type", value: "'container'" },
-      { _id: "ui.device.5.components.1.type", value: "'all-parameters'" },
-      { _id: "ui.device.6.type", value: "'device-actions'" },
-    ]);
-  }
-
-  if (options.index) {
-    resources["config"] = (resources["config"] || []).concat([
-      { _id: "ui.index.0.type", value: "'device-link'" },
-      { _id: "ui.index.0.label", value: "'Serial number'" },
-      { _id: "ui.index.0.parameter", value: "DeviceID.SerialNumber" },
-      { _id: "ui.index.0.components.0.type", value: "'parameter'" },
-      { _id: "ui.index.1.label", value: "'Product class'" },
-      { _id: "ui.index.1.parameter", value: "DeviceID.ProductClass" },
-      { _id: "ui.index.2.label", value: "'Software version'" },
-      {
-        _id: "ui.index.2.parameter",
-        value: "InternetGatewayDevice.DeviceInfo.SoftwareVersion",
-      },
-      { _id: "ui.index.3.label", value: "'IP'" },
-      {
-        _id: "ui.index.3.parameter",
-        value:
-          "InternetGatewayDevice.WANDevice.1.WANConnectionDevice.1.WANIPConnection.1.ExternalIPAddress",
-      },
-      { _id: "ui.index.4.label", value: "'SSID'" },
-      {
-        _id: "ui.index.4.parameter",
-        value: "InternetGatewayDevice.LANDevice.1.WLANConfiguration.1.SSID",
-      },
-      { _id: "ui.index.5.type", value: "'container'" },
-      { _id: "ui.index.5.label", value: "'Last inform'" },
-      { _id: "ui.index.5.element", value: "'span.inform'" },
-      { _id: "ui.index.5.parameter", value: "DATE_STRING(Events.Inform)" },
-      { _id: "ui.index.5.components.0.type", value: "'parameter'" },
-      { _id: "ui.index.5.components.1.chart", value: "'online'" },
-      { _id: "ui.index.5.components.1.type", value: "'overview-dot'" },
-      { _id: "ui.index.6.type", value: "'tags'" },
-      { _id: "ui.index.6.label", value: "'Tags'" },
-      { _id: "ui.index.6.parameter", value: "Tags" },
-      { _id: "ui.index.6.unsortable", value: "true" },
-      { _id: "ui.index.6.writable", value: "false" },
-    ]);
-  }
-
-  if (options.overview) {
-    resources["config"] = (resources["config"] || []).concat([
-      { _id: "ui.overview.charts.online.label", value: "'Online status'" },
-      {
-        _id: "ui.overview.charts.online.slices.1_onlineNow.color",
-        value: "'#31a354'",
-      },
-      {
-        _id: "ui.overview.charts.online.slices.1_onlineNow.filter",
-        value: "Events.Inform > NOW() - 5 * 60 * 1000",
-      },
-      {
-        _id: "ui.overview.charts.online.slices.1_onlineNow.label",
-        value: "'Online now'",
-      },
-      {
-        _id: "ui.overview.charts.online.slices.2_past24.color",
-        value: "'#a1d99b'",
-      },
-      {
-        _id: "ui.overview.charts.online.slices.2_past24.filter",
-        value:
-          "Events.Inform > (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000) AND Events.Inform < (NOW() - 5 * 60 * 1000)",
-      },
-      {
-        _id: "ui.overview.charts.online.slices.2_past24.label",
-        value: "'Past 24 hours'",
-      },
-      {
-        _id: "ui.overview.charts.online.slices.3_others.color",
-        value: "'#e5f5e0'",
-      },
-      {
-        _id: "ui.overview.charts.online.slices.3_others.filter",
-        value:
-          "Events.Inform < (NOW() - 5 * 60 * 1000) - (24 * 60 * 60 * 1000)",
-      },
-      {
-        _id: "ui.overview.charts.online.slices.3_others.label",
-        value: "'Others'",
-      },
-      { _id: "ui.overview.groups.online.label", value: "''" },
-      { _id: "ui.overview.groups.online.charts.0", value: "'online'" },
-    ]);
-  }
-
-  if (options.presets) {
-    resources["presets"] = [
-      {
-        _id: "bootstrap",
-        weight: 0,
-        channel: "bootstrap",
-        events: "0 BOOTSTRAP",
-        provision: "bootstrap",
-      },
-      { _id: "default", weight: 0, channel: "default", provision: "default" },
-      { _id: "inform", weight: 0, channel: "inform", provision: "inform" },
-    ];
-
-    resources["provisions"] = [
-      { _id: "bootstrap", script: BOOTSTRAP_SCRIPT },
-      { _id: "default", script: DEFAULT_SCRIPT },
-      { _id: "inform", script: INFORM_SCRIPT },
-    ];
-  }
-
-  if (resources["permissions"]) {
-    for (const p of resources["permissions"]) {
-      p["_id"] = `${p["role"]}:${p["resource"]}:${p["access"]}`;
-      proms.push(db.putPermission(p["_id"], p));
-    }
-  }
-
-  if (resources["users"]) {
-    for (const u of resources["users"]) {
-      u["salt"] = await generateSalt(64);
-      u["password"] = await hashPassword(u["password"], u["salt"]);
-      u["roles"] = (u["roles"] || []).join(",");
-      u["_id"] = u["username"];
-      delete u["username"];
-      proms.push(db.putUser(u["_id"], u));
-    }
-  }
-
-  if (resources["provisions"]) {
-    for (const p of resources["provisions"])
-      proms.push(db.putProvision(p["_id"], p));
-  }
-
-  if (resources["presets"])
-    for (const p of resources["presets"]) proms.push(db.putPreset(p["_id"], p));
-
-  if (resources["config"])
-    for (const c of resources["config"]) proms.push(db.putConfig(c["_id"], c));
-
-  await proms;
-  return del("presets_hash");
-}

lib/instance-set.js

@@ -0,0 +1,138 @@
+/**
+ * Copyright 2013-2017  Zaid Abdulla
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+"use strict";
+
+
+class InstanceSet {
+
+  constructor() {
+    this.set = new Set();
+  }
+
+  add(instance) {
+    this.set.add(instance);
+  }
+
+  delete(instance) {
+    this.set.delete(instance);
+  }
+
+  superset(instance) {
+    let res = [];
+    for (let inst of this.set) {
+      let match = true;
+      for (let k in instance) {
+        if (inst[k] !== instance[k]) {
+          match = false;
+          break;
+        }
+      }
+
+      if (match)
+        res.push(inst);
+    }
+
+    res.sort(function(a, b) {
+      let keysA = Object.keys(a);
+      let keysB = Object.keys(b);
+
+      if (keysA.length != keysB.length)
+        return keysB.length - keysA.length;
+
+      keysA.sort();
+      keysB.sort();
+
+      for (let i = 0; i < keysA.length; ++ i) {
+        if (keysA[i] > keysB[i])
+          return 1;
+        else if (keysA[i] < keysB[i])
+          return -1;
+        else if (a[keysA[i]] > b[keysB[i]])
+          return 1;
+        else if (a[keysA[i]] < b[keysB[i]])
+          return -1;
+      }
+
+      return 0;
+    });
+
+    return res;
+  }
+
+  subset(instance) {
+    let res = [];
+
+    for (let inst of this.set) {
+      let match = true;
+      for (let k in inst) {
+        if (inst[k] !== instance[k]) {
+          match = false;
+          break;
+        }
+      }
+
+      if (match)
+        res.push(inst);
+    }
+
+    res.sort(function(a, b) {
+      let keysA = Object.keys(a);
+      let keysB = Object.keys(b);
+
+      if (keysA.length != keysB.length)
+        return keysA.length - keysB.length;
+
+      keysA.sort();
+      keysB.sort();
+
+      for (let i = 0; i < keysA.length; ++ i) {
+        if (keysA[i] > keysB[i])
+          return 1;
+        else if (keysA[i] < keysB[i])
+          return -1;
+        else if (a[keysA[i]] > b[keysB[i]])
+          return 1;
+        else if (a[keysA[i]] < b[keysB[i]])
+          return -1;
+      }
+
+      return 0;
+    });
+
+    return res;
+  }
+
+  forEach(callback) {
+    return this.set.forEach(callback);
+  }
+
+  values() {
+    return this.set.values();
+  }
+
+  clear() {
+    this.set.clear();
+  }
+
+  get size() {
+    return this.set.size;
+  }
+}
+
+module.exports = InstanceSet;

lib/instance-set.ts

@@ -1,131 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-interface Instance {
-  [name: string]: string;
-}
-
-export default class InstanceSet {
-  private set: Set<Instance>;
-
-  public constructor() {
-    this.set = new Set();
-  }
-
-  public add(instance: Instance): void {
-    this.set.add(instance);
-  }
-
-  public delete(instance: Instance): void {
-    this.set.delete(instance);
-  }
-
-  public superset(instance: Instance): Instance[] {
-    const res = [];
-    for (const inst of this.set) {
-      let match = true;
-      for (const k in instance) {
-        if (inst[k] !== instance[k]) {
-          match = false;
-          break;
-        }
-      }
-
-      if (match) res.push(inst);
-    }
-
-    res.sort((a, b) => {
-      const keysA = Object.keys(a);
-      const keysB = Object.keys(b);
-
-      if (keysA.length !== keysB.length) return keysB.length - keysA.length;
-
-      keysA.sort();
-      keysB.sort();
-
-      for (let i = 0; i < keysA.length; ++i) {
-        if (keysA[i] > keysB[i]) return 1;
-        else if (keysA[i] < keysB[i]) return -1;
-        else if (a[keysA[i]] > b[keysB[i]]) return 1;
-        else if (a[keysA[i]] < b[keysB[i]]) return -1;
-      }
-
-      return 0;
-    });
-
-    return res;
-  }
-
-  public subset(instance: Instance): Instance[] {
-    const res = [];
-
-    for (const inst of this.set) {
-      let match = true;
-      for (const k in inst) {
-        if (inst[k] !== instance[k]) {
-          match = false;
-          break;
-        }
-      }
-
-      if (match) res.push(inst);
-    }
-
-    res.sort((a, b) => {
-      const keysA = Object.keys(a);
-      const keysB = Object.keys(b);
-
-      if (keysA.length !== keysB.length) return keysA.length - keysB.length;
-
-      keysA.sort();
-      keysB.sort();
-
-      for (let i = 0; i < keysA.length; ++i) {
-        if (keysA[i] > keysB[i]) return 1;
-        else if (keysA[i] < keysB[i]) return -1;
-        else if (a[keysA[i]] > b[keysB[i]]) return 1;
-        else if (a[keysA[i]] < b[keysB[i]]) return -1;
-      }
-
-      return 0;
-    });
-
-    return res;
-  }
-
-  public [Symbol.iterator](): IterableIterator<Instance> {
-    return this.set.values();
-  }
-
-  public forEach(callback: (instance: Instance) => void): void {
-    this.set.forEach(callback);
-  }
-
-  public values(): IterableIterator<Instance> {
-    return this.set.values();
-  }
-
-  public clear(): void {
-    this.set.clear();
-  }
-
-  public get size(): number {
-    return this.set.size;
-  }
-}

lib/local-cache.js

@@ -0,0 +1,356 @@
+/**
+ * Copyright 2013-2017  Zaid Abdulla
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+"use strict";
+
+const vm = require("vm");
+const crypto = require("crypto");
+
+const db = require("./db");
+const cache = require("./cache");
+const query = require("./query");
+const logger = require("./logger");
+const scheduling = require("./scheduling");
+
+const REFRESH = 3000;
+
+let nextRefresh = 1;
+let hash = null;
+let presets, provisions, virtualParameters, files;
+
+
+function computeHash() {
+  // MD5 hash for presets, provisions, virtual parameters for detecting changes
+  let h = crypto.createHash("md5");
+  for (let p of presets) {
+    h.update(JSON.stringify(p.name));
+    h.update(JSON.stringify(p.channel));
+    h.update(JSON.stringify(p.schedule));
+    h.update(JSON.stringify(p.events));
+    h.update(JSON.stringify(p.precondition));
+    h.update(JSON.stringify(p.provisions));
+  }
+
+  let keys;
+
+  keys = Object.keys(provisions).sort();
+  h.update(JSON.stringify(keys));
+  for (let k of keys)
+    h.update(provisions[k].md5);
+
+  keys = Object.keys(virtualParameters).sort();
+  h.update(JSON.stringify(keys));
+  for (let k of keys)
+    h.update(virtualParameters[k].md5);
+
+  hash = h.digest("hex");
+}
+
+function flattenObject(src, prefix = "", dst = {}) {
+  for (let k of Object.keys(src)) {
+    let v = src[k];
+    if (typeof v === "object" && !Array.isArray(v))
+      flattenObject(v, `${prefix}${k}.`, dst);
+    else
+      dst[`${prefix}${k}`] = v;
+  }
+  return dst;
+}
+
+function refresh(callback) {
+  if (!nextRefresh)
+    return setTimeout(function() {refresh(callback);}, 20);
+
+  nextRefresh = 0;
+  const now = Date.now();
+
+  cache.get("presets_hash", function(err, res) {
+    if (err)
+      return callback(err);
+
+    if (hash && res === hash) {
+      nextRefresh = now + (REFRESH - (now % REFRESH));
+      return callback();
+    }
+
+    cache.lock("presets_hash_lock", 3, function(err, unlockOrExtend) {
+      if (err)
+        return callback(err);
+
+      let counter = 3;
+
+      counter += 2;
+      db.getPresets(function(err, res) {
+        if (err) {
+          if (counter & 1)
+            callback(err);
+          return counter = 0;
+        }
+
+        db.getObjects(function(err, objects) {
+          if (err) {
+            if (counter & 1)
+              callback(err);
+            return counter = 0;
+          }
+
+          objects = objects.map(obj => {
+            // Flatten object
+            obj = flattenObject(obj);
+
+            // If no keys are defined, consider all parameters as keys to keep the
+            // same behavior from v1.0
+            if (!obj["_keys"] || !obj["_keys"].length)
+              obj["_keys"] = Object.keys(obj).filter(k => !k.startsWith("_"));
+
+            return obj
+          })
+
+          res.sort((a, b) => {
+            if (a.weight === b.weight)
+              return a._id > b._id;
+            else
+              return a.weight - b.weight;
+          });
+
+          presets = [];
+          for (let preset of res) {
+            let schedule = null;
+            if (preset.schedule) {
+              let parts = preset.schedule.trim().split(/\s+/);
+              schedule = {
+                md5: crypto.createHash("md5").update(preset.schedule).digest("hex")
+              };
+
+              try {
+                schedule.duration = +(parts.shift()) * 1000;
+                schedule.schedule = scheduling.parseCron(parts.join(" "));
+              }
+              catch (err) {
+                logger.warn({
+                  message: "Invalid preset schedule",
+                  preset: preset._id,
+                  schedule: preset.schedule
+                });
+                schedule.schedule = false;
+              }
+            }
+
+            let events = preset.events || {};
+            let precondition = query.convertMongoQueryToFilters(JSON.parse(preset.precondition));
+            let _provisions = preset.provisions || [];
+
+            // Generate provisions from the old configuration format
+            for (let c of preset.configurations) {
+              switch (c.type) {
+                case "age":
+                  _provisions.push(["refresh", c.name, c.age]);
+                  break;
+
+                case "value":
+                  _provisions.push(["value", c.name, c.value]);
+                  break;
+
+                case "add_tag":
+                  _provisions.push(["tag", c.tag, true]);
+                  break;
+
+                case "delete_tag":
+                  _provisions.push(["tag", c.tag, false]);
+                  break;
+
+                case "provision":
+                  _provisions.push([c.name].concat(c.args || []));
+                  break;
+
+                case "add_object":
+                  for (let obj of objects) {
+                    if (obj["_id"] === c.object) {
+                      let alias = obj["_keys"].map((k) => `${k}:${JSON.stringify(obj[k])}`).join(",");
+                      let p = `${c.name}.[${alias}]`;
+                      _provisions.push(["instances", p, 1]);
+
+                      for (let k in obj)
+                        if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
+                          _provisions.push(["value", `${p}.${k}`, obj[k]]);
+                    }
+                  }
+                  break;
+
+                case "delete_object":
+                  for (let obj of objects) {
+                    if (obj["_id"] === c.object) {
+                      let alias = obj["_keys"].map((k) => `${k}:${JSON.stringify(obj[k])}`).join(",");
+                      let p = `${c.name}.[${alias}]`;
+                      _provisions.push(["instances", p, 0]);
+                    }
+                  }
+                  break;
+
+                default:
+                  if (counter & 1)
+                    callback(new Error(`Unknown configuration type ${c.type}`));
+                  return counter = 0;
+                  break;
+              }
+            }
+
+            presets.push({
+              name: preset._id,
+              channel: preset.channel || "default",
+              schedule: schedule,
+              events: events,
+              precondition: precondition,
+              provisions: _provisions
+            });
+          }
+
+          if ((counter -= 2) === 1) {
+            computeHash();
+            cache.set("presets_hash", hash, 300, function(err) {
+              unlockOrExtend(0);
+              nextRefresh = now + (REFRESH - (now % REFRESH));
+              callback(err);
+            });
+          }
+        });
+      });
+
+      counter += 2;
+      db.getProvisions(function(err, res) {
+        if (err) {
+          if (counter & 1)
+            callback(err);
+          return counter = 0;
+        }
+
+        provisions = {};
+        for (let r of res) {
+          provisions[r._id] = {};
+          provisions[r._id].md5 = crypto.createHash("md5").update(r.script).digest("hex");
+          provisions[r._id].script = new vm.Script(`"use strict";(function(){\n${r.script}\n})();`, {filename: r._id, lineOffset: -1, timeout: 50});
+        }
+
+        if ((counter -= 2) === 1) {
+          computeHash();
+          cache.set("presets_hash", hash, 300, function(err) {
+            unlockOrExtend(0);
+            nextRefresh = now + (REFRESH - (now % REFRESH));
+            callback(err);
+          });
+        }
+      });
+
+      counter += 2;
+      db.getVirtualParameters(function(err, res) {
+        if (err) {
+          if (counter & 1)
+            callback(err);
+          return counter = 0;
+        }
+
+        virtualParameters = {};
+        for (let r of res) {
+          virtualParameters[r._id] = {}
+          virtualParameters[r._id].md5 = crypto.createHash("md5").update(r.script).digest("hex");
+          virtualParameters[r._id].script = new vm.Script(`"use strict";(function(){\n${r.script}\n})();`, {filename: r._id, lineOffset: -1, timeout: 50});
+        }
+
+        if ((counter -= 2) === 1) {
+          computeHash();
+          cache.set("presets_hash", hash, 300, function(err) {
+            unlockOrExtend(0);
+            nextRefresh = now + (REFRESH - (now % REFRESH));
+            callback(err);
+          });
+        }
+      });
+
+      counter += 2;
+      db.getFiles(function(err, res) {
+        if (err) {
+          if (counter & 1)
+            callback(err);
+          return counter = 0;
+        }
+
+        files = {};
+        for (let r of res) {
+          const id = r.filename || r._id.toString();
+          files[id] = {};
+          files[id].length = r.length;
+          files[id].md5 = r.md5;
+          files[id].contentType = r.contentType;
+        }
+
+        if ((counter -= 2) === 1) {
+          computeHash();
+          cache.set("presets_hash", hash, 300, function(err) {
+            unlockOrExtend(0);
+            nextRefresh = now + (REFRESH - (now % REFRESH));
+            callback(err);
+          });
+        }
+      });
+
+      if ((counter -= 2) === 1) {
+        computeHash();
+        cache.set("presets_hash", hash, 300, function(err) {
+          unlockOrExtend(0);
+          nextRefresh = now + (REFRESH - (now % REFRESH));
+          callback(err);
+        });
+      }
+    });
+  });
+}
+
+
+function getPresets(callback) {
+  if (Date.now() < nextRefresh)
+    return callback(null, hash, presets);
+
+  refresh(function(err) {
+    callback(err, hash, presets);
+  });
+}
+
+
+function getFiles(callback) {
+  if (Date.now() < nextRefresh)
+    return callback(null, hash, files);
+
+  refresh(function(err) {
+    callback(err, hash, files);
+  })
+}
+
+
+function getProvisionsAndVirtualParameters(callback) {
+  if (Date.now() < nextRefresh)
+    return callback(null, hash, provisions, virtualParameters);
+
+  refresh(function(err) {
+    callback(err, hash, provisions, virtualParameters);
+  });
+}
+
+
+exports.getPresets = getPresets;
+exports.getFiles = getFiles;
+exports.getProvisionsAndVirtualParameters = getProvisionsAndVirtualParameters;

lib/local-cache.ts

@@ -1,548 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as vm from "vm";
-import * as crypto from "crypto";
-import * as config from "./config";
-import * as db from "./db";
-import * as cache from "./cache";
-import { convertOldPrecondition } from "./mongodb-functions";
-import * as logger from "./logger";
-import * as scheduling from "./scheduling";
-import * as expression from "./common/expression";
-import { parse } from "./common/expression-parser";
-import {
-  Preset,
-  Expression,
-  Provisions,
-  VirtualParameters,
-  Files,
-  Users,
-  Permissions,
-  Config,
-  UiConfig,
-} from "./types";
-
-interface Snapshot {
-  presets: Preset[];
-  provisions: Provisions;
-  virtualParameters: VirtualParameters;
-  files: Files;
-  permissions: Permissions;
-  users: Users;
-  config: Config;
-  ui: UiConfig;
-}
-
-const REFRESH = 5000;
-const EVICT_TIMEOUT = 120000;
-
-const snapshots = new Map<string, Snapshot>();
-let currentSnapshot: string = null;
-let nextRefresh = 1;
-
-function computeHash(snapshot): string {
-  // MD5 hash for presets, provisions, virtual parameters for detecting changes
-  const h = crypto.createHash("md5");
-  for (const p of snapshot.presets) {
-    h.update(JSON.stringify(p.name));
-    h.update(JSON.stringify(p.channel));
-    h.update(JSON.stringify(p.schedule));
-    h.update(JSON.stringify(p.events));
-    h.update(JSON.stringify(p.precondition));
-    h.update(JSON.stringify(p.provisions));
-  }
-
-  let keys;
-
-  keys = Object.keys(snapshot.provisions).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(snapshot.provisions[k].md5);
-
-  keys = Object.keys(snapshot.virtualParameters).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(snapshot.virtualParameters[k].md5);
-
-  keys = Object.keys(snapshot.config).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.config[k]));
-
-  keys = Object.keys(snapshot.files).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.files[k]));
-
-  keys = Object.keys(snapshot.users).sort();
-  h.update(JSON.stringify(keys));
-  for (const k of keys) h.update(JSON.stringify(snapshot.users[k]));
-
-  const roles = Object.keys(snapshot.permissions).sort();
-  h.update(JSON.stringify(roles));
-  for (const r of roles) {
-    const levels = Object.keys(snapshot.permissions[r]).sort();
-    h.update(JSON.stringify(levels));
-    for (const l of levels) {
-      keys = Object.keys(snapshot.permissions[r][l]).sort();
-      h.update(JSON.stringify(keys));
-      for (const k of keys)
-        h.update(JSON.stringify(snapshot.permissions[r][l][k]));
-    }
-  }
-
-  return h.digest("hex");
-}
-
-function flattenObject(
-  src: Record<string, unknown>,
-  prefix = "",
-  dst = {}
-): Record<string, unknown> {
-  for (const k of Object.keys(src)) {
-    const v = src[k];
-    if (typeof v === "object" && !Array.isArray(v))
-      flattenObject(v as Record<string, unknown>, `${prefix}${k}.`, dst);
-    else dst[`${prefix}${k}`] = v;
-  }
-  return dst;
-}
-
-async function fetchPresets(): Promise<Preset[]> {
-  const res = await db.getPresets();
-  let objects = await db.getObjects();
-
-  objects = objects.map((obj) => {
-    // Flatten object
-    obj = flattenObject(obj);
-
-    // If no keys are defined, consider all parameters as keys to keep the
-    // same behavior from v1.0
-    if (!obj["_keys"] || !obj["_keys"].length)
-      obj["_keys"] = Object.keys(obj).filter((k) => !k.startsWith("_"));
-
-    return obj;
-  });
-
-  res.sort((a, b) => {
-    if (a["weight"] === b["weight"])
-      return a["_id"] > b["_id"] ? 1 : a["_id"] < b["_id"] ? -1 : 0;
-    else return a["weight"] - b["weight"];
-  });
-
-  const presets = [] as Preset[];
-  for (const preset of res) {
-    let schedule: { md5: string; duration: number; schedule: any } = null;
-    if (preset["schedule"]) {
-      const parts = preset["schedule"].trim().split(/\s+/);
-      schedule = {
-        md5: crypto.createHash("md5").update(preset["schedule"]).digest("hex"),
-        duration: null,
-        schedule: null,
-      };
-
-      try {
-        schedule.duration = +parts.shift() * 1000;
-        schedule.schedule = scheduling.parseCron(parts.join(" "));
-      } catch (err) {
-        logger.warn({
-          message: "Invalid preset schedule",
-          preset: preset["_id"],
-          schedule: preset["schedule"],
-        });
-        schedule.schedule = false;
-      }
-    }
-
-    const events = preset["events"] || {};
-    let precondition = true as Expression;
-    if (preset["precondition"]) {
-      try {
-        precondition = parse(preset["precondition"]);
-      } catch (error) {
-        precondition = convertOldPrecondition(
-          JSON.parse(preset["precondition"])
-        );
-      }
-
-      // Simplify expression
-      precondition = expression.evaluate(precondition);
-    }
-
-    const _provisions = preset["provisions"] || [];
-
-    // Generate provisions from the old configuration format
-    for (const c of preset["configurations"]) {
-      switch (c.type) {
-        case "age":
-          _provisions.push(["refresh", c.name, +c.age]);
-          break;
-
-        case "value":
-          _provisions.push(["value", c.name, c.value]);
-          break;
-
-        case "add_tag":
-          _provisions.push(["tag", c.tag, true]);
-          break;
-
-        case "delete_tag":
-          _provisions.push(["tag", c.tag, false]);
-          break;
-
-        case "provision":
-          _provisions.push([c.name].concat(c.args || []));
-          break;
-
-        case "add_object":
-          for (const obj of objects) {
-            if (obj["_id"] === c.object) {
-              const alias = obj["_keys"]
-                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
-                .join(",");
-              const p = `${c.name}.[${alias}]`;
-              _provisions.push(["instances", p, 1]);
-
-              for (const k in obj) {
-                if (!k.startsWith("_") && !(obj["_keys"].indexOf(k) !== -1))
-                  _provisions.push(["value", `${p}.${k}`, obj[k]]);
-              }
-            }
-          }
-
-          break;
-
-        case "delete_object":
-          for (const obj of objects) {
-            if (obj["_id"] === c.object) {
-              const alias = obj["_keys"]
-                .map((k) => `${k}:${JSON.stringify(obj[k])}`)
-                .join(",");
-              const p = `${c.name}.[${alias}]`;
-              _provisions.push(["instances", p, 0]);
-            }
-          }
-
-          break;
-
-        default:
-          throw new Error(`Unknown configuration type ${c.type}`);
-      }
-    }
-
-    presets.push({
-      name: preset["_id"],
-      channel: (preset["channel"] as string) || "default",
-      schedule: schedule,
-      events: events,
-      precondition: precondition,
-      provisions: _provisions,
-    });
-  }
-
-  return presets;
-}
-
-async function fetchProvisions(): Promise<Provisions> {
-  const res = await db.getProvisions();
-
-  const provisions = {};
-  for (const r of res) {
-    provisions[r["_id"]] = {};
-    provisions[r["_id"]].md5 = crypto
-      .createHash("md5")
-      .update(r["script"])
-      .digest("hex");
-    provisions[
-      r["_id"]
-    ].script = new vm.Script(
-      `"use strict";(function(){\n${r["script"]}\n})();`,
-      { filename: r["_id"], lineOffset: -1, timeout: 50 }
-    );
-  }
-
-  return provisions;
-}
-
-async function fetchVirtualParameters(): Promise<VirtualParameters> {
-  const res = await db.getVirtualParameters();
-
-  const virtualParameters = {};
-  for (const r of res) {
-    virtualParameters[r["_id"]] = {};
-    virtualParameters[r["_id"]].md5 = crypto
-      .createHash("md5")
-      .update(r["script"])
-      .digest("hex");
-    virtualParameters[
-      r["_id"]
-    ].script = new vm.Script(
-      `"use strict";(function(){\n${r["script"]}\n})();`,
-      { filename: r["_id"], lineOffset: -1, timeout: 50 }
-    );
-  }
-
-  return virtualParameters;
-}
-
-async function fetchPermissions(): Promise<Permissions> {
-  const perms = await db.getPermissions();
-  const permissions: Permissions = {};
-
-  for (const p of perms) {
-    if (!permissions[p.role]) permissions[p.role] = {};
-    if (!permissions[p.role][p.access]) permissions[p.role][p.access] = {};
-
-    permissions[p.role][p.access][p.resource] = {
-      access: p.access,
-      filter: parse(p.filter || "true"),
-    };
-    if (p.validate)
-      permissions[p.role][p.access][p.resource].validate = parse(p.validate);
-  }
-
-  return permissions;
-}
-
-async function fetchFiles(): Promise<Files> {
-  const res = await db.getFiles();
-  const files = {};
-
-  for (const r of res) {
-    const id = r["filename"] || r["_id"].toString();
-    files[id] = {};
-    files[id].length = r["length"];
-    files[id].md5 = r["md5"];
-    files[id].contentType = r["contentType"];
-  }
-
-  return files;
-}
-
-async function fetchUsers(): Promise<Users> {
-  const _users = await db.getUsers();
-  const users = {};
-
-  for (const user of _users) {
-    users[user._id] = {
-      password: user.password,
-      salt: user.salt,
-      roles: user.roles.split(",").map((s) => s.trim()),
-    };
-  }
-
-  return users;
-}
-
-async function fetchConfig(): Promise<[Config, UiConfig]> {
-  const conf = await db.getConfig();
-
-  conf.sort((a, b) => (a.id > b.id ? 1 : a.id < b.id ? -1 : 0));
-
-  const ui = {
-    filters: {},
-    device: {},
-    index: {},
-    overview: {},
-  };
-  const _config = {};
-
-  for (const c of conf) {
-    // Evaluate expressions to simplify them
-    const val = expression.evaluate(c.value);
-    _config[c.id] = val;
-    if (c.id.startsWith("ui.")) {
-      const keys = c.id.split(".");
-      // remove the first key(ui)
-      keys.shift();
-      let ref = ui;
-      while (keys.length > 1) {
-        const k = keys.shift();
-        if (typeof ref[k] !== "object") ref[k] = {};
-        ref = ref[k];
-      }
-      ref[keys[0]] = val;
-    }
-  }
-
-  return [_config, ui];
-}
-
-async function refresh(): Promise<void> {
-  if (!nextRefresh) {
-    await new Promise((resolve) => setTimeout(resolve, 20));
-    await refresh();
-    return;
-  }
-
-  nextRefresh = 0;
-  const now = Date.now();
-
-  const dbHash = await cache.get("presets_hash");
-
-  if (currentSnapshot && dbHash === currentSnapshot) {
-    nextRefresh = now + (REFRESH - (now % REFRESH));
-    return;
-  }
-
-  const lockToken = await cache.acquireLock("presets_hash_lock", 5000, 5000);
-
-  const res = await Promise.all([
-    fetchPresets(),
-    fetchProvisions(),
-    fetchVirtualParameters(),
-    fetchFiles(),
-    fetchPermissions(),
-    fetchUsers(),
-    fetchConfig(),
-  ]);
-
-  const snapshot = {
-    presets: res[0],
-    provisions: res[1],
-    virtualParameters: res[2],
-    files: res[3],
-    permissions: res[4],
-    users: res[5],
-    config: res[6][0],
-    ui: res[6][1],
-  };
-
-  if (currentSnapshot) {
-    const h = currentSnapshot;
-    const s = snapshots.get(h);
-    setTimeout(() => {
-      if (snapshots.get(h) === s) snapshots.delete(h);
-    }, EVICT_TIMEOUT).unref();
-  }
-
-  currentSnapshot = computeHash(snapshot);
-  snapshots.set(currentSnapshot, snapshot);
-  await cache.set("presets_hash", currentSnapshot, 300);
-  await cache.releaseLock("presets_hash_lock", lockToken);
-
-  nextRefresh = now + (REFRESH - (now % REFRESH));
-}
-
-export async function getCurrentSnapshot(): Promise<string> {
-  if (Date.now() > nextRefresh) await refresh();
-  return currentSnapshot;
-}
-
-export function hasSnapshot(hash: string): boolean {
-  return snapshots.has(hash);
-}
-
-export function getPresets(snapshotKey: string): Preset[] {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.presets;
-}
-
-export function getProvisions(snapshotKey: string): Provisions {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.provisions;
-}
-
-export function getVirtualParameters(snapshotKey: string): VirtualParameters {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.virtualParameters;
-}
-
-export function getFiles(snapshotKey: string): Files {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-  return snapshot.files;
-}
-
-export function getConfig(
-  snapshotKey: string,
-  key: string,
-  context: Record<string, unknown>,
-  now: number,
-  cb?: (e: Expression) => Expression
-): string | number | boolean | null {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  const oldOpts = {
-    "cwmp.downloadTimeout": "DOWNLOAD_TIMEOUT",
-    "cwmp.debug": "DEBUG",
-    "cwmp.retryDelay": "RETRY_DELAY",
-    "cwmp.sessionTimeout": "SESSION_TIMEOUT",
-    "cwmp.connectionRequestTimeout": "CONNECTION_REQUEST_TIMEOUT",
-    "cwmp.gpnNextLevel": "GPN_NEXT_LEVEL",
-    "cwmp.gpvBatchSize": "GPV_BATCH_SIZE",
-    "cwmp.cookiesPath": "COOKIES_PATH",
-    "cwmp.datetimeMilliseconds": "DATETIME_MILLISECONDS",
-    "cwmp.booleanLiteral": "BOOLEAN_LITERAL",
-    "cwmp.connectionRequestAllowBasicAuth":
-      "CONNECTION_REQUEST_ALLOW_BASIC_AUTH",
-    "cwmp.maxCommitIterations": "MAX_COMMIT_ITERATIONS",
-    "cwmp.deviceOnlineThreshold": "DEVICE_ONLINE_THRESHOLD",
-    "cwmp.udpConnectionRequestPort": "UDP_CONNECTION_REQUEST_PORT",
-  };
-
-  if (!(key in snapshot.config)) {
-    if (key in oldOpts) {
-      let id;
-      if (context && context["id"]) {
-        id = context["id"];
-      } else if (cb) {
-        id = cb(["PARAM", "DeviceID.ID"]);
-        if (Array.isArray(id)) id = null;
-      }
-      return config.get(oldOpts[key], id);
-    }
-    return null;
-  }
-
-  const v = expression.evaluate(snapshot.config[key], context, now, cb);
-  return Array.isArray(v) ? null : v;
-}
-
-export function getConfigExpression(
-  snapshotKey: string,
-  key: string
-): Expression {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.config[key];
-}
-
-export function getUsers(snapshotKey: string): Users {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.users;
-}
-
-export function getPermissions(snapshotKey: string): Permissions {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.permissions;
-}
-
-export function getUiConfig(snapshotKey: string): UiConfig {
-  const snapshot = snapshots.get(snapshotKey);
-  if (!snapshot) throw new Error("Cache snapshot does not exist");
-
-  return snapshot.ui;
-}

lib/logger.js

@@ -0,0 +1,320 @@
+/**
+ * Copyright 2013-2017  Zaid Abdulla
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+"use strict";
+
+const fs = require("fs");
+const os = require("os");
+
+const config = require("./config");
+
+const REOPEN_EVERY = 60000;
+
+const LOG_FORMAT = config.get("LOG_FORMAT");
+const ACCESS_LOG_FORMAT = config.get("ACCESS_LOG_FORMAT") || LOG_FORMAT;
+
+const defaultMeta = {};
+
+let LOG_SYSTEMD = false;
+let ACCESS_LOG_SYSTEMD = false;
+
+let LOG_FILE, ACCESS_LOG_FILE;
+
+let logStream = fs.createWriteStream(null, {fd: process.stderr.fd});
+let logStat = fs.fstatSync(logStream.fd);
+let accessLogStream = fs.createWriteStream(null, {fd: process.stdout.fd});
+let accessLogStat = fs.fstatSync(accessLogStream.fd);
+
+
+// Reopen if original files have been moved (e.g. logrotate)
+function reopen() {
+  let counter = 1;
+
+  if (LOG_FILE) {
+    ++ counter;
+    fs.stat(LOG_FILE, function(err, stat) {
+      if (err && !err.message.startsWith("ENOENT:"))
+        throw err;
+
+      if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
+        logStream.end();
+        logStream = fs.createWriteStream(null,
+          {fd: fs.openSync(LOG_FILE, "a")});
+        logStat = fs.fstatSync(logStream.fd);
+      }
+
+      if (-- counter === 0)
+        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+    });
+  }
+
+  if (ACCESS_LOG_FILE) {
+    ++ counter;
+    fs.stat(ACCESS_LOG_FILE, function(err, stat) {
+      if (err && !err.message.startsWith("ENOENT:"))
+        throw err;
+
+      if (!(stat && stat.dev === accessLogStat.dev && stat.ino === accessLogStat.ino)) {
+        accessLogStream.end();
+        accessLogStream = fs.createWriteStream(null,
+          {fd: fs.openSync(ACCESS_LOG_FILE, "a")});
+        accessLogStat = fs.fstatSync(accessLogStream.fd);
+      }
+
+      if (-- counter === 0)
+        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+    });
+  }
+
+  if (-- counter === 0)
+    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+}
+
+
+function init(service, version) {
+  defaultMeta.hostname = os.hostname();
+  defaultMeta.pid = process.pid;
+  defaultMeta.name = `genieacs-${service}`;
+  defaultMeta.version = version;
+
+  LOG_FILE = config.get(`${service.toUpperCase()}_LOG_FILE`);
+  ACCESS_LOG_FILE = config.get(`${service.toUpperCase()}_ACCESS_LOG_FILE`);
+
+  if (LOG_FILE) {
+    logStream = fs.createWriteStream(null,
+      {fd: fs.openSync(LOG_FILE, "a")});
+    logStat = fs.fstatSync(logStream.fd);
+  }
+
+  if (ACCESS_LOG_FILE) {
+    accessLogStream = fs.createWriteStream(null,
+      {fd: fs.openSync(ACCESS_LOG_FILE, "a")});
+    accessLogStat = fs.fstatSync(accessLogStream.fd);
+  }
+
+  // Determine if logs are going to journald
+  const JOURNAL_STREAM = process.env["JOURNAL_STREAM"];
+
+  if (JOURNAL_STREAM) {
+    let [dev, inode] = JOURNAL_STREAM.split(":");
+    dev = parseInt(dev);
+    inode = parseInt(inode);
+
+    LOG_SYSTEMD = (logStat.dev === dev && logStat.ino === inode);
+    ACCESS_LOG_SYSTEMD = (accessLogStat.dev === dev && accessLogStat.ino === inode);
+  }
+
+  if (LOG_FILE || ACCESS_LOG_FILE) {
+    // Can't use setInterval as we need all workers to cehck at the same time
+    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
+  }
+}
+
+
+function close() {
+  accessLogStream.end();
+  logStream.end();
+}
+
+
+function flatten(details) {
+  if (details.sessionContext) {
+    details.deviceId = details.sessionContext.deviceId;
+    details.remoteAddress = details.sessionContext.httpRequest.connection.remoteAddress;
+    delete details.sessionContext;
+  }
+
+  if (details.exception) {
+    details.exceptionName = details.exception.name;
+    details.exceptionMessage = details.exception.message;
+    details.exceptionStack = details.exception.stack;
+    delete details.exception;
+  }
+
+  if (details.task) {
+    details.taskId = details.task._id;
+    delete details.task;
+  }
+
+  if (details.rpc) {
+    if (details.rpc.acsRequest) {
+      details.acsRequestId = details.rpc.id;
+      details.acsRequestName = details.rpc.acsRequest.name;
+      if (details.rpc.acsRequest.commandKey)
+        details.acsRequestCommandKey = details.rpc.acsRequest.commandKey;
+    }
+    else if (details.rpc.cpeRequest) {
+      details.cpeRequestId = details.rpc.id;
+      if (details.rpc.cpeRequest.name === "Inform") {
+        details.informEvent = details.rpc.cpeRequest.event.join(",");
+        details.informRetryCount = details.rpc.cpeRequest.retryCount;
+      }
+      else {
+        details.cpeRequestName = details.rpc.cpeRequest.name;
+        if (details.rpc.cpeRequest.commandKey)
+          details.cpeRequestCommandKey = details.rpc.cpeRequest.commandKey;
+      }
+    }
+    else if (details.rpc.cpeFault) {
+      details.acsRequestId = details.rpc.id;
+      details.cpeFaultCode = details.rpc.cpeFault.detail.faultCode;
+      details.cpeFaultString = details.rpc.cpeFault.detail.faultString;
+    }
+    delete details.rpc;
+  }
+
+  if (details.fault) {
+    details.faultCode = details.fault.code;
+    details.faultMessage = details.fault.message;
+    delete details.fault;
+  }
+
+  return details;
+}
+
+
+function formatJson(details, systemd) {
+  if (systemd) {
+    let severity = "";
+    if (details.severity === "info")
+      severity = "<6>";
+    else if (details.severity === "warn")
+      severity = "<4>";
+    else if (details.severity === "error")
+      severity = "<3>";
+
+    return `${severity}${JSON.stringify(flatten(details))}${os.EOL}`;
+  }
+
+  return `${JSON.stringify(flatten(details))}${os.EOL}`;
+}
+
+
+function formatSimple(details, systemd) {
+  flatten(details);
+
+  let remote = "";
+  if (details.remoteAddress) {
+    if (details.deviceId)
+      remote = `${details.remoteAddress} ${details.deviceId}: `;
+    else
+      remote = `${details.remoteAddress}: `;
+  }
+
+  const keys = Object.keys(details);
+  const skip = {
+    deviceId: 1,
+    remoteAddress: 1,
+    severity: 1,
+    timestamp: 1,
+    message: 1
+  };
+
+  let meta = "";
+
+  let kv = [];
+  for (let k of keys)
+    if (!skip[k])
+      kv.push(`${k}=${JSON.stringify(details[k])}`);
+
+  if (kv.length)
+    meta = `; ${kv.join(" ")}`;
+
+  if (systemd) {
+    let severity = "";
+    if (details.severity === "info")
+      severity = "<6>";
+    else if (details.severity === "warn")
+      severity = "<4>";
+    else if (details.severity === "error")
+      severity = "<3>";
+
+    return `${severity}${remote}${details.message}${meta}${os.EOL}`;
+  }
+
+  return `${details.timestamp} [${details.severity.toUpperCase()}] ${remote}${details.message}${meta}${os.EOL}`;
+}
+
+
+function log(details) {
+  details.timestamp = new Date().toISOString();
+  if (LOG_FORMAT === "json") {
+    details = Object.assign({}, defaultMeta, details);
+    logStream.write(formatJson(details, LOG_SYSTEMD));
+  } else {
+    logStream.write(formatSimple(details, LOG_SYSTEMD));
+  }
+}
+
+
+function info(details) {
+  details.severity = "info";
+  log(details);
+}
+
+
+function warn(details) {
+  details.severity = "warn";
+  log(details);
+}
+
+
+function error(details) {
+  details.severity = "error";
+  log(details);
+}
+
+
+function accessLog(details) {
+  details.timestamp = new Date().toISOString();
+  if (ACCESS_LOG_FORMAT === "json") {
+    Object.assign(details, defaultMeta);
+    accessLogStream.write(formatJson(details, ACCESS_LOG_SYSTEMD));
+  } else {
+    accessLogStream.write(formatSimple(details, ACCESS_LOG_SYSTEMD));
+  }
+}
+
+
+function accessInfo(details) {
+  details.severity = "info";
+  accessLog(details);
+}
+
+
+function accessWarn(details) {
+  details.severity = "warn";
+  accessLog(details);
+}
+
+
+function accessError(details) {
+  details.severity = "error";
+  accessLog(details);
+}
+
+
+exports.init = init;
+exports.close = close;
+exports.flatten = flatten;
+exports.info = info;
+exports.warn = warn;
+exports.error = error;
+exports.accessInfo = accessInfo;
+exports.accessWarn = accessWarn;
+exports.accessError = accessError;

lib/logger.ts

@@ -1,350 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as fs from "fs";
-import * as os from "os";
-
-import * as config from "./config";
-import { getRequestOrigin } from "./forwarded";
-import {
-  SessionContext,
-  AcsRequest,
-  CpeRequest,
-  CpeFault,
-  InformRequest,
-  Fault,
-} from "./types";
-
-const REOPEN_EVERY = 60000;
-
-const LOG_FORMAT = config.get("LOG_FORMAT");
-const ACCESS_LOG_FORMAT = config.get("ACCESS_LOG_FORMAT") || LOG_FORMAT;
-
-const defaultMeta: { [name: string]: any } = {};
-
-let LOG_SYSTEMD = false;
-let ACCESS_LOG_SYSTEMD = false;
-
-let LOG_FILE, ACCESS_LOG_FILE;
-
-declare global {
-  /* eslint-disable-next-line @typescript-eslint/no-namespace */
-  namespace NodeJS {
-    export interface WritableStream {
-      fd?: number;
-    }
-  }
-}
-
-declare module "fs" {
-  interface WriteStream {
-    fd?: number;
-  }
-}
-
-let logStream = fs.createWriteStream(null, { fd: process.stderr.fd });
-let logStat = fs.fstatSync(logStream.fd);
-let accessLogStream = fs.createWriteStream(null, { fd: process.stdout.fd });
-let accessLogStat = fs.fstatSync(accessLogStream.fd);
-
-// Reopen if original files have been moved (e.g. logrotate)
-function reopen(): void {
-  let counter = 1;
-
-  if (LOG_FILE) {
-    ++counter;
-    fs.stat(LOG_FILE, (err, stat) => {
-      if (err && !err.message.startsWith("ENOENT:")) throw err;
-
-      if (!(stat && stat.dev === logStat.dev && stat.ino === logStat.ino)) {
-        logStream.end();
-        logStream = fs.createWriteStream(null, {
-          fd: fs.openSync(LOG_FILE, "a"),
-        });
-        logStat = fs.fstatSync(logStream.fd);
-      }
-
-      if (--counter === 0)
-        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-    });
-  }
-
-  if (ACCESS_LOG_FILE) {
-    ++counter;
-    fs.stat(ACCESS_LOG_FILE, (err, stat) => {
-      if (err && !err.message.startsWith("ENOENT:")) throw err;
-
-      if (
-        !(
-          stat &&
-          stat.dev === accessLogStat.dev &&
-          stat.ino === accessLogStat.ino
-        )
-      ) {
-        accessLogStream.end();
-        accessLogStream = fs.createWriteStream(null, {
-          fd: fs.openSync(ACCESS_LOG_FILE, "a"),
-        });
-        accessLogStat = fs.fstatSync(accessLogStream.fd);
-      }
-
-      if (--counter === 0)
-        setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-    });
-  }
-
-  if (--counter === 0)
-    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-}
-
-export function init(service: string, version: string): void {
-  defaultMeta.hostname = os.hostname();
-  defaultMeta.pid = process.pid;
-  defaultMeta.name = `genieacs-${service}`;
-  defaultMeta.version = version;
-
-  LOG_FILE = config.get(`${service.toUpperCase()}_LOG_FILE`);
-  ACCESS_LOG_FILE = config.get(`${service.toUpperCase()}_ACCESS_LOG_FILE`);
-
-  if (LOG_FILE) {
-    logStream = fs.createWriteStream(null, { fd: fs.openSync(LOG_FILE, "a") });
-    logStat = fs.fstatSync(logStream.fd);
-  }
-
-  if (ACCESS_LOG_FILE) {
-    accessLogStream = fs.createWriteStream(null, {
-      fd: fs.openSync(ACCESS_LOG_FILE, "a"),
-    });
-    accessLogStat = fs.fstatSync(accessLogStream.fd);
-  }
-
-  // Determine if logs are going to journald
-  const JOURNAL_STREAM = process.env["JOURNAL_STREAM"];
-
-  if (JOURNAL_STREAM) {
-    const [dev, inode] = JOURNAL_STREAM.split(":").map(parseInt);
-
-    LOG_SYSTEMD = logStat.dev === dev && logStat.ino === inode;
-    ACCESS_LOG_SYSTEMD =
-      accessLogStat.dev === dev && accessLogStat.ino === inode;
-  }
-
-  if (LOG_FILE || ACCESS_LOG_FILE)
-    // Can't use setInterval as we need all workers to cehck at the same time
-    setTimeout(reopen, REOPEN_EVERY - (Date.now() % REOPEN_EVERY)).unref();
-}
-
-export function close(): void {
-  accessLogStream.end();
-  logStream.end();
-}
-
-export function flatten(
-  details: Record<string, unknown>
-): Record<string, unknown> {
-  if (details.sessionContext) {
-    const sessionContext = details.sessionContext as SessionContext;
-    details.deviceId = sessionContext.deviceId;
-    details.remoteAddress = getRequestOrigin(
-      sessionContext.httpRequest
-    ).remoteAddress;
-    delete details.sessionContext;
-  }
-
-  if (details.exception) {
-    const err = details.exception as Error;
-    details.exceptionName = err.name;
-    details.exceptionMessage = err.message;
-    details.exceptionStack = err.stack;
-    delete details.exception;
-  }
-
-  if (details.task) {
-    details.taskId = details.task["_id"];
-    delete details.task;
-  }
-
-  if (details.rpc) {
-    const rpc = details.rpc as {
-      id: string;
-      acsRequest?: AcsRequest;
-      cpeRequest?: CpeRequest;
-      cpeFault?: CpeFault;
-    };
-    if (rpc.acsRequest) {
-      details.acsRequestId = rpc.id;
-      details.acsRequestName = rpc.acsRequest.name;
-      if (rpc.acsRequest["commandKey"])
-        details.acsRequestCommandKey = rpc.acsRequest["commandKey"];
-    } else if (rpc.cpeRequest) {
-      details.cpeRequestId = rpc.id;
-      if (rpc.cpeRequest.name === "Inform") {
-        details.informEvent = (rpc.cpeRequest as InformRequest).event.join(",");
-        details.informRetryCount = (rpc.cpeRequest as InformRequest).retryCount;
-      } else {
-        details.cpeRequestName = rpc.cpeRequest.name;
-        if (rpc.cpeRequest["commandKey"])
-          details.cpeRequestCommandKey = rpc.cpeRequest["commandKey"];
-      }
-    } else if (rpc.cpeFault) {
-      details.acsRequestId = rpc.id;
-      details.cpeFaultCode = rpc.cpeFault.detail.faultCode;
-      details.cpeFaultString = rpc.cpeFault.detail.faultString;
-    }
-    delete details.rpc;
-  }
-
-  if (details.fault) {
-    const fault = details.fault as Fault;
-    details.faultCode = fault.code;
-    details.faultMessage = fault.message;
-    delete details.fault;
-  }
-
-  // For genieacs-ui
-  if (details.context) {
-    details.remoteAddress = getRequestOrigin(
-      details.context["req"]
-    ).remoteAddress;
-    if (details.context["state"].user)
-      details.user = details.context["state"].user.username;
-    delete details.context;
-  }
-
-  for (const [k, v] of Object.entries(details))
-    if (v == null) delete details[k];
-
-  return details;
-}
-
-function formatJson(
-  details: Record<string, unknown>,
-  systemd: boolean
-): string {
-  if (systemd) {
-    let severity = "";
-    if (details.severity === "info") severity = "<6>";
-    else if (details.severity === "warn") severity = "<4>";
-    else if (details.severity === "error") severity = "<3>";
-
-    return `${severity}${JSON.stringify(flatten(details))}${os.EOL}`;
-  }
-
-  return `${JSON.stringify(flatten(details))}${os.EOL}`;
-}
-
-function formatSimple(
-  details: Record<string, unknown>,
-  systemd: boolean
-): string {
-  const skip = {
-    user: true,
-    remoteAddress: true,
-    severity: true,
-    timestamp: true,
-    message: true,
-    deviceId: !!details.sessionContext,
-  };
-
-  flatten(details);
-
-  let remote = "";
-  if (details.remoteAddress) {
-    if (details.deviceId && skip["deviceId"])
-      remote = `${details.remoteAddress} ${details.deviceId}: `;
-    else if (details.user)
-      remote = `${details.user}@${details.remoteAddress}: `;
-    else remote = `${details.remoteAddress}: `;
-  }
-
-  const keys = Object.keys(details);
-
-  let meta = "";
-
-  const kv = [];
-  for (const k of keys)
-    if (!skip[k]) kv.push(`${k}=${JSON.stringify(details[k])}`);
-
-  if (kv.length) meta = `; ${kv.join(" ")}`;
-
-  if (systemd) {
-    let severity = "";
-    if (details.severity === "info") severity = "<6>";
-    else if (details.severity === "warn") severity = "<4>";
-    else if (details.severity === "error") severity = "<3>";
-
-    return `${severity}${remote}${details.message}${meta}${os.EOL}`;
-  }
-
-  return `${
-    details.timestamp
-  } [${(details.severity as string).toUpperCase()}] ${remote}${
-    details.message
-  }${meta}${os.EOL}`;
-}
-
-function log(details: Record<string, unknown>): void {
-  details.timestamp = new Date().toISOString();
-  if (LOG_FORMAT === "json") {
-    details = Object.assign({}, defaultMeta, details);
-    logStream.write(formatJson(details, LOG_SYSTEMD));
-  } else {
-    logStream.write(formatSimple(details, LOG_SYSTEMD));
-  }
-}
-
-export function info(details: Record<string, unknown>): void {
-  details.severity = "info";
-  log(details);
-}
-
-export function warn(details: Record<string, unknown>): void {
-  details.severity = "warn";
-  log(details);
-}
-
-export function error(details: Record<string, unknown>): void {
-  details.severity = "error";
-  log(details);
-}
-
-export function accessLog(details: Record<string, unknown>): void {
-  details.timestamp = new Date().toISOString();
-  if (ACCESS_LOG_FORMAT === "json") {
-    Object.assign(details, defaultMeta);
-    accessLogStream.write(formatJson(details, ACCESS_LOG_SYSTEMD));
-  } else {
-    accessLogStream.write(formatSimple(details, ACCESS_LOG_SYSTEMD));
-  }
-}
-
-export function accessInfo(details: Record<string, unknown>): void {
-  details.severity = "info";
-  accessLog(details);
-}
-
-export function accessWarn(details: Record<string, unknown>): void {
-  details.severity = "warn";
-  accessLog(details);
-}
-
-export function accessError(details: Record<string, unknown>): void {
-  details.severity = "error";
-  accessLog(details);
-}

lib/mongodb-functions.ts

@@ -1,684 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ObjectID } from "mongodb";
-import { map, parse, stringify, parseList } from "./common/expression-parser";
-import { likePatternToRegExp } from "./common/expression";
-import { Expression, Fault, Task } from "./types";
-import { decodeTag, encodeTag } from "./common";
-
-const isArray = Array.isArray;
-
-export function processDeviceFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
-    if (!isArray(exp)) return exp;
-
-    if (exp[0] === "PARAM") {
-      const p = exp[1];
-      if (p === "DeviceID.ID") return ["PARAM", "_id"];
-      else if (p === "DeviceID") return ["PARAM", "_deviceId"];
-      else if (p.startsWith("DeviceID."))
-        return ["PARAM", "_deviceId._" + p.slice(9)];
-      else if (p === "Events.Inform") return ["PARAM", "_lastInform"];
-      else if (p === "Events.Registered") return ["PARAM", "_registered"];
-      else if (p === "Events.0_BOOTSTRAP") return ["PARAM", "_lastBootstrap"];
-      else if (p === "Events.1_BOOT") return ["PARAM", "_lastBoot"];
-    } else if (
-      isArray(exp[1]) &&
-      exp[1][0] === "PARAM" &&
-      exp[1][1].startsWith("Tags.")
-    ) {
-      const t = decodeTag(exp[1][1].slice(5));
-      if (exp[0] === "IS NULL") return ["_tags", t, false];
-      else if (exp[0] === "IS NOT NULL") return ["_tags", t, true];
-      else if (exp[0] === "=" && exp[2] === true) return ["_tags", t, true];
-      else if (exp[0] === "<>" && exp[2] !== true) return ["_tags", t, true];
-    } else if (
-      [
-        "=",
-        "<>",
-        ">",
-        ">=",
-        "<",
-        "<=",
-        "LIKE",
-        "NOT LIKE",
-        "IS NULL",
-        "IS NOT NULL",
-      ].includes(exp[0])
-    ) {
-      let e = map(exp, (ee) => {
-        if (
-          isArray(ee) &&
-          ee[0] === "PARAM" &&
-          typeof ee[1] === "string" &&
-          !ee[1].startsWith("_")
-        )
-          return ["PARAM", `${ee[1]}._value`];
-        return ee;
-      });
-
-      if (typeof e[2] === "number") {
-        const alt = (e as any[]).slice();
-        alt[2] = new Date(e[2]);
-        e = ["OR", e, alt];
-      }
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function processTasksFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
-    if (!isArray(exp)) return exp;
-    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
-      const e = exp.slice();
-      if (e[1][0] === "PARAM" && e[1][1] === "_id") e[2] = new ObjectID(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "timestamp")
-        e[2] = new Date(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
-        e[2] = new Date(e[2]);
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function processFaultsFilter(filter: Expression): Expression {
-  return map(filter, (exp) => {
-    if (!isArray(exp)) return exp;
-    if (["=", "<>", ">", ">=", "<", "<="].includes(exp[0])) {
-      const e = exp.slice();
-      if (e[1][0] === "PARAM" && e[1][1] === "timestamp") e[2] = new Date(e[2]);
-      else if (e[1][0] === "PARAM" && e[1][1] === "expiry")
-        e[2] = new Date(e[2]);
-      return e;
-    }
-    return exp;
-  });
-}
-
-export function filterToMongoQuery(exp: Expression): Record<string, unknown> {
-  function recursive(filter, negate): Record<string, unknown> {
-    const op = filter[0];
-
-    if (op === "AND") {
-      filter = filter.filter((f) => f !== true);
-      if (filter.length === 2) return recursive(filter[1], negate);
-    }
-
-    if (op === "OR") {
-      filter = filter.filter((f) => f !== false);
-      if (filter.length === 2) return recursive(filter[1], negate);
-    }
-
-    if ((!negate && op === "AND") || (negate && op === "OR"))
-      return { $and: filter.slice(1).map((f) => recursive(f, negate)) };
-
-    if ((!negate && op === "OR") || (negate && op === "AND"))
-      return { $or: filter.slice(1).map((f) => recursive(f, negate)) };
-
-    if (op === "NOT") return recursive(filter[1], !negate);
-
-    if (isArray(filter[2]))
-      throw new Error(`Invalid RHS operand of ${op} clause`);
-
-    if (op === "LIKE" || op === "NOT LIKE") {
-      if (op === "NOT LIKE") negate = !negate;
-      let param;
-      let flags;
-      if (!Array.isArray(filter[1]))
-        throw new Error(`Invalid LHS operand of ${op} clause`);
-      if (filter[1][0] === "FUNC" && filter[1][1] === "UPPER") {
-        if (filter[2] !== filter[2].toUpperCase())
-          throw new Error(`Invalid RHS operand of ${op} clause`);
-        param = filter[1][2][1];
-        flags = "i";
-      } else if (filter[1][0] === "FUNC" && filter[1][1] === "LOWER") {
-        if (filter[2] !== filter[2].toLowerCase())
-          throw new Error(`Invalid RHS operand of ${op} clause`);
-        param = filter[1][2][1];
-        flags = "i";
-      } else if (filter[1][0] === "PARAM") {
-        param = filter[1][1];
-        flags = "";
-      } else {
-        throw new Error(`Invalid LHS operand of ${op} clause`);
-      }
-      const r = likePatternToRegExp(filter[2], filter[3], flags);
-      if (negate) return { [param]: { $nin: [r, null] } };
-      else return { [param]: r };
-    }
-
-    if (op === "_tags") {
-      let t = filter[2];
-      if (negate) t = !t;
-      if (t) return { _tags: filter[1] };
-      else return { _tags: { $ne: filter[1] } };
-    }
-
-    if (!isArray(filter[1]) || filter[1][0] !== "PARAM")
-      throw new Error(`Invalid LHS operand of ${op} clause`);
-
-    if (op === "IS NULL") {
-      const p = filter[1][1];
-      return { [p]: null };
-    }
-
-    if (op === "IS NOT NULL") {
-      const p = filter[1][1];
-      return { [p]: { $ne: null } };
-    }
-
-    if (isArray(filter[2]) || filter[2] == null)
-      throw new Error(`Invalid RHS operand of ${op} clause`);
-
-    if (op === "=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (!negate) return { [p]: v };
-      else return { [p]: { $nin: [v, null] } };
-    }
-
-    if (op === "<>") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: v };
-      else return { [p]: { $nin: [v, null] } };
-    }
-
-    if (op === ">") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $lte: v } };
-      else return { [p]: { $gt: v } };
-    }
-
-    if (op === ">=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $lt: v } };
-      else return { [p]: { $gte: v } };
-    }
-
-    if (op === "<") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $gte: v } };
-      else return { [p]: { $lt: v } };
-    }
-
-    if (op === "<=") {
-      const p = filter[1][1];
-      const v = filter[2];
-      if (negate) return { [p]: { $gt: v } };
-      else return { [p]: { $lte: v } };
-    }
-
-    throw new Error(`Unrecognized operator ${op}`);
-  }
-
-  if (!isArray(exp)) {
-    if (exp === true) return {};
-    throw new Error("Primitives are not valid queries");
-  }
-
-  return recursive(exp, false);
-}
-
-export function processDeviceProjection(
-  projection: Record<string, 1>
-): Record<string, 1> {
-  if (!projection) return projection;
-  const p = {};
-  for (const [k, v] of Object.entries(projection)) {
-    if (k === "DeviceID.ID") {
-      p["_id"] = 1;
-    } else if (k.startsWith("DeviceID")) {
-      p["_deviceId._SerialNumber"] = v;
-      p["_deviceId._OUI"] = v;
-      p["_deviceId._ProductClass"] = v;
-      p["_deviceId._Manufacturer"] = v;
-    } else if (k.startsWith("Tags")) {
-      p["_tags"] = v;
-    } else if (k.startsWith("Events")) {
-      p["_lastInform"] = v;
-      p["_registered"] = v;
-      p["_lastBoot"] = v;
-      p["_lastBootstrap"] = v;
-    } else {
-      p[k] = v;
-    }
-  }
-
-  return p;
-}
-
-export function processDeviceSort(
-  sort: Record<string, number>
-): Record<string, number> {
-  if (!sort) return sort;
-  const s = {};
-  for (const [k, v] of Object.entries(sort)) {
-    if (k === "DeviceID.ID") s["_id"] = v;
-    else if (k.startsWith("DeviceID.")) s[`_deviceId._${k.slice(9)}`] = v;
-    else if (k === "Events.Inform") s["_lastInform"] = v;
-    else if (k === "Events.Registered") s["_registered"] = v;
-    else if (k === "Events.1_BOOT") s["_lastBoot"] = v;
-    else if (k === "Events.0_BOOTSTRAP") s["_lastBootstrap"] = v;
-    else s[`${k}._value`] = v;
-  }
-
-  return s;
-}
-
-function parseDate(d: Date): number | string {
-  const n = +d;
-  return isNaN(n) ? "" + d : n;
-}
-
-interface FlatAttributes {
-  object?: boolean;
-  objectTimestamp?: number;
-  writable?: boolean;
-  writableTimestamp?: number;
-  value?: [string | number | boolean, string];
-  valueTimestamp?: number;
-  notification?: number;
-  notificationTimestamp?: number;
-  accessList?: string[];
-  accessListTimestamp?: number;
-}
-
-interface FlatDevice {
-  [param: string]: FlatAttributes;
-}
-
-export function flattenDevice(device: Record<string, unknown>): FlatDevice {
-  function recursive(
-    input,
-    root: string,
-    output: FlatDevice,
-    timestamp: number
-  ): void {
-    for (const [name, tree] of Object.entries(input)) {
-      if (!root) {
-        if (name === "_lastInform") {
-          output["Events.Inform"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_registered") {
-          output["Events.Registered"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_lastBoot") {
-          output["Events.1_BOOT"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_lastBootstrap") {
-          output["Events.0_BOOTSTRAP"] = {
-            value: [parseDate(tree as Date), "xsd:dateTime"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_id") {
-          output["DeviceID.ID"] = {
-            value: [tree as string, "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_deviceId") {
-          output["DeviceID.Manufacturer"] = {
-            value: [tree["_Manufacturer"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.OUI"] = {
-            value: [tree["_OUI"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.ProductClass"] = {
-            value: [tree["_ProductClass"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-          output["DeviceID.SerialNumber"] = {
-            value: [tree["_SerialNumber"], "xsd:string"],
-            valueTimestamp: timestamp,
-            writable: false,
-            writableTimestamp: timestamp,
-            object: false,
-            objectTimestamp: timestamp,
-          };
-        } else if (name === "_tags") {
-          output["Tags"] = {
-            writable: false,
-            writableTimestamp: timestamp,
-            object: true,
-            objectTimestamp: timestamp,
-          };
-
-          for (const t of tree as string[]) {
-            output[`Tags.${encodeTag(t)}`] = {
-              value: [true, "xsd:boolean"],
-              valueTimestamp: timestamp,
-              writable: true,
-              writableTimestamp: timestamp,
-              object: false,
-              objectTimestamp: timestamp,
-            };
-          }
-        }
-      }
-
-      if (name.startsWith("_")) continue;
-
-      let childrenTimestamp = timestamp;
-
-      if (!root) childrenTimestamp = +(input["_timestamp"] || 1);
-      else if (+input["_timestamp"] > timestamp)
-        childrenTimestamp = +input["_timestamp"];
-
-      const attrs: FlatAttributes = {};
-      if (tree["_value"] != null) {
-        attrs.value = [
-          tree["_value"] instanceof Date ? +tree["_value"] : tree["_value"],
-          tree["_type"],
-        ];
-        attrs.valueTimestamp = +(tree["_timestamp"] || childrenTimestamp);
-        attrs.object = false;
-        attrs.objectTimestamp = childrenTimestamp;
-      } else if (tree["_object"] != null) {
-        attrs.object = tree["_object"];
-        attrs.objectTimestamp = childrenTimestamp;
-      }
-
-      if (tree["_writable"] != null) {
-        attrs.writable = tree["_writable"];
-        attrs.writableTimestamp = childrenTimestamp;
-      }
-
-      if (tree["_notification"] != null) {
-        attrs.notification = tree["_notification"];
-        attrs.notificationTimestamp = +tree["_attributesTimestamp"] || 1;
-      }
-
-      if (tree["_accessList"] != null) {
-        attrs.accessList = tree["_accessList"];
-        attrs.accessListTimestamp = +tree["_attributesTimestamp"] || 1;
-      }
-
-      const r = root ? `${root}.${name}` : name;
-      output[r] = attrs;
-
-      if (attrs.object || tree["object"] == null)
-        recursive(tree, r, output, childrenTimestamp);
-    }
-  }
-
-  const newDevice: FlatDevice = {};
-  const timestamp = new Date((device["_lastInform"] as Date) || 1).getTime();
-  recursive(device, "", newDevice, timestamp);
-  return newDevice;
-}
-
-export function flattenFault(fault: unknown): Fault {
-  const f = Object.assign({}, fault) as Fault;
-  if (f.timestamp) f.timestamp = +f.timestamp;
-  if (f["expiry"]) f["expiry"] = +f["expiry"];
-  return f as Fault;
-}
-
-export function flattenTask(task: unknown): Task {
-  const t = Object.assign({}, task) as Task;
-  t._id = "" + t._id;
-  if (t["timestamp"]) t["timestamp"] = +t["timestamp"];
-  if (t.expiry) t.expiry = +t.expiry;
-  return t;
-}
-
-export function convertOldPrecondition(
-  query: Record<string, unknown>
-): Expression {
-  function recursive(_query): Expression {
-    const expressions: Expression[] = [];
-    for (const [k, v] of Object.entries(_query)) {
-      if (k[0] === "$") {
-        if (k === "$and") {
-          const and: Expression = ["AND"];
-          for (const vv of Object.values(v)) and.push(recursive(vv));
-          expressions.push(and);
-        } else if (k === "$or") {
-          const or: Expression = ["OR"];
-          for (const vv of Object.values(v)) or.push(recursive(vv));
-          expressions.push(or);
-        } else {
-          throw new Error(`Operator ${k} not supported`);
-        }
-      } else if (k === "_tags") {
-        if (typeof v === "object") {
-          if (isArray(v)) throw new Error(`Invalid type`);
-          const conjs: Expression[] = [];
-          for (const [op, val] of Object.entries(v)) {
-            if (op === "$ne") {
-              if (typeof v["$ne"] !== "string")
-                throw new Error("Only string values are allowed for _tags");
-              conjs.push(["IS NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
-            } else if (op === "$eq") {
-              if (typeof v["$eq"] !== "string")
-                throw new Error("Only string values are allowed for _tags");
-              conjs.push(["IS NOT NULL", ["PARAM", `Tags.${encodeTag(val)}`]]);
-            } else {
-              throw new Error(`Invalid tag query`);
-            }
-          }
-          if (conjs.length === 1) expressions.push(conjs[0]);
-          else if (conjs.length > 1) expressions.push(["AND", ...conjs]);
-        } else {
-          expressions.push([
-            "IS NOT NULL",
-            ["PARAM", `Tags.${encodeTag(v as string)}`],
-          ]);
-        }
-      } else if (k.startsWith("Tags.")) {
-        let exists: boolean;
-        if (typeof v === "boolean") exists = v;
-        else if (v.hasOwnProperty("$eq")) exists = !!v["$eq"];
-        else if (v.hasOwnProperty("$ne")) exists = !v["$ne"];
-        else if (v.hasOwnProperty("$exists")) exists = !!v["$exists"];
-        else throw new Error(`Invalid tag query`);
-
-        expressions.push([exists ? "IS NOT NULL" : "IS NULL", ["PARAM", k]]);
-      } else if (typeof v === "object") {
-        if (isArray(v)) throw new Error(`Invalid type`);
-
-        const exps: Expression[] = [];
-        for (const [kk, vv] of Object.entries(v)) {
-          if (kk === "$eq") {
-            exps.push(["=", ["PARAM", k], vv]);
-          } else if (kk === "$ne") {
-            exps.push([
-              "OR",
-              ["<>", ["PARAM", k], vv],
-              ["IS NULL", ["PARAM", k]],
-            ]);
-          } else if (kk === "$lt") {
-            exps.push(["<", ["PARAM", k], vv]);
-          } else if (kk === "$lte") {
-            exps.push(["<=", ["PARAM", k], vv]);
-          } else if (kk === "$gt") {
-            exps.push([">", ["PARAM", k], vv]);
-          } else if (kk === "$gte") {
-            exps.push([">=", ["PARAM", k], vv]);
-          } else {
-            throw new Error(`Operator ${kk} not supported`);
-          }
-          if (!["string", "number", "boolean"].includes(typeof vv))
-            throw new Error(`Invalid value for ${kk} operator`);
-        }
-        if (exps.length === 1) {
-          expressions.push(exps[0]);
-        } else if (exps.length > 1) {
-          const and: Expression = ["AND"];
-          expressions.push(and.concat(exps));
-        }
-      } else {
-        expressions.push(["=", ["PARAM", k], v]);
-      }
-    }
-    if (expressions.length === 1) return expressions[0];
-    if (expressions.length === 0) return true;
-    return ["AND", ...expressions];
-  }
-
-  // empty filter
-  if (!Object.keys(query).length) return true;
-
-  return recursive(query);
-}
-
-export function flattenPreset(
-  preset: Record<string, unknown>
-): Record<string, unknown> {
-  const p = Object.assign({}, preset);
-  if (p.precondition) {
-    try {
-      // Try parse to check expression validity
-      parse(p.precondition as string);
-    } catch (error) {
-      p.precondition = convertOldPrecondition(
-        JSON.parse(p.precondition as string)
-      );
-      p.precondition = (p.precondition as string).length
-        ? stringify(p.precondition as Expression)
-        : "";
-    }
-  }
-
-  if (p.events) {
-    const e = [];
-    for (const [k, v] of Object.entries(p.events)) e.push(v ? k : `-${k}`);
-    p.events = e.join(", ");
-  }
-
-  const provision = p.configurations[0];
-  if (
-    (p.configurations as any[]).length === 1 &&
-    provision.type === "provision" &&
-    provision.name &&
-    provision.name.length
-  ) {
-    p.provision = provision.name;
-    p.provisionArgs = provision.args
-      ? provision.args.map((a) => stringify(a)).join(", ")
-      : "";
-  }
-
-  delete p.configurations;
-  return p;
-}
-
-export function flattenFile(
-  file: Record<string, unknown>
-): Record<string, unknown> {
-  const f = {};
-  f["_id"] = file["_id"];
-  if (file.metadata) {
-    f["metadata.fileType"] = file["metadata"]["fileType"] || "";
-    f["metadata.oui"] = file["metadata"]["oui"] || "";
-    f["metadata.productClass"] = file["metadata"]["productClass"] || "";
-    f["metadata.version"] = file["metadata"]["version"] || "";
-  }
-  return f;
-}
-
-export function preProcessPreset(
-  data: Record<string, unknown>
-): Record<string, unknown> {
-  const preset = Object.assign({}, data);
-
-  if (!preset.precondition) preset.precondition = "";
-  // Try parse to check expression validity
-  parse(preset.precondition as string);
-
-  preset.weight = parseInt(preset.weight as string) || 0;
-
-  const events = {};
-  if (preset.events) {
-    for (let e of (preset.events as string).split(",")) {
-      let v = true;
-      e = e.trim();
-      if (e.startsWith("-")) {
-        v = false;
-        e = e.slice(1).trim();
-      }
-      if (e) events[e] = v;
-    }
-  }
-
-  preset.events = events;
-
-  if (!preset.provision) throw new Error("Invalid preset provision");
-
-  const configuration = {
-    type: "provision",
-    name: preset.provision,
-    args: null,
-  };
-
-  if (preset.provisionArgs)
-    configuration.args = parseList(preset.provisionArgs as string);
-
-  delete preset.provision;
-  delete preset.provisionArgs;
-  preset.configurations = [configuration];
-  return preset;
-}

lib/nbi.coffee

@@ -0,0 +1,517 @@
+###
+# Copyright 2013-2017  Zaid Abdulla
+#
+# This file is part of GenieACS.
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This file incorporates work covered by the following copyright and
+# permission notice:
+#
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
+url = require 'url'
+mongodb = require 'mongodb'
+querystring = require 'querystring'
+vm = require 'vm'
+
+config = require './config'
+common = require './common'
+db = require './db'
+query = require './query'
+apiFunctions = require './api-functions'
+cache = require './cache'
+
+VERSION = require('../package.json').version
+
+# regular expression objects
+DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/tasks\/?$/
+TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-\_\%]+)(\/[a-zA-Z_]*)?$/
+TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/tags\/([a-zA-Z0-9\-\_\%]+)\/?$/
+PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-\_\%]+)\/?$/
+OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-\_\%]+)\/?$/
+FILES_REGEX = /^\/files\/([a-zA-Z0-9\%\!\*\'\(\)\;\:\@\&\=\+\$\,\?\#\[\]\-\_\.\~]+)\/?$/
+PING_REGEX = /^\/ping\/([a-zA-Z0-9\-\_\.\:]+)\/?$/
+QUERY_REGEX = /^\/([a-zA-Z0-9_]+)\/?$/
+DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-\_\%]+)\/?$/
+PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-\_\%]+)\/?$/
+VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-\_\%]+)\/?$/
+FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-\_\%\:]+)\/?$/
+
+
+
+throwError = (err, httpResponse) ->
+  if httpResponse
+    httpResponse.writeHead(500, {'Connection' : 'close'})
+    httpResponse.end("#{err.name}: #{err.message}")
+
+  throw err
+
+
+listener = (request, response) ->
+  chunks = []
+  bytes = 0
+
+  response.setHeader('GenieACS-Version', VERSION)
+
+  request.addListener 'data', (chunk) ->
+    chunks.push(chunk)
+    bytes += chunk.length
+
+  request.getBody = () ->
+    # Write all chunks into a Buffer
+    body = new Buffer(bytes)
+    offset = 0
+    chunks.forEach((chunk) ->
+      chunk.copy(body, offset, 0, chunk.length)
+      offset += chunk.length
+    )
+    return body
+
+  request.addListener 'end', () ->
+    body = request.getBody()
+    urlParts = url.parse(request.url, true)
+    if PRESETS_REGEX.test(urlParts.pathname)
+      presetName = querystring.unescape(PRESETS_REGEX.exec(urlParts.pathname)[1])
+      if request.method == 'PUT'
+        preset = JSON.parse(body)
+        preset._id = presetName
+
+        db.presetsCollection.save(preset, (err) ->
+          return throwError(err, response) if err
+          cache.del('presets_hash', (err) ->
+            return throwError(err, response) if err
+            response.writeHead(200)
+            response.end()
+          )
+        )
+      else if request.method == 'DELETE'
+        db.presetsCollection.remove({'_id' : presetName}, (err) ->
+          return throwError(err, response) if err
+          cache.del('presets_hash', (err) ->
+            return throwError(err, response) if err
+            response.writeHead(200)
+            response.end()
+          )
+        )
+      else
+        response.writeHead 405, {'Allow': 'PUT, DELETE'}
+        response.end('405 Method Not Allowed')
+    else if OBJECTS_REGEX.test(urlParts.pathname)
+      objectName = querystring.unescape(OBJECTS_REGEX.exec(urlParts.pathname)[1])
+      if request.method == 'PUT'
+        object = JSON.parse(body)
+        object._id = objectName
+
+        db.objectsCollection.save(object, (err) ->
+          return throwError(err, response) if err
+          cache.del('presets_hash', (err) ->
+            return throwError(err, response) if err
+            response.writeHead(200)
+            response.end()
+          )
+        )
+      else if request.method == 'DELETE'
+        db.objectsCollection.remove({'_id' : objectName}, (err) ->
+          return throwError(err, response) if err
+          cache.del('presets_hash', (err) ->
+            return throwError(err, response) if err
+            response.writeHead(200)
+            response.end()
+          )
+        )
+      else
+        response.writeHead 405, {'Allow': 'PUT, DELETE'}
+        response.end('405 Method Not Allowed')
+    else if PROVISIONS_REGEX.test(urlParts.pathname)
+      provisionName = querystring.unescape(PROVISIONS_REGEX.exec(urlParts.pathname)[1])
+      if request.method == 'PUT'
+        object = {
+          _id: provisionName
+          script: body.toString()
+        }
+        try
+          new vm.Script("\"use strict\";(function(){\n#{object.script}\n})();")
+        catch err
+          response.writeHead(400)
+          response.end("#{err.name}: #{err.message}")
+          return
+
+        db.provisionsCollection.save(object, (err) ->
+          return throwError(err, response) if err
+          cache.del('presets_hash', (err) ->
+            return throwError(err, response) if err
+            response.writeHead(200)
+            response.end()
+          )
+        )
+      else if request.method == 'DELETE'
+        db.provisionsCollection.remove({'_id' : provisionName}, (err) ->
+          return throwError(err, response) if err
+          cache.del('presets_hash', (err) ->
+            return throwError(err, response) if err
+            response.writeHead(200)
+            response.end()
+          )
+        )
+      else
+        response.writeHead 405, {'Allow': 'PUT, DELETE'}
+        response.end('405 Method Not Allowed')
+    else if VIRTUAL_PARAMETERS_REGEX.test(urlParts.pathname)
+      virtualParameterName = querystring.unescape(VIRTUAL_PARAMETERS_REGEX.exec(urlParts.pathname)[1])
+      if request.method == 'PUT'
+        object = {
+          _id: virtualParameterName
+          script: body.toString()
+        }
+        try
+          new vm.Script("\"use strict\";(function(){\n#{object.script}\n})();")
+        catch err
+          response.writeHead(400)
+          response.end("#{err.name}: #{err.message}")
+          return
+
+        db.virtualParametersCollection.save(object, (err) ->
+          return throwError(err, response) if err
+          cache.del('presets_hash', (err) ->
+            return throwError(err, response) if err
+            response.writeHead(200)
+            response.end()
+          )
+        )
+      else if request.method == 'DELETE'
+        db.virtualParametersCollection.remove({'_id' : virtualParameterName}, (err) ->
+          return throwError(err, response) if err
+          cache.del('presets_hash', (err) ->
+            return throwError(err, response) if err
+            response.writeHead(200)
+            response.end()
+          )
+        )
+      else
+        response.writeHead 405, {'Allow': 'PUT, DELETE'}
+        response.end('405 Method Not Allowed')
+    else if TAGS_REGEX.test(urlParts.pathname)
+      r = TAGS_REGEX.exec(urlParts.pathname)
+      deviceId = querystring.unescape(r[1])
+      tag = querystring.unescape(r[2])
+      if request.method == 'POST'
+        db.devicesCollection.update({'_id' : deviceId}, {'$addToSet' : {'_tags' : tag}}, {safe: true}, (err) ->
+          return throwError(err, response) if err
+          response.writeHead(200)
+          response.end()
+        )
+      else if request.method == 'DELETE'
+        db.devicesCollection.update({'_id' : deviceId}, {'$pull' : {'_tags' : tag}}, {safe: true}, (err) ->
+          return throwError(err, response) if err
+          response.writeHead(200)
+          response.end()
+        )
+      else
+        response.writeHead 405, {'Allow': 'POST, DELETE'}
+        response.end('405 Method Not Allowed')
+    else if FAULTS_REGEX.test(urlParts.pathname)
+      if request.method == 'DELETE'
+        faultId = querystring.unescape(FAULTS_REGEX.exec(urlParts.pathname)[1])
+        deviceId = faultId.split(':', 1)[0]
+        channel = faultId.slice(deviceId.length + 1)
+        db.faultsCollection.remove({_id : faultId}, (err) ->
+          return throwError(err, response) if err
+          if channel.startsWith('task_')
+            return db.tasksCollection.remove({_id : new mongodb.ObjectID(channel.slice(5))}, (err) ->
+              return throwError(err, response) if err
+              cache.del("#{deviceId}_tasks_faults_operations", (err) ->
+                return throwError(err, response) if err
+                response.writeHead(200)
+                response.end()
+              )
+            )
+
+          cache.del("#{deviceId}_tasks_faults_operations", (err) ->
+            return throwError(err, response) if err
+            response.writeHead(200)
+            response.end()
+          )
+        )
+      else
+        response.writeHead 405, {'Allow': 'DELETE'}
+        response.end('405 Method Not Allowed')
+    else if DEVICE_TASKS_REGEX.test(urlParts.pathname)
+      if request.method == 'POST'
+        deviceId = querystring.unescape(DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1])
+        if body.length
+          task = JSON.parse(body)
+          task.device = deviceId
+          apiFunctions.insertTasks(task, (err) ->
+            return throwError(err, response) if err
+            cache.del("#{deviceId}_tasks_faults_operations", (err) ->
+              return throwError(err, response) if err
+
+              if urlParts.query.connection_request?
+                apiFunctions.connectionRequest(deviceId, (err) ->
+                  if err
+                    response.writeHead(202, err.message, {'Content-Type' : 'application/json'})
+                    response.end(JSON.stringify(task))
+                  else
+                    taskTimeout = urlParts.query.timeout? and parseInt(urlParts.query.timeout) or config.get('DEVICE_ONLINE_THRESHOLD', deviceId)
+                    apiFunctions.watchTask(deviceId, task._id, taskTimeout, (err, status) ->
+                      return throwError(err, response) if err
+
+                      if status is 'timeout'
+                        response.writeHead(202, 'Task queued but not processed', {'Content-Type' : 'application/json'})
+                        response.end(JSON.stringify(task))
+                      else if status is 'fault'
+                        db.tasksCollection.findOne({_id : task._id}, (err, task) ->
+                          return throwError(err, response) if err
+
+                          response.writeHead(202, 'Task faulted', {'Content-Type' : 'application/json'})
+                          response.end(JSON.stringify(task))
+                        )
+                      else
+                        response.writeHead(200, {'Content-Type' : 'application/json'})
+                        response.end(JSON.stringify(task))
+                    )
+                )
+              else
+                response.writeHead(202, {'Content-Type' : 'application/json'})
+                response.end(JSON.stringify(task))
+            )
+          )
+        else if urlParts.query.connection_request?
+          # no task, send connection request only
+          apiFunctions.connectionRequest(deviceId, (err) ->
+            if err
+              response.writeHead 504
+              response.end("#{err.name}: #{err.message}")
+              return
+            response.writeHead 200
+            response.end()
+          )
+        else
+          response.writeHead(400)
+          response.end()
+      else
+        response.writeHead 405, {'Allow': 'POST'}
+        response.end('405 Method Not Allowed')
+    else if TASKS_REGEX.test(urlParts.pathname)
+      r = TASKS_REGEX.exec(urlParts.pathname)
+      taskId = new mongodb.ObjectID(querystring.unescape(r[1]))
+      action = r[2]
+      if not action? or action is '/'
+        if request.method == 'DELETE'
+          db.tasksCollection.findOne({'_id' : taskId}, {'device' : 1}, (err, task) ->
+            return throwError(err, response) if err
+            if not task?
+              response.writeHead(404)
+              response.end("Task not found")
+              return
+
+            deviceId = task.device
+            db.tasksCollection.remove({'_id' : taskId}, (err) ->
+              return throwError(err, response) if err
+              db.faultsCollection.remove({_id : "#{deviceId}:task_#{String(taskId)}"}, (err) ->
+                return throwError(err, response) if err
+                cache.del("#{deviceId}_tasks_faults_operations", (err) ->
+                  return throwError(err, response) if err
+                  response.writeHead(200)
+                  response.end()
+                )
+              )
+            )
+          )
+        else
+          response.writeHead 405, {'Allow': 'PUT DELETE'}
+          response.end('405 Method Not Allowed')
+      else if action is '/retry'
+        if request.method == 'POST'
+          db.tasksCollection.findOne({'_id' : taskId}, {'device' : 1}, (err, task) ->
+            return throwError(err, response) if err
+            deviceId = task.device
+            db.faultsCollection.remove({_id : "#{deviceId}:task_#{String(taskId)}"}, (err) ->
+              return throwError(err, response) if err
+              cache.del("#{deviceId}_tasks_faults_operations", (err) ->
+                return throwError(err, response) if err
+                response.writeHead(200)
+                response.end()
+              )
+            )
+          )
+        else
+          response.writeHead 405, {'Allow': 'POST'}
+          response.end('405 Method Not Allowed')
+      else
+        response.writeHead(404)
+        response.end()
+    else if FILES_REGEX.test(urlParts.pathname)
+      filename = querystring.unescape(FILES_REGEX.exec(urlParts.pathname)[1])
+      if request.method == 'PUT'
+        metadata = {
+          fileType : request.headers.filetype,
+          oui : request.headers.oui,
+          productClass : request.headers.productclass,
+          version : request.headers.version,
+        }
+
+        gs = new mongodb.GridStore(db.mongoDb, filename, filename, 'w', {metadata : metadata})
+        gs.open((err, gs) ->
+          gs.write(body, (err, res) ->
+            return throwError(err, response) if err
+            gs.close((err) ->
+              return throwError(err, response) if err
+              response.writeHead(201)
+              response.end()
+            )
+          )
+        )
+      else if request.method == 'DELETE'
+        mongodb.GridStore.unlink(db.mongoDb, filename, (err) ->
+          response.writeHead(200)
+          response.end()
+        )
+      else
+        response.writeHead 405, {'Allow': 'PUT, DELETE'}
+        response.end('405 Method Not Allowed')
+    else if PING_REGEX.test(urlParts.pathname)
+      host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1])
+      require('child_process').exec("ping -w 1 -i 0.2 -c 3 #{host}", (err, stdout, stderr) ->
+        if err
+          response.writeHead(404, {'Cache-Control' : 'no-cache'})
+          response.end("#{err.name}: #{err.message}")
+          return
+        response.writeHead(200, {'Content-Type' : 'text/plain', 'Cache-Control' : 'no-cache'})
+        response.end(stdout)
+      )
+    else if DELETE_DEVICE_REGEX.test(urlParts.pathname)
+      if request.method isnt 'DELETE'
+        response.writeHead(405, {'Allow' : 'DELETE'})
+        response.end('405 Method Not Allowed')
+        return
+
+      deviceId = querystring.unescape(DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1])
+      apiFunctions.deleteDevice(deviceId, (err) ->
+        return throwError(err, response) if err
+        response.writeHead(200)
+        response.end()
+      )
+    else if QUERY_REGEX.test(urlParts.pathname)
+      collectionName = QUERY_REGEX.exec(urlParts.pathname)[1]
+
+      # Convert to camel case
+      i = collectionName.indexOf('_')
+      while i >= 0
+        ++ i
+        up = if i < collectionName.length then collectionName[i].toUpperCase() else ''
+        collectionName = collectionName.slice(0, i - 1) + up + collectionName.slice(i + 1)
+        i = collectionName.indexOf('_', i)
+
+      if request.method not in ['GET', 'HEAD']
+        response.writeHead 405, {'Allow' : 'GET, HEAD'}
+        response.end('405 Method Not Allowed')
+        return
+      collection = db["#{collectionName}Collection"]
+      if not collection?
+        response.writeHead 404
+        response.end('404 Not Found')
+        return
+
+      if urlParts.query.query?
+        try
+          q = JSON.parse(urlParts.query.query)
+        catch err
+          response.writeHead(400)
+          response.end("#{err.name}: #{err.message}")
+          return
+      else
+        q = {}
+
+      switch collectionName
+        when 'devices'
+          q = query.expand(q)
+        when 'tasks'
+          q = query.sanitizeQueryTypes(q, {
+            _id: ((v) -> return new mongodb.ObjectID(v))
+            timestamp: ((v) -> return new Date(v))
+            retries: Number
+          })
+        when 'faults'
+          q = query.sanitizeQueryTypes(q, {
+            timestamp: ((v) -> return new Date(v))
+            retries: Number
+          })
+
+      if urlParts.query.projection?
+        projection = {}
+        for p in urlParts.query.projection.split(',')
+          p = p.trim()
+          projection[p] = 1
+
+      cur = collection.find(q, projection, {batchSize : 50})
+      if urlParts.query.sort?
+        s = JSON.parse(urlParts.query.sort)
+        sort = {}
+        for k,v of s
+          if k[k.lastIndexOf('.') + 1] != '_' and collectionName is 'devices'
+            sort["#{k}._value"] = v
+          else
+            sort[k] = v
+        cur.sort(sort)
+
+      cur.skip(parseInt(urlParts.query.skip)) if urlParts.query.skip?
+      cur.limit(limit = parseInt(urlParts.query.limit)) if urlParts.query.limit?
+      cur.count(false, (err, total) ->
+        response.writeHead(200, {'Content-Type' : 'application/json', 'total' : total})
+        if request.method is 'HEAD'
+          response.end()
+          return
+        response.write("[\n")
+        i = 0
+        cur.each((err, item) ->
+          if err
+            throwError(err)
+            return false
+
+          if item?
+            response.write(",\n") if i++
+            response.write(JSON.stringify(item))
+
+          if not item? or (limit? and i >= limit)
+            response.end("\n]")
+        )
+      )
+
+    else
+      response.writeHead 404
+      response.end('404 Not Found')
+
+
+exports.listener = listener

lib/nbi.ts

@@ -1,827 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as url from "url";
-import { Collection, GridFSBucket, ObjectId } from "mongodb";
-import * as querystring from "querystring";
-import * as vm from "vm";
-import * as config from "./config";
-import { onConnect } from "./db";
-import * as query from "./query";
-import * as apiFunctions from "./api-functions";
-import { IncomingMessage, ServerResponse } from "http";
-import * as cache from "./cache";
-import { version as VERSION } from "../package.json";
-import { ping } from "./ping";
-import * as logger from "./logger";
-
-const DEVICE_TASKS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tasks\/?$/;
-const TASKS_REGEX = /^\/tasks\/([a-zA-Z0-9\-_%]+)(\/[a-zA-Z_]*)?$/;
-const TAGS_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/tags\/([a-zA-Z0-9\-_%]+)\/?$/;
-const PRESETS_REGEX = /^\/presets\/([a-zA-Z0-9\-_%]+)\/?$/;
-const OBJECTS_REGEX = /^\/objects\/([a-zA-Z0-9\-_%]+)\/?$/;
-const FILES_REGEX = /^\/files\/([a-zA-Z0-9%!*'();:@&=+$,?#[\]\-_.~]+)\/?$/;
-const PING_REGEX = /^\/ping\/([a-zA-Z0-9\-_.:]+)\/?$/;
-const QUERY_REGEX = /^\/([a-zA-Z0-9_]+)\/?$/;
-const DELETE_DEVICE_REGEX = /^\/devices\/([a-zA-Z0-9\-_%]+)\/?$/;
-const PROVISIONS_REGEX = /^\/provisions\/([a-zA-Z0-9\-_%]+)\/?$/;
-const VIRTUAL_PARAMETERS_REGEX = /^\/virtual_parameters\/([a-zA-Z0-9\-_%]+)\/?$/;
-const FAULTS_REGEX = /^\/faults\/([a-zA-Z0-9\-_%:]+)\/?$/;
-
-const collections = {
-  tasks: null as Collection,
-  devices: null as Collection,
-  presets: null as Collection,
-  objects: null as Collection,
-  files: null as Collection,
-  provisions: null as Collection,
-  virtualParameters: null as Collection,
-  faults: null as Collection,
-};
-let filesBucket: GridFSBucket;
-
-onConnect(async (db) => {
-  for (const k of Object.keys(collections)) {
-    if (k === "files") collections[k] = db.collection("fs.files");
-    else collections[k] = db.collection(k);
-  }
-  filesBucket = new GridFSBucket(db);
-});
-
-function throwError(err: Error, httpResponse?: ServerResponse): never {
-  if (httpResponse) {
-    httpResponse.writeHead(500, { Connection: "close" });
-    httpResponse.end(`${err.name}: ${err.message}`);
-  }
-  throw err;
-}
-
-export function listener(
-  request: IncomingMessage,
-  response: ServerResponse
-): void {
-  const chunks = [];
-  let bytes = 0;
-  response.setHeader("GenieACS-Version", VERSION);
-
-  request.addListener("data", (chunk): void => {
-    chunks.push(chunk);
-    bytes += chunk.length;
-  });
-
-  function getBody(): Buffer {
-    // Write all chunks into a Buffer
-    const body = Buffer.allocUnsafe(bytes);
-    let offset = 0;
-    for (const chunk of chunks) {
-      chunk.copy(body, offset, 0, chunk.length);
-      offset += chunk.length;
-    }
-    return body;
-  }
-
-  request.addListener("end", () => {
-    const body = getBody();
-    const urlParts = url.parse(request.url, true);
-
-    logger.accessInfo(
-      Object.assign({}, urlParts.query, {
-        remoteAddress: request.connection.remoteAddress,
-        message: `${request.method} ${urlParts.pathname}`,
-      })
-    );
-
-    if (PRESETS_REGEX.test(urlParts.pathname)) {
-      const presetName = querystring.unescape(
-        PRESETS_REGEX.exec(urlParts.pathname)[1]
-      );
-      if (request.method === "PUT") {
-        const preset = JSON.parse(body.toString());
-        preset._id = presetName;
-        collections.presets.replaceOne(
-          { _id: presetName },
-          preset,
-          { upsert: true },
-          (err) => {
-            if (err) return void throwError(err, response);
-
-            cache
-              .del("presets_hash")
-              .then(() => {
-                response.writeHead(200);
-                response.end();
-              })
-              .catch((err) => {
-                setTimeout(() => {
-                  throwError(err, response);
-                });
-              });
-          }
-        );
-      } else if (request.method === "DELETE") {
-        collections.presets.deleteOne({ _id: presetName }, (err) => {
-          if (err) return void throwError(err, response);
-
-          cache
-            .del("presets_hash")
-            .then(() => {
-              response.writeHead(200);
-              response.end();
-            })
-            .catch((err) => {
-              setTimeout(() => {
-                throwError(err, response);
-              });
-            });
-        });
-      } else {
-        response.writeHead(405, { Allow: "PUT, DELETE" });
-        response.end("405 Method Not Allowed");
-      }
-    } else if (OBJECTS_REGEX.test(urlParts.pathname)) {
-      const objectName = querystring.unescape(
-        OBJECTS_REGEX.exec(urlParts.pathname)[1]
-      );
-      if (request.method === "PUT") {
-        const object = JSON.parse(body.toString());
-        object._id = objectName;
-        collections.objects.replaceOne(
-          { _id: objectName },
-          object,
-          { upsert: true },
-          (err) => {
-            if (err) return void throwError(err, response);
-
-            cache
-              .del("presets_hash")
-              .then(() => {
-                response.writeHead(200);
-                response.end();
-              })
-              .catch((err) => {
-                setTimeout(() => {
-                  throwError(err, response);
-                });
-              });
-          }
-        );
-      } else if (request.method === "DELETE") {
-        collections.objects.deleteOne({ _id: objectName }, (err) => {
-          if (err) return void throwError(err, response);
-
-          cache
-            .del("presets_hash")
-            .then(() => {
-              response.writeHead(200);
-              response.end();
-            })
-            .catch((err) => {
-              setTimeout(() => {
-                throwError(err, response);
-              });
-            });
-        });
-      } else {
-        response.writeHead(405, { Allow: "PUT, DELETE" });
-        response.end("405 Method Not Allowed");
-      }
-    } else if (PROVISIONS_REGEX.test(urlParts.pathname)) {
-      const provisionName = querystring.unescape(
-        PROVISIONS_REGEX.exec(urlParts.pathname)[1]
-      );
-      if (request.method === "PUT") {
-        const object = {
-          _id: provisionName,
-          script: body.toString(),
-        };
-
-        try {
-          new vm.Script(`"use strict";(function(){\n${object.script}\n})();`);
-        } catch (err) {
-          response.writeHead(400);
-          response.end(`${err.name}: ${err.message}`);
-          return;
-        }
-
-        collections.provisions.replaceOne(
-          { _id: provisionName },
-          object,
-          { upsert: true },
-          (err) => {
-            if (err) return void throwError(err, response);
-
-            cache
-              .del("presets_hash")
-              .then(() => {
-                response.writeHead(200);
-                response.end();
-              })
-              .catch((err) => {
-                setTimeout(() => {
-                  throwError(err, response);
-                });
-              });
-          }
-        );
-      } else if (request.method === "DELETE") {
-        collections.provisions.deleteOne({ _id: provisionName }, (err) => {
-          if (err) return void throwError(err, response);
-
-          cache
-            .del("presets_hash")
-            .then(() => {
-              response.writeHead(200);
-              response.end();
-            })
-            .catch((err) => {
-              setTimeout(() => {
-                throwError(err, response);
-              });
-            });
-        });
-      } else {
-        response.writeHead(405, { Allow: "PUT, DELETE" });
-        response.end("405 Method Not Allowed");
-      }
-    } else if (VIRTUAL_PARAMETERS_REGEX.test(urlParts.pathname)) {
-      const virtualParameterName = querystring.unescape(
-        VIRTUAL_PARAMETERS_REGEX.exec(urlParts.pathname)[1]
-      );
-      if (request.method === "PUT") {
-        const object = {
-          _id: virtualParameterName,
-          script: body.toString(),
-        };
-
-        try {
-          new vm.Script(`"use strict";(function(){\n${object.script}\n})();`);
-        } catch (err) {
-          response.writeHead(400);
-          response.end(`${err.name}: ${err.message}`);
-          return;
-        }
-
-        collections.virtualParameters.replaceOne(
-          { _id: virtualParameterName },
-          object,
-          { upsert: true },
-          (err) => {
-            if (err) return void throwError(err, response);
-
-            cache
-              .del("presets_hash")
-              .then(() => {
-                response.writeHead(200);
-                response.end();
-              })
-              .catch((err) => {
-                setTimeout(() => {
-                  throwError(err, response);
-                });
-              });
-          }
-        );
-      } else if (request.method === "DELETE") {
-        collections.virtualParameters.deleteOne(
-          { _id: virtualParameterName },
-          (err) => {
-            if (err) return void throwError(err, response);
-
-            cache
-              .del("presets_hash")
-              .then(() => {
-                response.writeHead(200);
-                response.end();
-              })
-              .catch((err) => {
-                setTimeout(() => {
-                  throwError(err, response);
-                });
-              });
-          }
-        );
-      } else {
-        response.writeHead(405, { Allow: "PUT, DELETE" });
-        response.end("405 Method Not Allowed");
-      }
-    } else if (TAGS_REGEX.test(urlParts.pathname)) {
-      const r = TAGS_REGEX.exec(urlParts.pathname);
-      const deviceId = querystring.unescape(r[1]);
-      const tag = querystring.unescape(r[2]);
-      if (request.method === "POST") {
-        collections.devices.updateOne(
-          { _id: deviceId },
-          { $addToSet: { _tags: tag } },
-          (err) => {
-            if (err) return void throwError(err, response);
-            response.writeHead(200);
-            response.end();
-          }
-        );
-      } else if (request.method === "DELETE") {
-        collections.devices.updateOne(
-          { _id: deviceId },
-          { $pull: { _tags: tag } },
-          (err) => {
-            if (err) return void throwError(err, response);
-
-            response.writeHead(200);
-            response.end();
-          }
-        );
-      } else {
-        response.writeHead(405, { Allow: "POST, DELETE" });
-        response.end("405 Method Not Allowed");
-      }
-    } else if (FAULTS_REGEX.test(urlParts.pathname)) {
-      if (request.method === "DELETE") {
-        const faultId = querystring.unescape(
-          FAULTS_REGEX.exec(urlParts.pathname)[1]
-        );
-        const deviceId = faultId.split(":", 1)[0];
-        const channel = faultId.slice(deviceId.length + 1);
-        collections.faults.deleteOne({ _id: faultId }, (err) => {
-          if (err) return void throwError(err, response);
-
-          if (channel.startsWith("task_")) {
-            const objId = new ObjectId(channel.slice(5));
-            return void collections.tasks.deleteOne({ _id: objId }, (err) => {
-              if (err) return void throwError(err, response);
-
-              cache
-                .del(`${deviceId}_tasks_faults_operations`)
-                .then(() => {
-                  response.writeHead(200);
-                  response.end();
-                })
-                .catch((err) => {
-                  setTimeout(() => {
-                    throwError(err, response);
-                  });
-                });
-            });
-          }
-
-          cache
-            .del(`${deviceId}_tasks_faults_operations`)
-            .then(() => {
-              response.writeHead(200);
-              response.end();
-            })
-            .catch((err) => {
-              setTimeout(() => {
-                throwError(err, response);
-              });
-            });
-        });
-      } else {
-        response.writeHead(405, { Allow: "DELETE" });
-        response.end("405 Method Not Allowed");
-      }
-    } else if (DEVICE_TASKS_REGEX.test(urlParts.pathname)) {
-      if (request.method === "POST") {
-        const deviceId = querystring.unescape(
-          DEVICE_TASKS_REGEX.exec(urlParts.pathname)[1]
-        );
-        if (body.length) {
-          const task = JSON.parse(body.toString());
-          task.device = deviceId;
-          apiFunctions
-            .insertTasks(task)
-            .then(async () => {
-              const lastInform = Date.now();
-
-              const socketTimeout: number = request.socket["timeout"];
-
-              // Disable socket timeout while waiting for session
-              if (socketTimeout) request.socket.setTimeout(0);
-
-              const notInSession = await apiFunctions.awaitSessionEnd(
-                deviceId,
-                30000
-              );
-              await cache.del(`${deviceId}_tasks_faults_operations`);
-              if (urlParts.query.connection_request == null) {
-                if (socketTimeout) request.socket.setTimeout(socketTimeout);
-                response.writeHead(202, {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(task));
-                return;
-              }
-
-              if (!notInSession) {
-                if (socketTimeout) request.socket.setTimeout(socketTimeout);
-                response.writeHead(202, "Task queued but not processed", {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(task));
-                return;
-              }
-
-              const status = await apiFunctions.connectionRequest(deviceId);
-
-              if (status) {
-                if (socketTimeout) request.socket.setTimeout(socketTimeout);
-                response.writeHead(202, status, {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(task));
-                return;
-              }
-
-              const onlineThreshold =
-                (urlParts.query.timeout &&
-                  parseInt(urlParts.query.timeout as string)) ||
-                (config.get("DEVICE_ONLINE_THRESHOLD", deviceId) as number);
-
-              const sessionStarted = await apiFunctions.awaitSessionStart(
-                deviceId,
-                lastInform,
-                onlineThreshold
-              );
-              if (!sessionStarted) {
-                if (socketTimeout) request.socket.setTimeout(socketTimeout);
-                response.writeHead(202, "Task queued but not processed", {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(task));
-                return;
-              }
-
-              const sessionEnded = await apiFunctions.awaitSessionEnd(
-                deviceId,
-                120000
-              );
-              if (!sessionEnded) {
-                if (socketTimeout) request.socket.setTimeout(socketTimeout);
-                response.writeHead(202, "Task queued but not processed", {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(task));
-                return;
-              }
-
-              const prom1 = collections.tasks.findOne(
-                { _id: task._id },
-                { projection: { _id: 1 } }
-              );
-              const prom2 = collections.faults.findOne(
-                { _id: `${deviceId}:task_${task._id}` },
-                {
-                  projection: { _id: 1 },
-                }
-              );
-
-              const [t, f] = await Promise.all([prom1, prom2]);
-
-              // Restore socket timeout
-              if (socketTimeout) request.socket.setTimeout(socketTimeout);
-
-              if (f) {
-                response.writeHead(202, "Task faulted", {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(t || task));
-              } else if (t) {
-                response.writeHead(202, "Task queued but not processed", {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(t));
-              } else {
-                response.writeHead(200, {
-                  "Content-Type": "application/json",
-                });
-                response.end(JSON.stringify(task));
-              }
-            })
-            .catch((err) => {
-              setTimeout(() => {
-                throwError(err, response);
-              });
-            });
-        } else if (urlParts.query.connection_request != null) {
-          // No task, send connection request only
-          apiFunctions
-            .connectionRequest(deviceId)
-            .then(() => {
-              response.writeHead(200);
-              response.end();
-            })
-            .catch((err) => {
-              response.writeHead(504);
-              response.end(`${err.name}: ${err.message}`);
-            });
-        } else {
-          response.writeHead(400);
-          response.end();
-        }
-      } else {
-        response.writeHead(405, { Allow: "POST" });
-        response.end("405 Method Not Allowed");
-      }
-    } else if (TASKS_REGEX.test(urlParts.pathname)) {
-      const r = TASKS_REGEX.exec(urlParts.pathname);
-      const taskId = querystring.unescape(r[1]);
-      const action = r[2];
-      if (!action || action === "/") {
-        if (request.method === "DELETE") {
-          collections.tasks.findOne(
-            { _id: new ObjectId(taskId) },
-            { projection: { device: 1 } },
-            (err, task) => {
-              if (err) return void throwError(err, response);
-
-              if (!task) {
-                response.writeHead(404);
-                response.end("Task not found");
-                return;
-              }
-
-              const deviceId = task.device;
-              collections.tasks.deleteOne(
-                { _id: new ObjectId(taskId) },
-                (err) => {
-                  if (err) return void throwError(err, response);
-
-                  collections.faults.deleteOne(
-                    { _id: `${deviceId}:task_${taskId}` },
-                    (err) => {
-                      if (err) return void throwError(err, response);
-
-                      cache
-                        .del(`${deviceId}_tasks_faults_operations`)
-                        .then(() => {
-                          response.writeHead(200);
-                          response.end();
-                        })
-                        .catch((err) => {
-                          setTimeout(() => {
-                            throwError(err, response);
-                          });
-                        });
-                    }
-                  );
-                }
-              );
-            }
-          );
-        } else {
-          response.writeHead(405, { Allow: "PUT DELETE" });
-          response.end("405 Method Not Allowed");
-        }
-      } else if (action === "/retry") {
-        if (request.method === "POST") {
-          collections.tasks.findOne(
-            { _id: new ObjectId(taskId) },
-            { projection: { device: 1 } },
-            (err, task) => {
-              if (err) return void throwError(err, response);
-
-              const deviceId = task.device;
-              collections.faults.deleteOne(
-                { _id: `${deviceId}:task_${taskId}` },
-                (err) => {
-                  if (err) return void throwError(err, response);
-
-                  cache
-                    .del(`${deviceId}_tasks_faults_operations`)
-                    .then(() => {
-                      response.writeHead(200);
-                      response.end();
-                    })
-                    .catch((err) => {
-                      setTimeout(() => {
-                        throwError(err, response);
-                      });
-                    });
-                }
-              );
-            }
-          );
-        } else {
-          response.writeHead(405, { Allow: "POST" });
-          response.end("405 Method Not Allowed");
-        }
-      } else {
-        response.writeHead(404);
-        response.end();
-      }
-    } else if (FILES_REGEX.test(urlParts.pathname)) {
-      const filename = querystring.unescape(
-        FILES_REGEX.exec(urlParts.pathname)[1]
-      );
-      if (request.method === "PUT") {
-        const metadata = {
-          fileType: request.headers.filetype,
-          oui: request.headers.oui,
-          productClass: request.headers.productclass,
-          version: request.headers.version,
-        };
-        filesBucket.delete((filename as unknown) as ObjectId, () => {
-          const uploadStream = filesBucket.openUploadStreamWithId(
-            filename,
-            filename,
-            {
-              metadata: metadata,
-            }
-          );
-
-          uploadStream.on("error", (err) => {
-            throwError(err, response);
-          });
-
-          uploadStream.end(body, () => {
-            response.writeHead(201);
-            response.end();
-          });
-        });
-      } else if (request.method === "DELETE") {
-        filesBucket.delete((filename as unknown) as ObjectId, (err) => {
-          if (err) {
-            if (err.message.startsWith("FileNotFound")) {
-              response.writeHead(404);
-              response.end("404 Not Found");
-              return;
-            }
-            return void throwError(err, response);
-          }
-
-          response.writeHead(200);
-          response.end();
-        });
-      } else {
-        response.writeHead(405, { Allow: "PUT, DELETE" });
-        response.end("405 Method Not Allowed");
-      }
-    } else if (PING_REGEX.test(urlParts.pathname)) {
-      const host = querystring.unescape(PING_REGEX.exec(urlParts.pathname)[1]);
-      ping(host, (err, res, stdout) => {
-        if (err) {
-          if (!res) {
-            response.writeHead(500, { Connection: "close" });
-            response.end(`${err.name}: ${err.message}`);
-            return;
-          }
-          response.writeHead(404, { "Cache-Control": "no-cache" });
-          response.end(`${err.name}: ${err.message}`);
-          return;
-        }
-
-        response.writeHead(200, {
-          "Content-Type": "text/plain",
-          "Cache-Control": "no-cache",
-        });
-        response.end(stdout);
-      });
-    } else if (DELETE_DEVICE_REGEX.test(urlParts.pathname)) {
-      if (request.method !== "DELETE") {
-        response.writeHead(405, { Allow: "DELETE" });
-        response.end("405 Method Not Allowed");
-        return;
-      }
-
-      const deviceId = querystring.unescape(
-        DELETE_DEVICE_REGEX.exec(urlParts.pathname)[1]
-      );
-      apiFunctions
-        .deleteDevice(deviceId)
-        .then(() => {
-          response.writeHead(200);
-          response.end();
-        })
-        .catch((err) => {
-          setTimeout(() => {
-            throwError(err, response);
-          });
-        });
-    } else if (QUERY_REGEX.test(urlParts.pathname)) {
-      let collectionName = QUERY_REGEX.exec(urlParts.pathname)[1];
-
-      // Convert to camel case
-      let i = collectionName.indexOf("_");
-      while (i++ >= 0) {
-        const up =
-          i < collectionName.length ? collectionName[i].toUpperCase() : "";
-        collectionName =
-          collectionName.slice(0, i - 1) + up + collectionName.slice(i + 1);
-        i = collectionName.indexOf("_", i);
-      }
-
-      if (request.method !== "GET" && request.method !== "HEAD") {
-        response.writeHead(405, { Allow: "GET, HEAD" });
-        response.end("405 Method Not Allowed");
-        return;
-      }
-
-      const collection = collections[collectionName];
-      if (!collection) {
-        response.writeHead(404);
-        response.end("404 Not Found");
-        return;
-      }
-
-      let q = {};
-      if (urlParts.query.query) {
-        try {
-          q = JSON.parse(urlParts.query.query as string);
-        } catch (err) {
-          response.writeHead(400);
-          response.end(`${err.name}: ${err.message}`);
-          return;
-        }
-      }
-
-      switch (collectionName) {
-        case "devices":
-          q = query.expand(q);
-          break;
-        case "tasks":
-          q = query.sanitizeQueryTypes(q, {
-            _id: (v) => new ObjectId(v as string),
-            timestamp: (v) => new Date(v as number),
-            retries: Number,
-          });
-          break;
-        case "faults":
-          q = query.sanitizeQueryTypes(q, {
-            timestamp: (v) => new Date(v as number),
-            retries: Number,
-          });
-      }
-
-      let projection = null;
-      if (urlParts.query.projection) {
-        projection = {};
-        for (const p of (urlParts.query.projection as string).split(","))
-          projection[p.trim()] = 1;
-      }
-
-      const cur = collection.find(q, { projection: projection });
-
-      if (urlParts.query.sort) {
-        const s = JSON.parse(urlParts.query.sort as string);
-        const sort = {};
-        for (const [k, v] of Object.entries(s)) {
-          if (k[k.lastIndexOf(".") + 1] !== "_" && collectionName === "devices")
-            sort[`${k}._value`] = v;
-          else sort[k] = v;
-        }
-
-        cur.sort(sort);
-      }
-
-      if (urlParts.query.skip)
-        cur.skip(parseInt(urlParts.query.skip as string));
-
-      if (urlParts.query.limit)
-        cur.limit(parseInt(urlParts.query.limit as string));
-
-      cur.count(false, (err, total) => {
-        if (err) return void throwError(err);
-
-        response.writeHead(200, {
-          "Content-Type": "application/json",
-          total: total,
-        });
-
-        if (request.method === "HEAD") {
-          response.end();
-          return;
-        }
-
-        response.write("[\n");
-        i = 0;
-        cur.forEach(
-          (item) => {
-            if (i++) response.write(",\n");
-            response.write(JSON.stringify(item));
-          },
-          (err) => {
-            if (err) return void throwError(err);
-            response.end("\n]");
-          }
-        );
-      });
-    } else {
-      response.writeHead(404);
-      response.end("404 Not Found");
-    }
-  });
-}

lib/path-set.js

@@ -0,0 +1,179 @@
+/**
+ * Copyright 2013-2017  Zaid Abdulla
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+"use strict";
+
+
+class PathSet {
+
+  constructor() {
+    this.lengthIndex = [];
+    this.fragmentIndex = [];
+  }
+
+
+  get depth() {
+    return this.lengthIndex.length;
+  }
+
+
+  add(path) {
+    let p = this.get(path);
+
+    if (p)
+      return p;
+
+    if (!this.lengthIndex[path.length])
+      this.lengthIndex[path.length] = new Set();
+
+    this.lengthIndex[path.length].add(path);
+
+    let wildcard = 0;
+    for (let i = 0; i < path.length; ++ i) {
+      let fragment = path[i];
+      if (fragment === "*")
+        wildcard = wildcard | (1 << i);
+
+      if (!this.fragmentIndex[i])
+        this.fragmentIndex[i] = {};
+
+      if (!this.fragmentIndex[i][fragment])
+        this.fragmentIndex[i][fragment] = new Set();
+
+      this.fragmentIndex[i][fragment].add(path);
+    }
+
+    if (path.wildcard == null) {
+      path.alias = 0;
+      path.wildcard = wildcard;
+      Object.freeze(path);
+    }
+    return path;
+  }
+
+
+  get(pattern) {
+    if (!this.lengthIndex[pattern.length])
+      return null;
+
+    if (this.lengthIndex[pattern.length].has(pattern))
+      return pattern;
+
+    let sets = [];
+
+    sets.push(this.lengthIndex[pattern.length]);
+
+    for (let i = 0; i < pattern.length; ++ i) {
+      if (!this.fragmentIndex[i])
+        return null;
+
+      let fragment = pattern[i];
+
+      if (!this.fragmentIndex[i][fragment])
+        return null;
+
+      sets.push(this.fragmentIndex[i][fragment]);
+    }
+
+    sets.sort(function(a, b) {
+      return a.size - b.size;
+    });
+
+    let smallestSet = sets.shift();
+    for (let path of smallestSet) {
+      for (let s of sets) {
+        if (!s.has(path)) {
+          path = null;
+          break
+        }
+      }
+      if (path)
+        return path;
+    }
+
+    return null;
+  }
+
+
+  find(pattern, superset = false, subset = false, depth = pattern.length) {
+    let res = [];
+    let groups = [[]];
+
+    for (let i = pattern.length; i <= depth; ++ i)
+      if (this.lengthIndex[i])
+        groups[0].push(this.lengthIndex[i]);
+
+    if (!groups[0].length)
+      return res;
+
+    for (let i = 0; i < pattern.length; ++ i) {
+      if (!this.fragmentIndex[i])
+        return res;
+
+      let fragment = pattern[i];
+
+      if (fragment === "*" && subset)
+        continue;
+
+      let g = [];
+
+      if (this.fragmentIndex[i][fragment])
+        g.push(this.fragmentIndex[i][fragment]);
+
+      if (fragment !== "*" && superset && this.fragmentIndex[i]["*"])
+        g.push(this.fragmentIndex[i]["*"]);
+
+      if (!g.length)
+        return res;
+
+      groups.push(g);
+    }
+
+    groups.sort(function(a, b) {
+      return a.reduce(function(prev, cur) {return prev + cur.size;}, 0) -
+        b.reduce(function(prev, cur) {return prev + cur.size;}, 0);
+    });
+
+    let smallestGroup = groups.shift();
+    for (let set of smallestGroup) {
+      for (let path of set) {
+        for (let group of groups) {
+          let found = false;
+          for (let s of group) {
+            if (s.has(path)) {
+              found = true;
+              break;
+            }
+          }
+          if (!found) {
+            path = null;
+            break;
+          }
+        }
+        if (path)
+          res.push(path);
+      }
+    }
+
+    return res;
+  }
+
+}
+
+
+module.exports = PathSet;

lib/ping.ts

@@ -1,90 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { platform } from "os";
-import { exec } from "child_process";
-
-export interface PingResult {
-  packetsTransmitted: number;
-  packetsReceived: number;
-  packetLoss: number;
-  min: number;
-  avg: number;
-  max: number;
-  mdev: number;
-}
-
-export function ping(
-  host: string,
-  callback: (err: Error, res?: PingResult, stdout?: string) => void
-): void {
-  let cmd: string, parseRegExp1: RegExp, parseRegExp2: RegExp;
-  host = host.replace("[", "").replace("]", "");
-  switch (platform()) {
-    case "linux":
-      cmd = `ping -w 1 -i 0.2 -c 3 ${host}`;
-      parseRegExp1 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss[^]*([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+)/;
-      parseRegExp2 = /(\d+) packets transmitted, (\d+) received, ([\d.]+)% packet loss/;
-      break;
-
-    case "freebsd":
-      // Send a single packet because on FreeBSD only superuser can send
-      // packets that are only 200 ms apart.
-      cmd = `ping -t 1 -c 3 ${host}`;
-      parseRegExp1 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss\nround-trip min\/avg\/max\/stddev = ([\d.]+)\/([\d.]+)\/([\d.]+)\/([\d.]+) ms/;
-      parseRegExp2 = /(\d+) packets transmitted, (\d+) packets received, ([\d.]+)% packet loss/;
-      break;
-
-    default:
-      return callback(new Error("Platform not supported"));
-  }
-
-  exec(cmd, (err, stdout) => {
-    let parsed: PingResult;
-    if (stdout) {
-      const m1 = stdout.match(parseRegExp1);
-      if (m1) {
-        parsed = {
-          packetsTransmitted: +m1[1],
-          packetsReceived: +m1[2],
-          packetLoss: +m1[3],
-          min: +m1[4],
-          avg: +m1[5],
-          max: +m1[6],
-          mdev: +m1[7],
-        };
-      } else {
-        const m2 = stdout.match(parseRegExp2);
-        if (m2) {
-          parsed = {
-            packetsTransmitted: +m2[1],
-            packetsReceived: +m2[2],
-            packetLoss: +m2[3],
-            min: null,
-            avg: null,
-            max: null,
-            mdev: null,
-          };
-        }
-      }
-    }
-
-    callback(err, parsed, stdout);
-  });
-}

lib/query.coffee

@@ -0,0 +1,372 @@
+###
+# Copyright 2013-2017  Zaid Abdulla
+#
+# This file is part of GenieACS.
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This file incorporates work covered by the following copyright and
+# permission notice:
+#
+# Copyright 2013 Fanoos Telecom
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+###
+
+common = require './common'
+mongodb = require 'mongodb'
+
+
+stringToRegexp = (input, flags) ->
+  if (input.indexOf('*') == -1)
+    return false
+
+  output = input.replace(/[\[\]\\\^\$\.\|\?\+\(\)]/, "\\$&")
+  if output[0] == '*'
+    output = output.replace(/^\*+/g, '')
+  else
+    output = '^' + output
+
+  if output[output.length - 1] == '*'
+    output = output.replace(/\*+$/g, '')
+  else
+    output = output + '$'
+
+  output = output.replace(/[\*]/, '.*')
+
+  return new RegExp(output, flags)
+
+
+normalize = (input) ->
+  if common.typeOf(input) is common.STRING_TYPE
+    vals = [input]
+    if (m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input))
+      vals.push({'$regex' : new RegExp(m[1], m[2])})
+
+    if +input == parseFloat(input)
+      vals.push(+input)
+
+    d = new Date(input)
+    if input.length >= 8 and d.getFullYear() > 1983
+      vals.push(d)
+
+    r = stringToRegexp(input)
+    if r isnt false
+      vals.push({'$regex' : r})
+
+    return vals
+  return input
+
+
+expandValue = (value) ->
+  if common.typeOf(value) is common.ARRAY_TYPE
+    a = []
+    for j in value
+      a = a.concat(expandValue(j))
+    return [a]
+  else if common.typeOf(value) isnt common.OBJECT_TYPE
+    n = normalize(value)
+    if common.typeOf(n) isnt common.ARRAY_TYPE
+      return [n]
+    else
+      return n
+
+  objs = []
+  indices = []
+  keys = []
+  values = []
+  for k,v of value
+    keys.push(k)
+    values.push(expandValue(v))
+    indices.push(0)
+
+  i = 0
+  while i < indices.length
+    obj = {}
+    for i in [0...keys.length]
+      obj[keys[i]] = values[i][indices[i]]
+    objs.push(obj)
+
+    for i in [0...indices.length]
+      indices[i] += 1
+      if indices[i] < values[i].length
+        break
+      indices[i] = 0
+  return objs
+
+
+permute = (param, val) ->
+  conditions = []
+
+  values = expandValue(val)
+  if param[param.lastIndexOf('.') + 1] != '_'
+    param += '._value'
+
+  for v in values
+    obj = {}
+    obj[param] = v
+    conditions.push(obj)
+
+  return conditions
+
+
+expand = (query) ->
+  new_query = {}
+  for k,v of query
+    if k[0] == '$' # operator
+      expressions = []
+      for e in v
+        expressions.push(expand(e))
+      new_query[k] = expressions
+    else
+      conditions = permute(k, v)
+      if conditions.length > 1
+        new_query['$and'] ?= []
+        if v?['$ne']? or v?['$not']?
+          if Object.keys(v) > 1
+            throw new Error("Cannot mix $ne or $not with other operators")
+          for c in conditions
+            new_query['$and'].push(c)
+        else
+          new_query['$and'].push({'$or' : conditions})
+      else
+        Object.assign(new_query, conditions[0])
+
+  return new_query
+
+
+# Generate parameter projection from a query
+# If second arg is given, it's edited and returned
+queryProjection = (query, proj) ->
+  proj ?= {}
+  for k,v of query
+    if k.charAt(0) == '$' # this is a logical operator
+      for q in v
+        queryProjection(q, proj)
+    else
+      proj[k] = 1
+  return proj
+
+
+testExpressions = (params, expressions, lop) ->
+  for f in expressions
+    res = test(params, f)
+
+    switch lop
+      when '$and'
+        return false if not res
+      when '$or'
+        return true if res
+      when '$nor'
+        return false if res
+
+  return switch lop
+    when '$and' then true
+    when '$or' then false
+    when '$nor' then true
+    else throw new Error('Unknown logical operator')
+
+
+test = (params, query) ->
+  for k,v of query
+    if k.charAt(0) == '$' # this is a logical operator
+      res = testExpressions(params, v, k)
+    else
+      value = params[k]
+
+      if common.typeOf(v) isnt common.OBJECT_TYPE
+        # TODO comparing array to regex, array to array, and object to object
+        if common.typeOf(value) is common.ARRAY_TYPE
+          res = value.indexOf(v) != -1
+        else
+          if common.typeOf(v) is common.REGEXP_TYPE
+            res = v.test(value)
+          else
+            res = v == value
+      else
+        for k2,v2 of v
+          switch k2
+            when '$ne'
+              if common.typeOf(value) is common.ARRAY_TYPE
+                res = value.indexOf(v2) == -1
+              else
+                res = value != v2
+            when '$lt'
+              res = value < v2
+            when '$lte'
+              res = value <= v2
+            when '$gt'
+              res = value > v2
+            when '$gte'
+              res = value >= v2
+            when '$regex'
+              res = v2.test(value)
+            when '$in'
+              throw new Error('Operator not supported')
+            when '$nin'
+              throw new Error('Operator not supported')
+            when '$all'
+              throw new Error('Operator not supported')
+            when '$exists'
+              throw new Error('Operator not supported')
+            else
+              throw new Error('Operator not supported')
+
+    if not res
+      return false
+
+  return true
+
+
+matchType = (src, dst) ->
+  if typeof src is 'string'
+    return String(dst)
+  else if typeof src is 'number'
+    if +dst == parseFloat(dst)
+      return +dst
+    else if not isNaN(Date.parse(dst))
+      return Date.parse(dst)
+  else if typeof src is 'boolean'
+    v = String(dst).trim()
+    if v is 'true' or v is 'TRUE' or v is 'True' or v is '1'
+      return true
+    else if v is 'false' or v is 'FALSE' or v is 'False' or v is '0'
+      return false
+
+  return dst
+
+
+testFilter = (obj, filter) ->
+  for k, v of filter
+    [param, op] = k.split(/([^a-zA-Z0-9\-\_\.].*)/, 2)
+    val = matchType(obj[param], v)
+    switch op
+      when '=', undefined
+        return false if obj[param] != val
+      when '>'
+        return false if not (obj[param] > val)
+      when '>='
+        return false if not (obj[param] >= val)
+      when '<'
+        return false if not (obj[param] < val)
+      when '<='
+        return false if not (obj[param] <= val)
+      when '!'
+        return false if obj[param] == val
+      else
+        throw new Error("Unrecognized operator #{op}")
+
+  return true
+
+
+convertMongoQueryToFilters = (query, filters) ->
+  filters ?= {}
+  for k, v of query
+    if k[0] == '$'
+      if k == '$and'
+        for vv in v
+          convertMongoQueryToFilters(vv, filters)
+      else
+        throw new Error("Operator #{k} not supported")
+    else if k == '_tags'
+      if common.typeOf(v) is common.OBJECT_TYPE
+        for kk, vv of v
+          vv = vv.replace(/[^a-zA-Z0-9\-]+/g, '_')
+          switch kk
+            when '$ne'
+              filters["Tags.#{vv}!"] = true
+            else
+              throw new Error("Operator #{kk} not supported")
+      else
+        v = v.replace(/[^a-zA-Z0-9\-]+/g, '_')
+        filters["Tags.#{v}"] = true
+    else
+      switch k
+        when '_id' then k = 'DeviceID.ID'
+        when '_deviceId._Manufacturer' then k = 'DeviceID.Manufacturer'
+        when '_deviceId._OUI' then k = 'DeviceID.OUI'
+        when '_deviceId._ProductClass' then k = 'DeviceID.ProductClass'
+        when '_deviceId._SerialNumber' then k = 'DeviceID.SerialNumber'
+        when '_lastInform' then k = 'Events.Inform'
+        when '_lastBootstrap' then k = 'Events.0_BOOTSTRAP'
+        when '_lastBoot' then k = 'Events.1_BOOT'
+        when '_registered' then k = 'Events.Registered'
+
+      if common.typeOf(v) is common.OBJECT_TYPE
+        for kk, vv of v
+          switch kk
+            when '$eq'
+              filters[k] = vv
+            when '$ne'
+              filters["#{k}!"] = vv
+            when '$lt'
+              filters["#{k}<"] = vv
+            when '$lte'
+              filters["#{k}<="] = vv
+            when '$gt'
+              filters["#{k}>"] = vv
+            when '$gte'
+              filters["#{k}>="] = vv
+            else
+              throw new Error("Oprator #{kk} not supported")
+      else
+        filters[k] = v
+
+  return filters
+
+
+sanitizeQueryTypes = (query, types) ->
+  for k, v of query
+    if k[0] == '$' # logical operator
+      sanitizeQueryTypes(vv, types) for vv in v
+    else if k of types
+      if common.typeOf(v) is common.OBJECT_TYPE
+        for kk, vv of v
+          switch kk # operator
+            when '$in', '$nin'
+              vv[i] = types[k](vv[i]) for i in [0...vv.length]
+            when '$eq', '$gt', '$gte', '$lt', '$lte', '$ne'
+              v[kk] = types[k](vv)
+            when '$exists', '$type' then
+              # ignore
+            else
+              throw new Error('Operator not supported')
+      else
+        query[k] = types[k](query[k])
+
+  return query
+
+
+exports.expand = expand
+exports.queryProjection = queryProjection
+exports.test = test
+exports.sanitizeQueryTypes = sanitizeQueryTypes
+exports.convertMongoQueryToFilters = convertMongoQueryToFilters
+exports.testFilter = testFilter

lib/query.ts

@@ -1,175 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-function isObject(obj: any): boolean {
-  return Object.prototype.toString.call(obj) === "[object Object]";
-}
-
-function stringToRegexp(input, flags?): RegExp | false {
-  if (input.indexOf("*") === -1) return false;
-
-  let output = input.replace(/[[\]\\^$.|?+()]/, "\\$&");
-  if (output[0] === "*") output = output.replace(/^\*+/g, "");
-  else output = "^" + output;
-
-  if (output[output.length - 1] === "*") output = output.replace(/\*+$/g, "");
-  else output = output + "$";
-
-  output = output.replace(/[*]/, ".*");
-  return new RegExp(output, flags);
-}
-
-function normalize(input): any {
-  if (typeof input === "string") {
-    const vals: any = [input];
-    const m = /^\/(.*?)\/(g?i?m?y?)$/.exec(input);
-    if (m) vals.push({ $regex: new RegExp(m[1], m[2]) });
-
-    if (+input === parseFloat(input)) vals.push(+input);
-
-    const d = new Date(input);
-    if (input.length >= 8 && d.getFullYear() > 1983) vals.push(d);
-
-    const r = stringToRegexp(input);
-    if (r !== false) vals.push({ $regex: r });
-
-    return vals;
-  }
-  return input;
-}
-
-function expandValue(value: unknown): unknown[] {
-  if (Array.isArray(value)) {
-    let a = [];
-    for (const j of value) a = a.concat(expandValue(j));
-    return [a];
-  } else if (!isObject(value)) {
-    const n = normalize(value);
-    if (!Array.isArray(n)) return [n];
-    else return n;
-  }
-
-  const objs = [];
-  const indices = [];
-  const keys = [];
-  const values = [];
-  for (const [k, v] of Object.entries(value)) {
-    keys.push(k);
-    values.push(expandValue(v));
-    indices.push(0);
-  }
-
-  let i = 0;
-  while (i < indices.length) {
-    const obj = {};
-    for (let j = 0; j < keys.length; ++j) obj[keys[j]] = values[j][indices[j]];
-    objs.push(obj);
-
-    for (i = 0; i < indices.length; ++i) {
-      indices[i] += 1;
-      if (indices[i] < values[i].length) break;
-      indices[i] = 0;
-    }
-  }
-  return objs;
-}
-
-function permute(param, val): any[] {
-  const conditions = [];
-  const values = expandValue(val);
-
-  if (param[param.lastIndexOf(".") + 1] !== "_") param += "._value";
-
-  for (const v of values) {
-    const obj = {};
-    obj[param] = v;
-    conditions.push(obj);
-  }
-
-  return conditions;
-}
-
-export function expand(
-  query: Record<string, unknown>
-): Record<string, unknown> {
-  const newQuery = {};
-  for (const [k, v] of Object.entries(query)) {
-    if (k[0] === "$") {
-      // Operator
-      newQuery[k] = (v as any[]).map((e) => expand(e));
-    } else {
-      const conditions = permute(k, v);
-      if (conditions.length > 1) {
-        newQuery["$and"] = newQuery["$and"] || [];
-        if (v && (v["$ne"] != null || v["$not"] != null)) {
-          if (Object.keys(v).length > 1)
-            throw new Error("Cannot mix $ne or $not with other operators");
-          for (const c of conditions) newQuery["$an"].push(c);
-        } else {
-          newQuery["$and"].push({ $or: conditions });
-        }
-      } else {
-        Object.assign(newQuery, conditions[0]);
-      }
-    }
-  }
-
-  return newQuery;
-}
-
-export function sanitizeQueryTypes(
-  query: Record<string, unknown>,
-  types: Record<string, (v: unknown) => unknown>
-): Record<string, unknown> {
-  for (const [k, v] of Object.entries(query)) {
-    if (k[0] === "$") {
-      // Logical operator
-      for (const vv of v as any[]) sanitizeQueryTypes(vv, types);
-    } else if (k in types) {
-      if (isObject(v)) {
-        for (const [kk, vv] of Object.entries(v)) {
-          switch (kk) {
-            case "$in":
-            case "$nin":
-              for (let i = 0; i < vv.length; ++i) vv[i] = types[k](vv[i]);
-              break;
-            case "$eq":
-            case "$gt":
-            case "$gte":
-            case "$lt":
-            case "$lte":
-            case "$ne":
-              v[kk] = types[k](vv);
-              break;
-            case "$exists":
-            case "$type":
-              // Ignore
-              break;
-            default:
-              throw new Error("Operator not supported");
-          }
-        }
-      } else {
-        query[k] = types[k](query[k]);
-      }
-    }
-  }
-
-  return query;
-}

lib/sandbox.js

@@ -0,0 +1,388 @@
+/**
+ * Copyright 2013-2017  Zaid Abdulla
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+"use strict";
+
+const vm = require("vm");
+
+const seedrandom = require("seedrandom");
+
+const common = require("./common");
+const device = require("./device");
+const extensions = require("./extensions");
+const logger = require("./logger");
+const scheduling = require("./scheduling");
+
+
+// Used for throwing to exit user script and commit
+const COMMIT = Symbol();
+
+// Used to execute extensions and restart
+const EXT = Symbol();
+
+const context = vm.createContext();
+
+let state;
+
+const runningExtensions = new WeakMap();
+function runExtension(sessionContext, key, extCall, callback) {
+  let re = runningExtensions.get(sessionContext);
+  if (!re) {
+    re = {};
+    runningExtensions.set(sessionContext, re);
+  }
+
+  if (re[key]) {
+    re[key].push(callback);
+  }
+  else {
+    re[key] = [callback];
+    extensions.run(extCall, function(err, fault, res) {
+      const callbacks = re[key];
+      delete re[key];
+      if (!err && !fault)
+        sessionContext.extensionsCache[key] = res;
+      for (let c of callbacks)
+        c(err, fault);
+    });
+  }
+}
+
+class SandboxDate {
+  constructor() {
+    if (arguments.length)
+      return new Date(...arguments);
+
+    return new Date(state.sessionContext.timestamp);
+  }
+
+  static now(intervalOrCron, variance) {
+    let t = state.sessionContext.timestamp;
+
+    if (typeof intervalOrCron === "number") {
+
+      if (variance == null)
+        variance = intervalOrCron;
+
+      let offset = 0;
+      if (variance)
+        offset = scheduling.variance(state.sessionContext.deviceId, variance);
+
+      t = scheduling.interval(t, intervalOrCron, offset);
+    }
+    else if (typeof intervalOrCron === "string") {
+      let offset = 0;
+      if (variance)
+        offset = scheduling.variance(state.sessionContext.deviceId, variance);
+      let cron = scheduling.parseCron(intervalOrCron);
+      t = scheduling.cron(t, cron, offset)[0];
+    }
+    else if (intervalOrCron) {
+      throw new Error("Invalid Date.now() argument");
+    }
+
+    return t;
+  }
+};
+
+function random() {
+  if (!state.rng)
+    state.rng = seedrandom(state.sessionContext.deviceId);
+
+  return state.rng();
+}
+
+random.seed = function(s) {
+  state.rng = seedrandom(s);
+}
+
+
+class ParameterWrapper {
+  constructor(path, attributes, unpacked, unpackedRevision) {
+    for (let attrName of attributes) {
+      Object.defineProperty(this, attrName, {get: function() {
+        if (state.uncommitted)
+          commit();
+
+        if (state.revision !== unpackedRevision) {
+          unpackedRevision = state.revision;
+          unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision);
+        }
+
+        if (!unpacked.length)
+          return undefined;
+
+        let attr = state.sessionContext.deviceData.attributes.get(unpacked[0], state.revision)[attrName];
+
+        if (!attr)
+          return undefined;
+
+        return attr[1];
+      }});
+    }
+
+    Object.defineProperty(this, "path", {get: function() {
+      if (state.uncommitted)
+        commit();
+
+      if (state.revision !== unpackedRevision) {
+        unpackedRevision = state.revision;
+        unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision);
+      }
+
+      if (!unpacked.length)
+        return undefined;
+
+      return unpacked[0].join(".");
+    }});
+
+    Object.defineProperty(this, "size", {get: function() {
+      if (state.uncommitted)
+        commit();
+
+      if (state.revision !== unpackedRevision) {
+        unpackedRevision = state.revision;
+        unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision);
+      }
+
+      if (!unpacked.length)
+        return undefined;
+
+      return unpacked.length;
+    }});
+
+    this[Symbol.iterator] = function*() {
+      if (state.uncommitted)
+        commit();
+
+      if (state.revision !== unpackedRevision) {
+        unpackedRevision = state.revision;
+        unpacked = device.unpack(state.sessionContext.deviceData, path, state.revision);
+      }
+
+      for (let p of unpacked)
+        yield new ParameterWrapper(p, attributes, [p], state.revision);
+    }
+  }
+}
+
+
+function declare(path, timestamps, values) {
+  state.uncommitted = true;
+  if (!timestamps)
+    timestamps = {};
+
+  if (!values)
+    values = {};
+
+  let parsedPath = common.parsePath(path);
+
+  let declaration = [parsedPath, 1];
+  let attrs = {};
+
+  for (let attrName in values) {
+    if (!timestamps[attrName])
+      timestamps[attrName] = 1;
+
+    if (attrName === "path")
+      declaration[3] = values.path;
+    else {
+      attrs[attrName] = 1;
+      declaration[4] = declaration[4] || {};
+      if (attrName === "value" && !Array.isArray(values.value))
+        declaration[4].value = [values.value];
+      else
+        declaration[4][attrName] = values[attrName];
+    }
+  }
+
+  for (let attrName in timestamps) {
+    if (attrName === "path")
+      declaration[1] = timestamps.path;
+    else {
+      attrs[attrName] = 1;
+      declaration[2] = declaration[2] || {};
+      declaration[2][attrName] = timestamps[attrName];
+    }
+  }
+
+  state.declarations.push(declaration);
+
+  return new ParameterWrapper(parsedPath, Object.keys(attrs));
+}
+
+
+function clear(path, timestamp, attributes) {
+  state.uncommitted = true;
+
+  if (state.revision === state.maxRevision)
+    state.clear.push([common.parsePath(path), timestamp, attributes]);
+}
+
+
+function commit() {
+  ++ state.revision;
+  state.uncommitted = false;
+
+  if (state.revision === state.maxRevision + 1)
+    throw COMMIT;
+  else if (state.revision > state.maxRevision + 1)
+    throw new Error("Declare function should not be called from within a try/catch block");
+}
+
+
+function ext() {
+  ++ state.extCounter;
+  let extCall = Array.from(arguments).map(String);
+  let key = `${state.revision}: ${JSON.stringify(extCall)}`;
+
+  if (key in state.sessionContext.extensionsCache)
+    return state.sessionContext.extensionsCache[key];
+
+  state.extensions[key] = extCall;
+  throw EXT;
+}
+
+
+function log(msg, meta) {
+  if (state.revision === state.maxRevision && state.extCounter >= 0) {
+    const details = Object.assign({}, meta, {
+      sessionContext: state.sessionContext,
+      message: `Script: ${msg}`
+    });
+
+    delete details.hostname;
+    delete details.pid;
+    delete details.name;
+    delete details.version;
+    delete details.deviceId;
+    delete details.remoteAddress;
+
+    logger.accessInfo(details);
+  }
+}
+
+
+Object.defineProperty(context, "Date", {value: SandboxDate});
+Object.defineProperty(context, "declare", {value: declare});
+Object.defineProperty(context, "clear", {value: clear});
+Object.defineProperty(context, "commit", {value: commit});
+Object.defineProperty(context, "ext", {value: ext});
+Object.defineProperty(context, "log", {value: log});
+
+// Monkey-patch Math.random() to make it deterministic
+context.random = random;
+vm.runInContext("Math.random = random;", context);
+delete context.random;
+
+
+function errorToFault(err) {
+  if (!err)
+    return null;
+
+  if (!err.name)
+    return {code: "script", message: `${err}`};
+
+  let fault = {
+    code: `script.${err.name}`,
+    message: err.message,
+    detail: {
+      name: err.name,
+      message: err.message
+    }
+  };
+
+  if (err.stack) {
+    fault.detail.stack = err.stack;
+    // Trim the stack trace at the self-executing anonymous wrapper function
+    let stackTrimIndex = fault.detail.stack.match(/\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/);
+    if (stackTrimIndex)
+      fault.detail.stack = fault.detail.stack.slice(0, stackTrimIndex.index);
+  }
+
+  return fault;
+}
+
+
+function run(script, globals, sessionContext, startRevision, maxRevision, callback, extCounter = 0) {
+  state = {
+    sessionContext: sessionContext,
+    revision: startRevision,
+    maxRevision: maxRevision,
+    uncommitted: false,
+    declarations: [],
+    extensions: {},
+    clear: [],
+    rng: null,
+    extCounter: extCounter
+  };
+
+  for (let n of Object.keys(context))
+    delete context[n];
+
+  Object.assign(context, globals);
+
+  let ret, status;
+
+  try {
+    ret = script.runInContext(context, {displayErrors: false});
+    status = 0;
+  }
+  catch (err) {
+    if (err === COMMIT)
+      status = 1;
+    else if (err === EXT)
+      status = 2;
+    else
+      return callback(null, errorToFault(err));
+  }
+
+  let _state = state;
+  let args = Array.from(arguments);
+
+  // Need to maintain a counter of ext() calls to avoid calling certain
+  // functions (e.g. log()) multiple times as calling ext() results in
+  // re-execution of script without revision being incremented.
+  args[6] = 0 - (state.extCounter - extCounter);
+
+  let counter = 3;
+  for (let key of Object.keys(_state.extensions)) {
+    counter += 2;
+    runExtension(_state.sessionContext, key, _state.extensions[key], function(err, fault) {
+      if (err || fault) {
+        if (counter & 1)
+          callback(err, fault);
+        return counter = 0;
+      }
+      if ((counter -= 2) === 1)
+        if (status === 2)
+          return run.apply(null, args);
+        else
+          return callback(null, null, _state.clear, _state.declarations, status === 0, ret);
+    });
+  }
+
+  if ((counter -= 2) === 1)
+    if (status === 2)
+      return run.apply(null, args);
+    else
+      return callback(null, null, _state.clear, _state.declarations, status === 0, ret);
+}
+
+
+exports.run = run;

lib/sandbox.ts

@@ -1,448 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as vm from "vm";
-import seedrandom from "seedrandom";
-import * as device from "./device";
-import * as extensions from "./extensions";
-import * as logger from "./logger";
-import * as scheduling from "./scheduling";
-import Path from "./common/path";
-import { Fault, SessionContext, ScriptResult } from "./types";
-
-// Used for throwing to exit user script and commit
-const COMMIT = Symbol();
-
-// Used to execute extensions and restart
-const EXT = Symbol();
-
-const UNDEFINED = undefined;
-
-const context = vm.createContext();
-
-let state;
-
-const runningExtensions = new WeakMap<
-  SessionContext,
-  Map<string, Promise<Fault>>
->();
-function runExtension(sessionContext, key, extCall): Promise<Fault> {
-  let re = runningExtensions.get(sessionContext);
-  if (!re) {
-    re = new Map<string, Promise<Fault>>();
-    runningExtensions.set(sessionContext, re);
-  }
-
-  let prom = re.get(key);
-  if (!prom) {
-    re.set(
-      key,
-      (prom = new Promise((resolve, reject) => {
-        extensions
-          .run(extCall)
-          .then(({ fault, value }) => {
-            re.delete(key);
-            if (!fault) sessionContext.extensionsCache[key] = value;
-            resolve(fault);
-          })
-          .catch(reject);
-      }))
-    );
-  }
-
-  return prom;
-}
-
-class SandboxDate {
-  public constructor(
-    ...argumentList: [
-      number?,
-      number?,
-      number?,
-      number?,
-      number?,
-      number?,
-      number?
-    ]
-  ) {
-    if (argumentList.length) return new Date(...argumentList);
-
-    return new Date(state.sessionContext.timestamp);
-  }
-
-  public static now(intervalOrCron, variance): number {
-    let t = state.sessionContext.timestamp;
-
-    if (typeof intervalOrCron === "number") {
-      if (variance == null) variance = intervalOrCron;
-
-      let offset = 0;
-      if (variance)
-        offset = scheduling.variance(state.sessionContext.deviceId, variance);
-
-      t = scheduling.interval(t, intervalOrCron, offset);
-    } else if (typeof intervalOrCron === "string") {
-      let offset = 0;
-      if (variance)
-        offset = scheduling.variance(state.sessionContext.deviceId, variance);
-      const cron = scheduling.parseCron(intervalOrCron);
-      t = scheduling.cron(t, cron, offset)[0];
-    } else if (intervalOrCron) {
-      throw new Error("Invalid Date.now() argument");
-    }
-
-    return t;
-  }
-
-  public static parse(dateString: string): number {
-    return Date.parse(dateString);
-  }
-
-  public static UTC(
-    ...args: [number, number?, number?, number?, number?, number?, number?]
-  ): number {
-    return Date.UTC(...args);
-  }
-}
-
-function random(): number {
-  if (!state.rng) state.rng = seedrandom(state.sessionContext.deviceId);
-
-  return state.rng();
-}
-
-random.seed = function (s) {
-  state.rng = seedrandom(s);
-};
-
-class ParameterWrapper {
-  public constructor(path: Path, attributes, unpacked?, unpackedRevision?) {
-    for (const attrName of attributes) {
-      Object.defineProperty(this, attrName, {
-        get: function () {
-          if (state.uncommitted) commit();
-
-          if (state.revision !== unpackedRevision) {
-            unpackedRevision = state.revision;
-            unpacked = device.unpack(
-              state.sessionContext.deviceData,
-              path,
-              state.revision
-            );
-          }
-
-          if (!unpacked.length) return UNDEFINED;
-
-          const attr = state.sessionContext.deviceData.attributes.get(
-            unpacked[0],
-            state.revision
-          )[attrName];
-
-          if (!attr) return UNDEFINED;
-
-          return attr[1];
-        },
-      });
-    }
-
-    Object.defineProperty(this, "path", {
-      get: function () {
-        if (state.uncommitted) commit();
-
-        if (state.revision !== unpackedRevision) {
-          unpackedRevision = state.revision;
-          unpacked = device.unpack(
-            state.sessionContext.deviceData,
-            path,
-            state.revision
-          );
-        }
-
-        if (!unpacked.length) return UNDEFINED;
-
-        return unpacked[0].toString();
-      },
-    });
-
-    Object.defineProperty(this, "size", {
-      get: function () {
-        if (state.uncommitted) commit();
-
-        if (state.revision !== unpackedRevision) {
-          unpackedRevision = state.revision;
-          unpacked = device.unpack(
-            state.sessionContext.deviceData,
-            path,
-            state.revision
-          );
-        }
-
-        if (!unpacked.length) return UNDEFINED;
-
-        return unpacked.length;
-      },
-    });
-
-    this[Symbol.iterator] = function* () {
-      if (state.uncommitted) commit();
-
-      if (state.revision !== unpackedRevision) {
-        unpackedRevision = state.revision;
-        unpacked = device.unpack(
-          state.sessionContext.deviceData,
-          path,
-          state.revision
-        );
-      }
-
-      for (const p of unpacked)
-        yield new ParameterWrapper(p, attributes, [p], state.revision);
-    };
-  }
-}
-
-function declare(
-  path: string,
-  timestamps: { [attr: string]: number },
-  values: { [attr: string]: any }
-): ParameterWrapper {
-  state.uncommitted = true;
-  if (!timestamps) timestamps = {};
-
-  if (!values) values = {};
-
-  const parsedPath = Path.parse(path);
-
-  const declaration = {
-    path: parsedPath,
-    pathGet: 1,
-    pathSet: null,
-    attrGet: null,
-    attrSet: null,
-    defer: true,
-  };
-
-  const attrs = new Set();
-
-  for (const [attrName, attrValue] of Object.entries(values)) {
-    if (attrName === "path") {
-      declaration.pathSet = attrValue;
-    } else {
-      attrs.add(attrName);
-      if (!declaration.attrGet) declaration.attrGet = {};
-      if (!declaration.attrSet) declaration.attrSet = {};
-      declaration.attrGet[attrName] = 1;
-      if (attrName === "value" && !Array.isArray(values.value))
-        declaration.attrSet.value = [values.value];
-      else declaration.attrSet[attrName] = values[attrName];
-    }
-  }
-
-  for (const [attrName, attrTimestamp] of Object.entries(timestamps)) {
-    if (attrName === "path") {
-      declaration.pathGet = attrTimestamp;
-    } else {
-      attrs.add(attrName);
-      if (!declaration.attrGet) declaration.attrGet = {};
-      declaration.attrGet[attrName] = attrTimestamp;
-    }
-  }
-
-  state.declarations.push(declaration);
-
-  return new ParameterWrapper(parsedPath, attrs);
-}
-
-function clear(path: string, timestamp: number, attributes?): void {
-  state.uncommitted = true;
-
-  if (state.revision === state.maxRevision)
-    state.clear.push([Path.parse(path), timestamp, attributes]);
-}
-
-function commit(): void {
-  ++state.revision;
-  state.uncommitted = false;
-
-  if (state.revision === state.maxRevision + 1) {
-    for (const d of state.declarations) d.defer = false;
-    throw COMMIT;
-  } else if (state.revision > state.maxRevision + 1) {
-    throw new Error(
-      "Declare function should not be called from within a try/catch block"
-    );
-  }
-}
-
-function ext(...args: unknown[]): any {
-  ++state.extCounter;
-  const extCall = args.map(String);
-  const key = `${state.revision}: ${JSON.stringify(extCall)}`;
-
-  if (key in state.sessionContext.extensionsCache)
-    return state.sessionContext.extensionsCache[key];
-
-  state.extensions[key] = extCall;
-  throw EXT;
-}
-
-function log(msg: string, meta: Record<string, unknown>): void {
-  if (state.revision === state.maxRevision && state.extCounter >= 0) {
-    const details = Object.assign({}, meta, {
-      sessionContext: state.sessionContext,
-      message: `Script: ${msg}`,
-    });
-
-    delete details["hostname"];
-    delete details["pid"];
-    delete details["name"];
-    delete details["version"];
-    delete details["deviceId"];
-    delete details["remoteAddress"];
-
-    logger.accessInfo(details);
-  }
-}
-
-Object.defineProperty(context, "Date", { value: SandboxDate });
-Object.defineProperty(context, "declare", { value: declare });
-Object.defineProperty(context, "clear", { value: clear });
-Object.defineProperty(context, "commit", { value: commit });
-Object.defineProperty(context, "ext", { value: ext });
-Object.defineProperty(context, "log", { value: log });
-
-// Monkey-patch Math.random() to make it deterministic
-context.random = random;
-vm.runInContext("Math.random = random;", context);
-delete context.random;
-
-function errorToFault(err: Error): Fault {
-  if (!err) return null;
-
-  if (!err.name) return { code: "script", message: `${err}` };
-
-  const fault: Fault = {
-    code: `script.${err.name}`,
-    message: err.message,
-    detail: {
-      name: err.name,
-      message: err.message,
-    },
-  };
-
-  if (err.stack) {
-    fault.detail["stack"] = err.stack;
-    // Trim the stack trace at the self-executing anonymous wrapper function
-    const stackTrimIndex = fault.detail["stack"].match(
-      /\s+at\s[^\s]+\s+at\s[^\s]+\s\(vm\.js.+\)/
-    );
-    if (stackTrimIndex) {
-      fault.detail["stack"] = fault.detail["stack"].slice(
-        0,
-        stackTrimIndex.index
-      );
-    }
-  }
-
-  return fault;
-}
-
-export async function run(
-  script: vm.Script,
-  globals: Record<string, unknown>,
-  sessionContext: SessionContext,
-  startRevision: number,
-  maxRevision: number,
-  extCounter = 0
-): Promise<ScriptResult> {
-  state = {
-    sessionContext: sessionContext,
-    revision: startRevision,
-    maxRevision: maxRevision,
-    uncommitted: false,
-    declarations: [],
-    extensions: {},
-    clear: [],
-    rng: null,
-    extCounter: extCounter,
-  };
-
-  for (const n of Object.keys(context)) delete context[n];
-
-  Object.assign(context, globals);
-
-  let ret, status;
-
-  try {
-    ret = script.runInContext(context, { displayErrors: false });
-    status = 0;
-  } catch (err) {
-    if (err === COMMIT) {
-      status = 1;
-    } else if (err === EXT) {
-      status = 2;
-    } else {
-      return {
-        fault: errorToFault(err),
-        clear: null,
-        declare: null,
-        done: false,
-        returnValue: null,
-      };
-    }
-  }
-
-  const _state = state;
-  let fault;
-
-  await Promise.all(
-    Object.entries(_state.extensions).map(async ([k, v]) => {
-      fault = (await runExtension(_state.sessionContext, k, v)) || fault;
-    })
-  );
-
-  if (fault) {
-    return {
-      fault: fault,
-      clear: null,
-      declare: null,
-      done: false,
-      returnValue: null,
-    };
-  }
-
-  if (status === 2) {
-    return run(
-      script,
-      globals,
-      sessionContext,
-      startRevision,
-      maxRevision,
-      extCounter - _state.extCounter
-    );
-  }
-
-  return {
-    fault: null,
-    clear: _state.clear,
-    declare: _state.declarations,
-    done: status === 0,
-    returnValue: ret,
-  };
-}

lib/scheduling.js

@@ -1,5 +1,5 @@
 /**
- * Copyright 2013-2019  GenieACS Inc.
+ * Copyright 2013-2017  Zaid Abdulla
  *
  * This file is part of GenieACS.
  *
@@ -16,12 +16,18 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
  */
+"use strict";
 
-import * as crypto from "crypto";
-import * as later from "later";
+const crypto = require("crypto");
 
-function md532(str): number {
-  const digest = crypto.createHash("md5").update(str).digest();
+const later = require("later");
+
+
+function md532(str) {
+  const digest = crypto
+    .createHash("md5")
+    .update(str)
+    .digest();
   return (
     digest.readUInt32LE(0) ^
     digest.readUInt32LE(4) ^
@@ -30,38 +36,43 @@ function md532(str): number {
   );
 }
 
-export function variance(deviceId: string, vrnc: number): number {
-  return (md532(deviceId) >>> 0) % vrnc;
+
+function variance(deviceId, variance) {
+  return (md532(deviceId) >>> 0) % variance;
 }
 
-export function interval(
-  timestamp: number,
-  intrvl: number,
-  offset = 0
-): number {
-  return Math.trunc((timestamp + offset) / intrvl) * intrvl - offset;
+
+function interval(timestamp, interval, offset = 0) {
+  return Math.trunc((timestamp + offset) / interval) * interval - offset;
 }
 
-export function parseCron(cronExp: string): any {
-  const parts = cronExp.trim().split(/\s+/);
-  if (parts.length === 5) parts.unshift("*");
+
+function parseCron(cronExp) {
+  let parts = cronExp.trim().split(/\s+/);
+  if (parts.length === 5)
+    parts.unshift("*");
 
   return later.schedule(later.parse.cron(parts.join(" "), true));
 }
 
-export function cron(
-  timestamp: number,
-  schedule: unknown,
-  offset = 0
-): number[] {
+
+function cron(timestamp, schedule, offset = 0) {
   // TODO later.js doesn't throw erorr if expression is invalid!
   const ret = [0, 0];
 
-  const prev = (schedule as any).prev(1, new Date(timestamp + offset));
-  if (prev) ret[0] = prev.setMilliseconds(0) - offset;
+  let prev = schedule.prev(1, new Date(timestamp + offset));
+  if (prev)
+    ret[0] = prev.setMilliseconds(0) - offset;
 
-  const next = (schedule as any).next(1, new Date(timestamp + offset + 1000));
-  if (next) ret[1] = next.setMilliseconds(0) - offset;
+  let next = schedule.next(1, new Date(timestamp + offset + 1000));
+  if (next)
+    ret[1] = next.setMilliseconds(0) - offset;
 
   return ret;
 }
+
+
+exports.variance = variance;
+exports.interval = interval;
+exports.parseCron = parseCron;
+exports.cron = cron;

lib/server.coffee

@@ -0,0 +1,123 @@
+###
+# Copyright 2013-2017  Zaid Abdulla
+#
+# This file is part of GenieACS.
+#
+# GenieACS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# GenieACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+###
+
+service = process.argv[2]
+if not service?
+  throw new Error('Missing argument cwmp, fs, or nbi')
+
+util = require 'util'
+cluster = require 'cluster'
+logger = require './logger'
+logger.init(service, require('../package.json').version)
+config = require './config'
+db = require './db'
+cache = require './cache'
+extensions = require './extensions'
+
+networkInterface = config.get("#{service.toUpperCase()}_INTERFACE")
+port = config.get("#{service.toUpperCase()}_PORT")
+useHttps = config.get("#{service.toUpperCase()}_SSL")
+
+listener = require("./#{service}").listener
+onConnection = require("./#{service}").onConnection
+
+server = null
+
+closeServer = (timeout, callback) ->
+  return callback() if not server
+
+  setTimeout(() ->
+    return if not callback
+
+    # Ignore HTTP requests from connection that may still be open
+    server.removeListener('request', listener)
+    server.setTimeout(1)
+
+    cb = callback
+    callback = null
+    setTimeout(cb, 1000)
+  , timeout).unref()
+
+  server.close(() ->
+    return if not callback
+
+    cb = callback
+    callback = null
+    cb()
+  )
+
+
+exit = () ->
+  setTimeout(() ->
+    extensions.killAll(() ->
+      process.exit(1)
+    )
+  , 30000).unref()
+
+  closeServer(20000, () ->
+    db.disconnect()
+    cache.disconnect()
+    extensions.killAll()
+    cluster.worker?.disconnect()
+  )
+
+
+process.on('uncaughtException', (err) ->
+  logger.error({
+    message: 'Uncaught exception'
+    exception: err,
+    pid: process.pid
+  })
+  exit()
+)
+
+
+if useHttps
+  path = require 'path'
+  fs = require 'fs'
+  httpsKey = path.resolve(config.get('CONFIG_DIR'), "#{service}.key")
+  httpsCert = path.resolve(config.get('CONFIG_DIR'), "#{service}.crt")
+  httpsCa = path.resolve(config.get('CONFIG_DIR'), "#{service}.ca-bundle")
+  options = {
+    key: fs.readFileSync(httpsKey),
+    cert: fs.readFileSync(httpsCert)
+  }
+
+  try
+    # Use intermediate certificates if available
+    options.ca = fs.readFileSync(httpsCa).toString()
+      .match(/\-+BEGIN CERTIFICATE\-+[0-9a-zA-Z\+\-\/\=\s]+?\-+END CERTIFICATE\-+/g)
+
+  server = require('https').createServer(options, listener)
+  server.on('secureConnection', onConnection) if onConnection?
+else
+  server = require('http').createServer(listener)
+  server.on('connection', onConnection) if onConnection?
+
+db.connect((err) ->
+  throw err if err
+  cache.connect((err) ->
+    throw err if err
+    server.listen(port, networkInterface)
+  )
+)
+
+
+process.on('SIGINT', exit)
+process.on('SIGTERM', exit)

lib/server.ts

@@ -1,101 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import * as fs from "fs";
-import * as http from "http";
-import * as https from "https";
-import { Socket } from "net";
-import * as path from "path";
-import { ROOT_DIR } from "./config";
-
-let server: http.Server | https.Server;
-let listener: (...args) => void;
-
-function closeServer(timeout, callback): void {
-  if (!server) return void callback();
-
-  setTimeout(() => {
-    if (!callback) return;
-
-    // Ignore HTTP requests from connection that may still be open
-    server.removeListener("request", listener);
-    server.setTimeout(1);
-
-    const cb = callback;
-    callback = null;
-    setTimeout(cb, 1000);
-  }, timeout).unref();
-
-  server.close(() => {
-    if (!callback) return;
-
-    const cb = callback;
-    callback = null;
-    // Allow some time for connection close events to fire
-    setTimeout(cb, 50);
-  });
-}
-
-interface ServerOptions {
-  port?: number;
-  host?: string;
-  ssl?: { key: string; cert: string };
-  timeout?: number;
-  keepAliveTimeout?: number;
-  onConnection?: (socket: Socket) => void;
-}
-
-export function start(
-  options: ServerOptions,
-  _listener: http.RequestListener
-): void {
-  listener = _listener;
-
-  if (options.ssl) {
-    const opts = {
-      key: options.ssl.key
-        .split(":")
-        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
-      cert: options.ssl.cert
-        .split(":")
-        .map((f) => fs.readFileSync(path.resolve(ROOT_DIR, f.trim()))),
-    };
-
-    server = https.createServer(opts, listener);
-    if (options.onConnection)
-      server.on("secureConnection", options.onConnection);
-  } else {
-    server = http.createServer(listener);
-    if (options.onConnection) server.on("connection", options.onConnection);
-  }
-
-  server.timeout = options.timeout || 0;
-  if (options.keepAliveTimeout != null)
-    server.keepAliveTimeout = options.keepAliveTimeout;
-  server.listen({ port: options.port, host: options.host });
-}
-
-export function stop(): Promise<void> {
-  return new Promise((resolve, reject) => {
-    setTimeout(() => {
-      reject(new Error("Could not close server in a timely manner"));
-    }, 30000).unref();
-    closeServer(20000, resolve);
-  });
-}

lib/soap.js

@@ -0,0 +1,612 @@
+/**
+ * Copyright 2013-2017  Zaid Abdulla
+ *
+ * This file is part of GenieACS.
+ *
+ * GenieACS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * GenieACS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+"use strict";
+
+const libxmljs = require("libxmljs");
+
+const config = require("./config");
+
+const SERVER_NAME = `GenieACS/${require("../package.json").version}`;
+
+const NAMESPACES = {
+  "1.0": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    "xsd": "http://www.w3.org/2001/XMLSchema",
+    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
+    "cwmp": "urn:dslforum-org:cwmp-1-0"
+  },
+  "1.1": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    "xsd": "http://www.w3.org/2001/XMLSchema",
+    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
+    "cwmp": "urn:dslforum-org:cwmp-1-1"
+  },
+  "1.2": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    "xsd": "http://www.w3.org/2001/XMLSchema",
+    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
+    "cwmp": "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.3": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    "xsd": "http://www.w3.org/2001/XMLSchema",
+    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
+    "cwmp": "urn:dslforum-org:cwmp-1-2"
+  },
+  "1.4": {
+    "soap-enc": "http://schemas.xmlsoap.org/soap/encoding/",
+    "soap-env": "http://schemas.xmlsoap.org/soap/envelope/",
+    "xsd": "http://www.w3.org/2001/XMLSchema",
+    "xsi": "http://www.w3.org/2001/XMLSchema-instance",
+    "cwmp": "urn:dslforum-org:cwmp-1-3"
+  }
+};
+
+const LIBXMLJS_PARSE_OPTIONS = {
+  "nocdata": true,
+  "recover": config.get("XML_RECOVER"),
+  "ignore_enc": config.get("XML_IGNORE_ENC")
+};
+
+const LIBXMLJS_SAVE_OPTIONS = {
+  "format": config.get("XML_FORMAT"),
+  "declaration": !config.get("XML_NO_DECL"),
+  "selfCloseEmpty": !config.get("XML_NO_EMPTY")
+};
+
+const XML_IGNORE_NAMESPACE = config.get("XML_IGNORE_NAMESPACE");
+
+if (XML_IGNORE_NAMESPACE) {
+  libxmljs.Element.prototype.__find = libxmljs.Element.prototype.find;
+  libxmljs.Element.prototype.find = function(xpath, namespaces) {
+    const p = xpath.replace(/([^\/:]+:)?([^\/]+)(\/|$)/g, "$2$3").replace(/([a-zA-Z0-9_-]+)([^\/]*)(\/|$)/g, "*[local-name()=\"$1\"]$2$3");
+    return libxmljs.Element.prototype.__find.call(this, p, namespaces);
+  };
+}
+
+let warnings;
+
+
+function parseBool(v) {
+  v = "" + v;
+  if (v === "true" || v === "TRUE" || v === "True" || v === "1")
+    return true;
+  else if (v === "false" || v === "FALSE" || v === "False" || v === "0")
+    return false;
+  else
+    return null;
+}
+
+
+function event(xml) {
+  return xml.find("EventStruct/EventCode").map(e => e.text().trim());
+}
+
+
+function parameterInfoList(xml) {
+  return xml.find("ParameterInfoStruct").map(function(e) {
+    let param = e.get("Name").text();
+    let value = e.get("Writable").text();
+    let parsed = parseBool(value);
+
+    if (parsed === null) {
+      warnings.push({
+        message: "Invalid writable attribute",
+        parameter: param
+      });
+      parsed = false;
+    }
+
+    return [param, parsed];
+  });
+}
+
+
+function parameterValueList(xml) {
+  return xml.find("ParameterValueStruct").map(function(e) {
+    const valueElement = e.get("Value");
+    const valueType = valueElement.attr("type").value();
+    let param = e.get("Name").text();
+    let value = valueElement.text();
+    let parsed = value;
+    if (valueType === "xsd:boolean") {
+      parsed = parseBool(value);
+      if (parsed === null) {
+        warnings.push({
+          message: "Invalid value attribute",
+          parameter: param
+        });
+        parsed = value;
+      }
+    }
+    else if (valueType === "xsd:int" || valueType === "xsd:unsignedInt") {
+      parsed = parseInt(value);
+      if (isNaN(parsed)) {
+        warnings.push({
+          message: "Invalid value attribute",
+          parameter: param
+        });
+        parsed = value;
+      }
+    }
+    else if (valueType === "xsd:dateTime") {
+      parsed = Date.parse(value);
+      if (isNaN(parsed)) {
+        warnings.push({
+          message: "Invalid value attribute",
+          parameter: param
+        });
+        parsed = value;
+      }
+    }
+
+   return [param, parsed, valueType];
+  });
+}
+
+
+function GetParameterNames(xml, methodRequest) {
+  const el = xml.node("cwmp:GetParameterNames");
+  el.node("ParameterPath").text(methodRequest.parameterPath);
+  el.node("NextLevel").text(+methodRequest.nextLevel);
+}
+
+
+function GetParameterNamesResponse(xml) {
+  return {
+    name: "GetParameterNamesResponse",
+    parameterList: parameterInfoList(xml.get("ParameterList"))
+  };
+}
+
+
+function GetParameterValues(xml, methodRequest) {
+  const el = xml.node("cwmp:GetParameterValues").node("ParameterNames");
+  el.attr({
+    "soap-enc:arrayType": `xsd:string[${methodRequest.parameterNames.length}]`
+  });
+
+  for (const p of methodRequest.parameterNames)
+    el.node("string").text(p);
+}
+
+
+function GetParameterValuesResponse(xml) {
+  return {
+    name: "GetParameterValuesResponse",
+    parameterList: parameterValueList(xml.get("ParameterList"))
+  };
+}
+
+
+function SetParameterValues(xml, methodRequest) {
+  const el = xml.node("cwmp:SetParameterValues");
+  const paramList = el.node("ParameterList");
+  paramList.attr({
+    "soap-enc:arrayType": `cwmp:ParameterValueStruct[${methodRequest.parameterList.length}]`
+  });
+
+  for (const p of methodRequest.parameterList) {
+    const pvs = paramList.node("ParameterValueStruct");
+    pvs.node("Name").text(p[0]);
+    let val = p[1];
+    if (p[2] === 'xsd:dateTime' && typeof val === 'number') {
+      val = (new Date(val)).toISOString();
+      if (methodRequest.DATETIME_MILLISECONDS === false)
+        val = val.replace('.000', '');
+    }
+    if (p[2] === 'xsd:boolean' && typeof val === 'boolean') {
+      if (methodRequest.BOOLEAN_LITERAL === false)
+        val = +val;
+    }
+    pvs.node("Value").attr({"xsi:type": p[2]}).text(val);
+  }
+
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+
+function SetParameterValuesResponse(xml) {
+  return {
+    name: "SetParameterValuesResponse",
+    status: parseInt(xml.get("Status").text())
+  };
+}
+
+
+function AddObject(xml, methodRequest) {
+  const el = xml.node("cwmp:AddObject");
+  el.node("ObjectName").text(methodRequest.objectName);
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+
+function AddObjectResponse(xml) {
+  return {
+    name: "AddObjectResponse",
+    instanceNumber: parseInt(xml.get("InstanceNumber").text()),
+    status: parseInt(xml.get("Status").text())
+  };
+}
+
+
+function DeleteObject(xml, methodRequest) {
+  const el = xml.node("cwmp:DeleteObject");
+  el.node("ObjectName").text(methodRequest.objectName);
+  el.node("ParameterKey").text(methodRequest.parameterKey || "");
+}
+
+
+function DeleteObjectResponse(xml) {
+  return {
+    name: "DeleteObjectResponse",
+    status: parseInt(xml.get("Status").text())
+  };
+}
+
+
+function Reboot(xml, methodRequest) {
+  const el = xml.node("cwmp:Reboot");
+  el.node("CommandKey").text(methodRequest.commandKey || "");
+}
+
+
+function RebootResponse(xml) {
+  return {
+    name: "RebootResponse"
+  };
+}
+
+
+function FactoryReset(xml, methodRequest) {
+  xml.node("cwmp:FactoryReset");
+}
+
+
+function FactoryResetResponse(xml, methodRequest) {
+  return {
+    name: "FactoryResetResponse"
+  };
+}
+
+
+function Download(xml, methodRequest) {
+  const el = xml.node("cwmp:Download");
+  el.node("CommandKey").text(methodRequest.commandKey || "");
+  el.node("FileType").text(methodRequest.fileType);
+  el.node("URL").text(methodRequest.url);
+  el.node("Username").text(methodRequest.username || "");
+  el.node("Password").text(methodRequest.password || "");
+  el.node("FileSize").text(methodRequest.fileSize || "0");
+  el.node("TargetFileName").text(methodRequest.targetFileName || "");
+  el.node("DelaySeconds").text(methodRequest.delaySeconds || "0");
+  el.node("SuccessURL").text(methodRequest.successUrl || "");
+  el.node("FailureURL").text(methodRequest.failureUrl || "");
+}
+
+
+function DownloadResponse(xml) {
+  const res = {
+    name: "DownloadResponse",
+    status: parseInt(xml.get("Status").text())
+  };
+
+  if (res.status === 0) {
+    res.startTime = Date.parse(xml.get("StartTime").text());
+    res.completeTime = Date.parse(xml.get("CompleteTime").text());
+  }
+
+  return res;
+}
+
+
+function Inform(xml) {
+  return {
+    name: "Inform",
+    parameterList: parameterValueList(xml.get("ParameterList")),
+    deviceId: {
+      "Manufacturer": xml.get("DeviceId/Manufacturer").text(),
+      "OUI": xml.get("DeviceId/OUI").text(),
+      "ProductClass": xml.get("DeviceId/ProductClass").text(),
+      "SerialNumber": xml.get("DeviceId/SerialNumber").text()
+    },
+    event: event(xml.get("Event")),
+    retryCount: parseInt(xml.get("RetryCount").text())
+  };
+}
+
+
+function InformResponse(xml, methodResponse) {
+  xml.node("cwmp:InformResponse").node("MaxEnvelopes").text(1);
+}
+
+
+function GetRPCMethods(xml) {
+  return {name: "GetRPCMethods"};
+}
+
+
+function GetRPCMethodsResponse(xml, methodResponse) {
+  const el = xml.node("cwmp:GetRPCMethodsResponse").node("MethodList");
+  el.attr({
+    "soap-enc:arrayType": `xsd:string[${methodResponse.methodList.length}]`
+  });
+
+  for (const m of methodResponse.methodList)
+    el.node("string").text(m);
+}
+
+
+function TransferComplete(xml) {
+  return {
+    name: "TransferComplete",
+    commandKey: xml.get("CommandKey").text(),
+    faultStruct: faultStruct(xml.get("FaultStruct")),
+    startTime: Date.parse(xml.get("StartTime").text()),
+    completeTime: Date.parse(xml.get("CompleteTime").text())
+  };
+}
+
+
+function TransferCompleteResponse(xml, methodResponse) {
+  xml.node("cwmp:TransferCompleteResponse").text("");
+}
+
+
+function RequestDownload(xml) {
+  return {
+    name: "RequestDownload",
+    fileType: xml.get("FileType").text()
+  };
+}
+
+
+function RequestDownloadResponse(xml, methodResponse) {
+  xml.node("cwmp:RequestDownloadResponse").text("");
+}
+
+
+function faultStruct(xml) {
+  const f = {
+    faultCode: xml.get("FaultCode").text(),
+    faultString: xml.get("FaultString").text()
+  };
+
+  for (const e of xml.find("SetParameterValuesFault")) {
+    if (!f.setParameterValuesFault)
+      f.setParameterValuesFault = [];
+
+    f.setParameterValuesFault.push({
+      parameterName: e.get("ParameterName").text(),
+      faultCode: e.get("FaultCode").text(),
+      faultString: e.get("FaultString").text()
+    });
+  }
+
+  return f;
+}
+
+
+function fault(xml, cwmpVersion) {
+  return {
+    faultCode: xml.get("faultcode").text(),
+    faultString: xml.get("faultstring").text(),
+    detail: faultStruct(xml.get("detail/cwmp:Fault", NAMESPACES[cwmpVersion]))
+  };
+}
+
+
+function request(data, cwmpVersion, warn) {
+  warnings = warn;
+
+  const rpc = {
+    cwmpVersion: cwmpVersion
+  };
+
+  if (!data.length)
+    return rpc;
+
+  const xml = libxmljs.parseXml(data, LIBXMLJS_PARSE_OPTIONS);
+
+  let methodElement;
+
+  if (rpc.cwmpVersion) {
+    methodElement = xml.get("/soap-env:Envelope/soap-env:Body/cwmp:*", NAMESPACES[rpc.cwmpVersion]);
+  }
+  else {
+    // cwmpVersion not passed, thus it's an inform request
+    methodElement = xml.get("/soap-env:Envelope/soap-env:Body/*", NAMESPACES["1.0"]);
+    switch (methodElement.namespace().href()) {
+      case "urn:dslforum-org:cwmp-1-0":
+        rpc.cwmpVersion = "1.0";
+        break;
+      case "urn:dslforum-org:cwmp-1-1":
+        rpc.cwmpVersion = "1.1";
+        break;
+      case "urn:dslforum-org:cwmp-1-2":
+        const timeoutElement = xml.get("/soap-env:Envelope/soap-env:Header/cwmp:sessionTimeout", NAMESPACES["1.2"]);
+        if (timeoutElement) {
+          rpc.sessionTimeout = parseInt(timeoutElement.text());
+          rpc.cwmpVersion = "1.3";
+        }
+        else {
+          rpc.cwmpVersion = "1.2";
+        }
+        break;
+      case "urn:dslforum-org:cwmp-1-3":
+        rpc.cwmpVersion = "1.4";
+        break;
+      default:
+        throw new Error("Unrecognized CWMP version");
+    }
+
+    if (!rpc.sessionTimeout) {
+      const timeoutElement = xml.get("/soap-env:Envelope/soap-env:Header/cwmp:sessionTimeout", NAMESPACES[rpc.cwmpVersion]);
+      if (timeoutElement)
+        rpc.sessionTimeout = parseInt(timeoutElement.text());
+    }
+  }
+
+  let idElement = xml.get("/soap-env:Envelope/soap-env:Header/cwmp:ID", NAMESPACES[rpc.cwmpVersion]);
+  if (idElement)
+    rpc.id = idElement.text();
+
+  if (methodElement && !(XML_IGNORE_NAMESPACE && methodElement.name() === "Fault")) {
+    switch (methodElement.name()) {
+      case "Inform":
+        rpc.cpeRequest = Inform(methodElement);
+        break;
+      case "GetRPCMethods":
+        rpc.cpeRequest = GetRPCMethods(methodElement);
+        break;
+      case "TransferComplete":
+        rpc.cpeRequest = TransferComplete(methodElement);
+        break;
+      case "RequestDownload":
+        rpc.cpeRequest = RequestDownload(methodElement);
+        break;
+      case "GetParameterNamesResponse":
+        rpc.cpeResponse = GetParameterNamesResponse(methodElement);
+        break;
+      case "GetParameterValuesResponse":
+        rpc.cpeResponse = GetParameterValuesResponse(methodElement);
+        break;
+      case "SetParameterValuesResponse":
+        rpc.cpeResponse = SetParameterValuesResponse(methodElement);
+        break;
+      case "AddObjectResponse":
+        rpc.cpeResponse = AddObjectResponse(methodElement);
+        break;
+      case "DeleteObjectResponse":
+        rpc.cpeResponse = DeleteObjectResponse(methodElement);
+        break;
+      case "RebootResponse":
+        rpc.cpeResponse = RebootResponse(methodElement);
+        break;
+      case "FactoryResetResponse":
+        rpc.cpeResponse = FactoryResetResponse(methodElement);
+        break;
+      case "DownloadResponse":
+        rpc.cpeResponse = DownloadResponse(methodElement);
+        break;
+      default:
+        throw new Error(`8000 Method not supported ${methodElement.name()}`);
+    }
+  }
+  else {
+    const faultElement = xml.get("/soap-env:Envelope/soap-env:Body/soap-env:Fault", NAMESPACES[rpc.cwmpVersion]);
+    rpc.cpeFault = fault(faultElement, rpc.cwmpVersion);
+  }
+
+  return rpc;
+}
+
+
+function createSoapEnv(cwmpVersion) {
+  const xml = libxmljs.Document();
+  const env = xml.node("soap-env:Envelope");
+
+  for (const prefix of Object.keys(NAMESPACES[cwmpVersion]))
+    env.defineNamespace(prefix, NAMESPACES[cwmpVersion][prefix]);
+
+  return env;
+}
+
+
+function response(rpc) {
+  const headers = {
+    "Server": SERVER_NAME,
+    "SOAPServer": SERVER_NAME
+  };
+
+  if (!rpc)
+    return {code: 204, headers: headers, data: new Buffer(0)};
+
+  const env = createSoapEnv(rpc.cwmpVersion);
+  const header = env.node("soap-env:Header");
+  header.node("cwmp:ID").attr({
+    "soap-env:mustUnderstand": 1
+  }).text(rpc.id);
+
+  const body = env.node("soap-env:Body");
+
+  if (rpc.acsResponse) {
+    switch (rpc.acsResponse.name) {
+      case "InformResponse":
+        InformResponse(body, rpc.acsResponse);
+        break;
+      case "GetRPCMethodsResponse":
+        GetRPCMethodsResponse(body, rpc.acsResponse);
+        break;
+      case "TransferCompleteResponse":
+        TransferCompleteResponse(body, rpc.acsResponse);
+        break;
+      case "RequestDownloadResponse":
+        RequestDownloadResponse(body, rpc.acsResponse);
+        break;
+      default:
+        throw new Error(`Unknown method response type ${rpc.acsResponse.name}`);
+    }
+  }
+  else if (rpc.acsRequest) {
+    switch (rpc.acsRequest.name) {
+      case "GetParameterNames":
+        GetParameterNames(body, rpc.acsRequest);
+        break;
+      case "GetParameterValues":
+        GetParameterValues(body, rpc.acsRequest);
+        break;
+      case "SetParameterValues":
+        SetParameterValues(body, rpc.acsRequest);
+        break;
+      case "AddObject":
+        AddObject(body, rpc.acsRequest);
+        break;
+      case "DeleteObject":
+        DeleteObject(body, rpc.acsRequest);
+        break;
+      case "Reboot":
+        Reboot(body, rpc.acsRequest);
+        break;
+      case "FactoryReset":
+        FactoryReset(body, rpc.acsRequest);
+        break;
+      case "Download":
+        Download(body, rpc.acsRequest);
+        break;
+      default:
+        throw new Error(`Unknown method request ${rpc.acsRequest.name}`);
+    }
+  }
+
+  headers["Content-Type"] = "text/xml; charset=\"utf-8\"";
+  return {
+    code: 200,
+    headers: headers,
+    data: new Buffer(env.doc().toString(LIBXMLJS_SAVE_OPTIONS))
+  };
+}
+
+
+exports.request = request;
+exports.response = response;

lib/types.ts

@@ -1,597 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import Path from "./common/path";
-import PathSet from "./common/path-set";
-import VersionedMap from "./versioned-map";
-import InstanceSet from "./instance-set";
-import { IncomingMessage, ServerResponse } from "http";
-import { Script } from "vm";
-
-export type Expression = string | number | boolean | null | any[];
-
-export interface Fault {
-  code: string;
-  message: string;
-  detail?:
-    | FaultStruct
-    | {
-        name: string;
-        message: string;
-        stack?: string;
-      };
-  timestamp?: number;
-}
-
-export interface SessionFault extends Fault {
-  timestamp: number;
-  provisions: string[][];
-  retryNow?: boolean;
-  precondition?: boolean;
-  retries?: number;
-  expiry?: number;
-}
-
-export interface Attributes {
-  object?: [number, 1 | 0];
-  writable?: [number, 1 | 0];
-  value?: [number, [string | number | boolean, string]];
-  notification?: [number, number];
-  accessList?: [number, string[]];
-}
-
-export interface AttributeTimestamps {
-  object?: number;
-  writable?: number;
-  value?: number;
-  notification?: number;
-  accessList?: number;
-}
-
-export interface AttributeValues {
-  object?: boolean;
-  writable?: boolean;
-  value?: [string | number | boolean, string?];
-  notification?: number;
-  accessList?: string[];
-}
-
-export interface DeviceData {
-  paths: PathSet;
-  timestamps: VersionedMap<Path, number>;
-  attributes: VersionedMap<Path, Attributes>;
-  trackers: Map<Path, { [name: string]: number }>;
-  changes: Set<string>;
-}
-
-export type VirtualParameterDeclaration = [
-  Path,
-  {
-    path?: number;
-    object?: number;
-    writable?: number;
-    value?: number;
-    notification?: number;
-    accessList?: number;
-  }?,
-  {
-    path?: [number, number];
-    object?: boolean;
-    writable?: boolean;
-    value?: [string | number | boolean, string?];
-    notification?: number;
-    accessList?: string[];
-  }?
-];
-
-export interface SyncState {
-  refreshAttributes: {
-    exist: Set<Path>;
-    object: Set<Path>;
-    writable: Set<Path>;
-    value: Set<Path>;
-    notification: Set<Path>;
-    accessList: Set<Path>;
-  };
-  spv: Map<Path, [string | number | boolean, string]>;
-  spa: Map<Path, { notification: number; accessList: string[] }>;
-  gpn: Set<Path>;
-  gpnPatterns: Map<Path, number>;
-  tags: Map<Path, boolean>;
-  virtualParameterDeclarations: VirtualParameterDeclaration[][];
-  instancesToDelete: Map<Path, Set<Path>>;
-  instancesToCreate: Map<Path, InstanceSet>;
-  downloadsToDelete: Set<Path>;
-  downloadsToCreate: InstanceSet;
-  downloadsValues: Map<Path, string | number>;
-  downloadsDownload: Map<Path, number>;
-  duStatesToDelete: Set<Path>;
-  duStatesToCreate: InstanceSet;
-  duStatesValues: Map<Path, string | number>;
-  duStatesChangeDUState: Map<Path, number>;
-  reboot: number;
-  factoryReset: number;
-}
-
-export interface SessionContext {
-  sessionId?: string;
-  timestamp: number;
-  deviceId: string;
-  deviceData: DeviceData;
-  cwmpVersion: string;
-  timeout: number;
-  provisions: any[];
-  channels: { [channel: string]: number };
-  virtualParameters: [
-    string,
-    AttributeTimestamps,
-    AttributeValues,
-    AttributeTimestamps,
-    AttributeValues
-  ][][];
-  revisions: number[];
-  rpcCount: number;
-  iteration: number;
-  cycle: number;
-  extensionsCache: any;
-  declarations: Declaration[][];
-  faults?: { [channel: string]: SessionFault };
-  retries?: { [channel: string]: number };
-  cacheSnapshot?: string;
-  httpResponse?: ServerResponse;
-  httpRequest?: IncomingMessage;
-  faultsTouched?: { [channel: string]: boolean };
-  presetCycles?: number;
-  new?: boolean;
-  debug?: boolean;
-  state: number;
-  authState: number;
-  tasks?: Task[];
-  operations?: { [commandKey: string]: Operation };
-  cacheUntil?: number;
-  syncState?: SyncState;
-  lastActivity?: number;
-  extendLock?: number;
-  rpcRequest?: AcsRequest;
-  operationsTouched?: { [commandKey: string]: 1 | 0 };
-  provisionsRet?: any[];
-  doneTasks?: string[];
-}
-
-export interface Task {
-  _id?: string;
-  name: string;
-  parameterNames?: string[];
-  parameterValues?: [string, string | number | boolean, string?][];
-  objectName?: string;
-  fileType?: string;
-  fileName?: string;
-  targetFileName?: string;
-  expiry?: number;
-  provisions?: (string | number | boolean)[][];
-}
-
-interface OperationBase {
-  name: string;
-  timestamp: number;
-  provisions: string[][];
-  channels: { [channel: string]: number };
-  retries: { [channel: string]: number };
-  args: any;
-}
-
-export interface DownloadOperation extends OperationBase {
-  name: "Download";
-  args: {
-    instance: string;
-    fileType: string;
-    fileName: string;
-    targetFileName: string;
-  };
-}
-
-export interface ChangeDUStateOperation extends OperationBase {
-  name: "ChangeDUState";
-  args: {
-    operations: ChangeDUState["operations"];
-  };
-}
-
-export type Operation = DownloadOperation | ChangeDUStateOperation;
-
-export type AcsRequest =
-  | GetParameterNames
-  | GetParameterValues
-  | GetParameterAttributes
-  | SetParameterValues
-  | SetParameterAttributes
-  | AddObject
-  | DeleteObject
-  | FactoryReset
-  | Reboot
-  | Download
-  | ChangeDUState;
-
-export interface GetParameterNames {
-  name: "GetParameterNames";
-  parameterPath: string;
-  nextLevel: boolean;
-}
-
-export interface GetParameterValues {
-  name: "GetParameterValues";
-  parameterNames: string[];
-}
-
-export interface GetParameterAttributes {
-  name: "GetParameterAttributes";
-  parameterNames: string[];
-}
-
-export interface SetParameterValues {
-  name: "SetParameterValues";
-  parameterList: [string, boolean | number | string, string][];
-  parameterKey?: string;
-  DATETIME_MILLISECONDS?: boolean;
-  BOOLEAN_LITERAL?: boolean;
-}
-
-export interface SetParameterAttributes {
-  name: "SetParameterAttributes";
-  parameterList: [string, number, string[]][];
-}
-
-export interface AddObject {
-  name: "AddObject";
-  objectName: string;
-  parameterKey?: string;
-  instanceValues: Record<string, string>;
-}
-
-export interface DeleteObject {
-  name: "DeleteObject";
-  objectName: string;
-  parameterKey?: string;
-}
-
-export interface FactoryReset {
-  name: "FactoryReset";
-  commandKey?: string;
-}
-
-export interface Reboot {
-  name: "Reboot";
-  commandKey?: string;
-}
-
-export interface Download {
-  name: "Download";
-  commandKey: string;
-  instance: string;
-  fileType: string;
-  fileName?: string;
-  url?: string;
-  username?: string;
-  password?: string;
-  fileSize?: number;
-  targetFileName?: string;
-  delaySecods?: number;
-  successUrl?: string;
-  failureUrl?: string;
-}
-
-export interface ChangeDUState {
-  name: "ChangeDUState";
-  commandKey: string;
-  operations: {
-    operationType: "install" | "update" | "uninstall";
-    instance: string;
-    url: string;
-    uuid: string;
-    version: string;
-    executionEnvRef: string;
-    username: string;
-    password: string;
-  }[];
-}
-
-export interface SpvFault {
-  parameterName: string;
-  faultCode: string;
-  faultString: string;
-}
-
-export interface FaultStruct {
-  faultCode: string;
-  faultString: string;
-  setParameterValuesFault?: SpvFault[];
-}
-
-export interface CpeFault {
-  faultCode: "Client" | "Server";
-  faultString: "CWMP fault";
-  detail?: FaultStruct;
-}
-
-export type CpeResponse =
-  | GetParameterNamesResponse
-  | GetParameterValuesResponse
-  | GetParameterAttributesResponse
-  | SetParameterValuesResponse
-  | SetParameterAttributesResponse
-  | AddObjectResponse
-  | DeleteObjectResponse
-  | RebootResponse
-  | FactoryResetResponse
-  | DownloadResponse
-  | ChangeDUStateResponse;
-
-export interface GetParameterNamesResponse {
-  name: "GetParameterNamesResponse";
-  parameterList: [Path, boolean, boolean][];
-}
-
-export interface GetParameterValuesResponse {
-  name: "GetParameterValuesResponse";
-  parameterList: [Path, string | number | boolean, string][];
-}
-
-export interface GetParameterAttributesResponse {
-  name: "GetParameterAttributesResponse";
-  parameterList: [Path, number, string[]][];
-}
-
-export interface SetParameterValuesResponse {
-  name: "SetParameterValuesResponse";
-  status: number;
-}
-
-export interface SetParameterAttributesResponse {
-  name: "SetParameterAttributesResponse";
-}
-
-export interface AddObjectResponse {
-  name: "AddObjectResponse";
-  instanceNumber: string;
-  status: number;
-}
-
-export interface DeleteObjectResponse {
-  name: "DeleteObjectResponse";
-  status: number;
-}
-
-export interface ChangeDUStateResponse {
-  name: "ChangeDUStateResponse";
-}
-
-export interface RebootResponse {
-  name: "RebootResponse";
-}
-
-export interface FactoryResetResponse {
-  name: "FactoryResetResponse";
-}
-
-export interface DownloadResponse {
-  name: "DownloadResponse";
-  status: number;
-  startTime?: number;
-  completeTime?: number;
-}
-
-export type CpeRequest =
-  | InformRequest
-  | TransferCompleteRequest
-  | GetRPCMethodsRequest
-  | RequestDownloadRequest
-  | DUStateChangeCompleteRequest;
-
-export interface InformRequest {
-  name: "Inform";
-  deviceId: {
-    Manufacturer: string;
-    OUI: string;
-    ProductClass?: string;
-    SerialNumber: string;
-  };
-  event: string[];
-  retryCount: number;
-  parameterList: [Path, string | number | boolean, string][];
-}
-
-export interface TransferCompleteRequest {
-  name: "TransferComplete";
-  commandKey?: string;
-  faultStruct?: FaultStruct;
-  startTime?: number;
-  completeTime?: number;
-}
-
-export interface DUStateChangeCompleteRequest {
-  name: "DUStateChangeComplete";
-  commandKey: string;
-  results: {
-    uuid: string;
-    deploymentUnitRef: string;
-    version: string;
-    currentState: string;
-    resolved: boolean;
-    executionUnitRefList: string;
-    fault?: { faultCode: string; faultString: string };
-    startTime?: number;
-    completeTime?: number;
-  }[];
-}
-
-export interface GetRPCMethodsRequest {
-  name: "GetRPCMethods";
-}
-
-export interface RequestDownloadRequest {
-  name: "RequestDownload";
-  fileType: string;
-}
-
-export type AcsResponse =
-  | InformResponse
-  | GetRPCMethodsResponse
-  | TransferCompleteResponse
-  | RequestDownloadResponse
-  | DUStateChangeCompleteResponse;
-
-export interface InformResponse {
-  name: "InformResponse";
-}
-
-export interface GetRPCMethodsResponse {
-  name: "GetRPCMethodsResponse";
-  methodList: string[];
-}
-
-export interface TransferCompleteResponse {
-  name: "TransferCompleteResponse";
-}
-
-export interface RequestDownloadResponse {
-  name: "RequestDownloadResponse";
-}
-export interface DUStateChangeCompleteResponse {
-  name: "DUStateChangeCompleteResponse";
-}
-
-export interface QueryOptions {
-  projection?: any;
-  skip?: number;
-  limit?: number;
-  sort?: {
-    [param: string]: number;
-  };
-}
-
-export interface Declaration {
-  path: Path;
-  pathGet: number;
-  pathSet?: number | [number, number];
-  attrGet?: {
-    object?: number;
-    writable?: number;
-    value?: number;
-    notification?: number;
-    accessList?: number;
-  };
-  attrSet?: {
-    object?: boolean;
-    writable?: boolean;
-    value?: [string | number | boolean, string?];
-    notification?: number;
-    accessList?: string[];
-  };
-  defer: boolean;
-}
-
-export type Clear = [
-  Path,
-  number,
-  {
-    object?: number;
-    writable?: number;
-    value?: number;
-    notification?: number;
-    accessList?: number;
-  }?,
-  number?
-];
-
-export interface Preset {
-  name: string;
-  channel: string;
-  schedule?: { md5: string; duration: number; schedule: any };
-  events?: { [event: string]: boolean };
-  precondition?: Expression;
-  provisions: [string, ...Expression[]][];
-}
-
-export interface Provisions {
-  [name: string]: { md5: string; script: Script };
-}
-
-export interface VirtualParameters {
-  [name: string]: { md5: string; script: Script };
-}
-
-export interface Files {
-  [name: string]: { length: number; md5: string; contentType: string };
-}
-
-export interface Users {
-  [name: string]: { password: string; salt: string; roles: string[] };
-}
-
-export interface Permissions {
-  [role: string]: {
-    [access: number]: {
-      [resource: string]: {
-        access: number;
-        filter: Expression;
-        validate?: Expression;
-      };
-    };
-  };
-}
-
-export type PermissionSet = {
-  [resource: string]: {
-    access: number;
-    validate?: Expression;
-    filter: Expression;
-  };
-}[];
-
-export interface Config {
-  [name: string]: Expression;
-}
-
-export interface UiConfig {
-  filters: Record<string, unknown>;
-  device: Record<string, unknown>;
-  index: Record<string, unknown>;
-  overview: {
-    charts?: Record<string, unknown>;
-    groups?: Record<string, unknown>;
-  };
-  pageSize?: Expression;
-}
-
-export interface SoapMessage {
-  id: string;
-  cwmpVersion: string;
-  sessionTimeout: number;
-  cpeRequest?: CpeRequest;
-  cpeFault?: CpeFault;
-  cpeResponse?: CpeResponse;
-}
-
-export interface ScriptResult {
-  fault: Fault;
-  clear: Clear[];
-  declare: Declaration[];
-  done: boolean;
-  returnValue: any;
-}

lib/ui.ts

@@ -1,275 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { constants } from "zlib";
-import Koa from "koa";
-import Router from "koa-router";
-import * as jwt from "jsonwebtoken";
-import koaStatic from "koa-static";
-import koaCompress from "koa-compress";
-import koaBodyParser from "koa-bodyparser";
-import koaJwt from "koa-jwt";
-import * as config from "./config";
-import api from "./ui/api";
-import Authorizer from "./common/authorizer";
-import * as logger from "./logger";
-import * as localCache from "./local-cache";
-import { PermissionSet } from "./types";
-import { authLocal } from "./ui/api-functions";
-import * as init from "./init";
-import { version as VERSION } from "../package.json";
-import memoize from "./common/memoize";
-
-declare module "koa" {
-  interface Request {
-    body: any;
-  }
-}
-
-const koa = new Koa();
-const router = new Router();
-
-const JWT_SECRET = "" + config.get("UI_JWT_SECRET");
-const JWT_COOKIE = "genieacs-ui-jwt";
-
-const getAuthorizer = memoize(
-  (snapshot: string, rolesStr: string): Authorizer => {
-    const roles: string[] = JSON.parse(rolesStr);
-    const allPermissions = localCache.getPermissions(snapshot);
-    const permissionSets: PermissionSet[] = roles.map((r) =>
-      Object.values(allPermissions[r] || {})
-    );
-    return new Authorizer(permissionSets);
-  }
-);
-
-koa.on("error", async (err, ctx) => {
-  setTimeout(() => {
-    // Ignored errors resulting from aborted requests
-    if (ctx && ctx.req.aborted) return;
-    throw err;
-  });
-});
-
-koa.use(async (ctx, next) => {
-  const configSnapshot = await localCache.getCurrentSnapshot();
-  ctx.state.configSnapshot = configSnapshot;
-  ctx.set("X-Config-Snapshot", configSnapshot);
-  ctx.set("GenieACS-Version", VERSION);
-  return next();
-});
-
-koa.use(
-  koaJwt({
-    secret: JWT_SECRET,
-    passthrough: true,
-    cookie: JWT_COOKIE,
-    isRevoked: async (ctx, token) => {
-      if (token["authMethod"] === "local") {
-        return !localCache.getUsers(ctx.state.configSnapshot)[
-          token["username"]
-        ];
-      }
-
-      return true;
-    },
-  })
-);
-
-koa.use(async (ctx, next) => {
-  let roles: string[] = [];
-
-  if (ctx.state.user && ctx.state.user.username) {
-    let user;
-    if (ctx.state.user.authMethod === "local") {
-      user = localCache.getUsers(ctx.state.configSnapshot)[
-        ctx.state.user.username
-      ];
-    } else {
-      throw new Error("Invalid auth method");
-    }
-    roles = user.roles || [];
-  }
-
-  ctx.state.authorizer = getAuthorizer(
-    ctx.state.configSnapshot,
-    JSON.stringify(roles)
-  );
-
-  return next();
-});
-
-router.post("/login", async (ctx) => {
-  if (!JWT_SECRET) {
-    ctx.status = 500;
-    ctx.body = "UI_JWT_SECRET is not set";
-    logger.error({ message: "UI_JWT_SECRET is not set" });
-    return;
-  }
-
-  const username = ctx.request.body.username;
-  const password = ctx.request.body.password;
-
-  const log = {
-    message: "Log in",
-    context: ctx,
-    username: username,
-    method: null,
-  };
-
-  function success(authMethod): void {
-    log.method = authMethod;
-    const token = jwt.sign({ username, authMethod }, JWT_SECRET);
-    ctx.cookies.set(JWT_COOKIE, token, { sameSite: "lax" });
-    ctx.body = JSON.stringify(token);
-    logger.accessInfo(log);
-  }
-
-  function failure(): void {
-    ctx.status = 400;
-    ctx.body = "Incorrect username or password";
-    log.message += " failed";
-    logger.accessWarn(log);
-  }
-
-  if (await authLocal(ctx.state.configSnapshot, username, password))
-    return void success("local");
-
-  failure();
-});
-
-router.post("/logout", async (ctx) => {
-  ctx.cookies.set(JWT_COOKIE); // Delete cookie
-  ctx.body = "";
-
-  logger.accessInfo({
-    message: "Log out",
-    context: ctx,
-  });
-});
-
-koa.use(async (ctx, next) => {
-  if (ctx.request.type === "application/octet-stream")
-    ctx.disableBodyParser = true;
-
-  return next();
-});
-
-koa.use(koaBodyParser());
-router.use("/api", api.routes(), api.allowedMethods());
-
-router.get("/status", (ctx) => {
-  ctx.body = "OK";
-});
-
-router.get("/init", async (ctx) => {
-  const status = await init.getStatus();
-  if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
-    if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
-    if (!ctx.state.authorizer.hasAccess("permissions", 3))
-      status["users"] = false;
-    if (!ctx.state.authorizer.hasAccess("config", 3)) {
-      status["filters"] = false;
-      status["device"] = false;
-      status["index"] = false;
-      status["overview"] = false;
-    }
-    if (!ctx.state.authorizer.hasAccess("presets", 3))
-      status["presets"] = false;
-    if (!ctx.state.authorizer.hasAccess("provisions", 3))
-      status["presets"] = false;
-  }
-
-  ctx.body = status;
-});
-
-router.post("/init", async (ctx) => {
-  const status = ctx.request.body;
-  if (Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length) {
-    if (!ctx.state.authorizer.hasAccess("users", 3)) status["users"] = false;
-    if (!ctx.state.authorizer.hasAccess("permissions", 3))
-      status["users"] = false;
-    if (!ctx.state.authorizer.hasAccess("config", 3)) {
-      status["filters"] = false;
-      status["device"] = false;
-      status["index"] = false;
-      status["overview"] = false;
-    }
-    if (!ctx.state.authorizer.hasAccess("presets", 3))
-      status["presets"] = false;
-    if (!ctx.state.authorizer.hasAccess("provisions", 3))
-      status["presets"] = false;
-  }
-  await init.seed(status);
-  ctx.body = "";
-});
-
-router.get("/", async (ctx) => {
-  const permissionSets: PermissionSet[] = ctx.state.authorizer.getPermissionSets();
-
-  let wizard = "";
-  if (!Object.keys(localCache.getUsers(ctx.state.configSnapshot)).length)
-    wizard = '<script>window.location.hash = "#!/wizard";</script>';
-
-  ctx.body = `
-  <html>
-    <head>
-      <title>GenieACS</title>
-      <link rel="shortcut icon" type="image/png" href="favicon.png" />
-      <link rel="stylesheet" href="app.css">
-    </head>
-    <body>
-    <noscript>GenieACS UI requires JavaScript to work. Please enable JavaScript in your browser.</noscript>
-      <script>
-        window.clientConfig = ${JSON.stringify({
-          ui: localCache.getUiConfig(ctx.state.configSnapshot),
-        })};
-        window.configSnapshot = ${JSON.stringify(ctx.state.configSnapshot)};
-        window.genieacsVersion = ${JSON.stringify(VERSION)};
-        window.username = ${JSON.stringify(
-          ctx.state.user ? ctx.state.user.username : ""
-        )};
-        window.permissionSets = ${JSON.stringify(permissionSets)};
-      </script>
-      <script type="module" src="app.js"></script>${wizard} 
-    </body>
-  </html>
-  `;
-});
-
-koa.use(
-  koaCompress({
-    gzip: {
-      flush: constants.Z_SYNC_FLUSH,
-    },
-    deflate: {
-      flush: constants.Z_SYNC_FLUSH,
-    },
-    br: {
-      params: {
-        [constants.BROTLI_PARAM_QUALITY]: 5,
-      },
-    },
-  })
-);
-
-koa.use(router.routes());
-koa.use(koaStatic(config.ROOT_DIR + "/public"));
-
-export const listener = koa.callback();

lib/ui/api-functions.ts

@@ -1,211 +0,0 @@
-/**
- * Copyright 2013-2019  GenieACS Inc.
- *
- * This file is part of GenieACS.
- *
- * GenieACS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * GenieACS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { ObjectID } from "mongodb";
-import * as db from "./db";
-import { del } from "../cache";
-import { getUsers } from "../local-cache";
-import { hashPassword } from "../auth";
-import {
-  insertTasks,
-  awaitSessionEnd,
-  awaitSessionStart,
-  connectionRequest,
-  deleteDevice,
-} from "../api-functions";
-import { Task } from "../types";
-
-async function deleteFault(id): Promise<void> {
-  const deviceId = id.split(":", 1)[0];
-  const channel = id.slice(deviceId.length + 1);
-
-  await Promise.all([
-    db.deleteFault(id),
-    channel.startsWith("task_")
-      ? db.deleteTask(new ObjectID(channel.slice(5)))
-      : null,
-  ]);
-
-  await del(`${deviceId}_tasks_faults_operations`);
-}
-
-export async function deleteResource(
-  resource: string,
-  id: string
-): Promise<void> {
-  switch (resource) {
-    case "devices":
-      await deleteDevice(id);
-      break;
-
-    case "files":
-      await db.deleteFile(id);
-      break;
-
-    case "faults":
-      await deleteFault(id);
-      break;
-
-    case "provisions":
-      await db.deleteProvision(id);
-      break;
-
-    case "presets":
-      await db.deletePreset(id);
-      break;
-
-    case "virtualParameters":
-      await db.deleteVirtualParameter(id);
-      break;
-
-    case "config":
-      await db.deleteConfig(id);
-      break;
-
-    case "permissions":
-      await db.deletePermission(id);
-      break;
-
-    case "users":
-      await db.deleteUser(id);
-      break;
-  }
-
-  await del("presets_hash");
-}
-
-export async function postTasks(
-  deviceId: string,
-  tasks: Task[],
-  timeout: number,
-  device: Record<string, { value?: [boolean | number | string, string] }>
-): Promise<{ connectionRequest: string; tasks: any[] }> {
-  for (const task of tasks) {
-    delete task._id;
-    task["device"] = deviceId;
-  }
-
-  tasks = await insertTasks(tasks);
-  const statuses = tasks.map((t) => {
-    return { _id: t._id, status: "pending" };
-  });
-  const lastInform = Date.now();
-  const notInSession = await awaitSessionEnd(deviceId, 30000);
-  if (!notInSession) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  await del(`${deviceId}_tasks_faults_operations`);
-
-  const status = await connectionRequest(deviceId, device);
-
-  if (status) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    return {
-      connectionRequest: status,
-      tasks: statuses,
-    };
-  }
-
-  const sessionStarted = await awaitSessionStart(deviceId, lastInform, timeout);
-  if (!sessionStarted) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    return {
-      connectionRequest: "No contact from CPE",
-      tasks: statuses,
-    };
-  }
-
-  const sessionEnded = await awaitSessionEnd(deviceId, 120000);
-  if (!sessionEnded) {
-    await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-    return {
-      connectionRequest: "Session took too long to complete",
-      tasks: statuses,
-    };
-  }
-
-  const promises = [];
-  for (const s of statuses) {
-    promises.push(db.query("tasks", ["=", ["PARAM", "_id"], s._id]));
-    promises.push(
-      db.query("faults", ["=", ["PARAM", "_id"], `${deviceId}:task_${s._id}`])
-    );
-  }
-
-  const res = await Promise.all(promises);
-  for (const [i, r] of statuses.entries()) {
-    if (res[i * 2].length === 0) {
-      r.status = "done";
-    } else if (res[i * 2 + 1].length === 1) {
-      r.status = "fault";
-      r["fault"] = res[i * 2 + 1][0];
-    }
-  }
-
-  await Promise.all(statuses.map((t) => db.deleteTask(new ObjectID(t._id))));
-
-  return { connectionRequest: "OK", tasks: statuses };
-}
-
-export async function putResource(
-  resource: string,
-  id: string,
-  data: Record<string, unknown>
-): Promise<void> {
-  if (resource === "presets") {
-    await db.putPreset(id, data);
-  } else if (resource === "provisions") {
-    await db.putProvision(id, data);
-  } else if (resource === "virtualParameters") {
-    await db.putVirtualParameter(id, data);
-  } else if (resource === "config") {
-    await db.putConfig(id, data);
-  } else if (resource === "permissions") {
-    await db.putPermission(id, data);
-  } else if (resource === "users") {
-    delete data["password"];
-    delete data["salt"];
-    await db.putUser(id, data);
-  }
-
-  await del("presets_hash");
-}
-
-export function authLocal(
-  snapshot: string,
-  username: string,
-  password: string
-): Promise<boolean> {
-  return new Promise((resolve, reject) => {
-    const users = getUsers(snapshot);
-    const user = users[username];
-    if (!user || !user.password) return void resolve(null);
-    hashPassword(password, user.salt)
-      .then((hash) => {
-        if (hash === user.password) resolve(true);
-        else resolve(false);
-      })
-      .catch(reject);
-  });
-}